Merge branch '34272-support-variables-in-rules-changes' into 'master'

Support variables in rules:changes

See merge request gitlab-org/gitlab!45037

config/feature_flags/development/ci_variable_expansion_in_rules_changes.yml

+---
+name: ci_variable_expansion_in_rules_changes
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45037
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/267192
+type: development
+group: group::pipeline authoring
+default_enabled: false

doc/ci/yaml/README.md

@@ -244,7 +244,7 @@ a preconfigured `workflow: rules` entry.
 `workflow: rules` accepts these keywords:
 
 - [`if`](#rulesif): Check this rule to determine when to run a pipeline.
-- [`when`](#when): Specify what to do when the `if` rule evaluates to true. 
+- [`when`](#when): Specify what to do when the `if` rule evaluates to true.
   - To run a pipeline, set to `always`.
   - To prevent pipelines from running, set to `never`.
 
@@ -1347,6 +1347,53 @@ Tag pipelines, scheduled pipelines, and so on do **not** have a Git `push` event
 associated with them. A `rules: changes` job is **always** added to those pipeline
 if there is no `if:` statement that limits the job to branch or merge request pipelines.
 
+##### Variables in `rules:changes`
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34272) in GitLab 13.6.
+> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-variables-support-in-ruleschanges). **(CORE ONLY)**
+
+CAUTION: **Warning:**
+This feature might not be available to you. Check the **version history** note above for details.
+
+Environment variables can be used in `rules:changes` expressions to determine when
+to add jobs to a pipeline:
+
+```yaml
+docker build:
+  variables:
+    DOCKERFILES_DIR: 'path/to/files/'
+  script: docker build -t my-image:$CI_COMMIT_REF_SLUG .
+  rules:
+    - changes:
+        - $DOCKERFILES_DIR/*
+```
+
+The `$` character can be used for both variables and paths. For example, if the
+`$DOCKERFILES_DIR` variable exists, its value is used. If it does not exist, the
+`$` is interpreted as being part of a path.
+
+###### Enable or disable variables support in `rules:changes` **(CORE ONLY)**
+
+Variables support in `rules:changes` is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:ci_variable_expansion_in_rules_changes)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:ci_variable_expansion_in_rules_changes)
+```
+
 #### `rules:exists`
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4.

lib/expand_variables.rb

 # frozen_string_literal: true
 
 module ExpandVariables
+  VARIABLES_REGEXP = /\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/.freeze
+
   class << self
     def expand(value, variables)
+      replace_with(value, variables) do |vars_hash, last_match|
+        match_or_blank_value(vars_hash, last_match)
+      end
+    end
+
+    def expand_existing(value, variables)
+      replace_with(value, variables) do |vars_hash, last_match|
+        match_or_original_value(vars_hash, last_match)
+      end
+    end
+
+    private
+
+    def replace_with(value, variables)
       variables_hash = nil
 
-      value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do
+      value.gsub(VARIABLES_REGEXP) do
         variables_hash ||= transform_variables(variables)
-        variables_hash[Regexp.last_match(1) || Regexp.last_match(2)]
+        yield(variables_hash, Regexp.last_match)
       end
     end
 
-    private
+    def match_or_blank_value(variables, last_match)
+      variables[last_match[1] || last_match[2]]
+    end
+
+    def match_or_original_value(variables, last_match)
+      match_or_blank_value(variables, last_match) || last_match[0]
+    end
 
     def transform_variables(variables)
       # Lazily initialise variables

lib/gitlab/ci/build/rules/rule/clause/changes.rb

@@ -11,12 +11,22 @@ module Gitlab
         def satisfied_by?(pipeline, context)
           return true if pipeline.modified_paths.nil?
 
+          expanded_globs = expand_globs(pipeline, context)
           pipeline.modified_paths.any? do |path|
-            @globs.any? do |glob|
+            expanded_globs.any? do |glob|
               File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
             end
           end
         end
+
+        def expand_globs(pipeline, context)
+          return @globs unless ::Feature.enabled?(:ci_variable_expansion_in_rules_changes, pipeline.project)
+          return @globs unless context
+
+          @globs.map do |glob|
+            ExpandVariables.expand_existing(glob, context.variables)
+          end
+        end
       end
     end
   end

spec/lib/expand_variables_spec.rb

@@ -3,106 +3,132 @@
 require 'spec_helper'
 
 RSpec.describe ExpandVariables do
+  shared_examples 'common variable expansion' do |expander|
+    using RSpec::Parameterized::TableSyntax
+
+    where do
+      {
+        "no expansion": {
+          value: 'key',
+          result: 'key',
+          variables: []
+        },
+        "simple expansion": {
+          value: 'key$variable',
+          result: 'keyvalue',
+          variables: [
+            { key: 'variable', value: 'value' }
+          ]
+        },
+        "simple with hash of variables": {
+          value: 'key$variable',
+          result: 'keyvalue',
+          variables: {
+            'variable' => 'value'
+          }
+        },
+        "complex expansion": {
+          value: 'key${variable}',
+          result: 'keyvalue',
+          variables: [
+            { key: 'variable', value: 'value' }
+          ]
+        },
+        "simple expansions": {
+          value: 'key$variable$variable2',
+          result: 'keyvalueresult',
+          variables: [
+            { key: 'variable', value: 'value' },
+            { key: 'variable2', value: 'result' }
+          ]
+        },
+        "complex expansions": {
+          value: 'key${variable}${variable2}',
+          result: 'keyvalueresult',
+          variables: [
+            { key: 'variable', value: 'value' },
+            { key: 'variable2', value: 'result' }
+          ]
+        },
+        "out-of-order expansion": {
+          value: 'key$variable2$variable',
+          result: 'keyresultvalue',
+          variables: [
+            { key: 'variable', value: 'value' },
+            { key: 'variable2', value: 'result' }
+          ]
+        },
+        "out-of-order complex expansion": {
+          value: 'key${variable2}${variable}',
+          result: 'keyresultvalue',
+          variables: [
+            { key: 'variable', value: 'value' },
+            { key: 'variable2', value: 'result' }
+          ]
+        },
+        "review-apps expansion": {
+          value: 'review/$CI_COMMIT_REF_NAME',
+          result: 'review/feature/add-review-apps',
+          variables: [
+            { key: 'CI_COMMIT_REF_NAME', value: 'feature/add-review-apps' }
+          ]
+        },
+        "do not lazily access variables when no expansion": {
+          value: 'key',
+          result: 'key',
+          variables: -> { raise NotImplementedError }
+        },
+        "lazily access variables": {
+          value: 'key$variable',
+          result: 'keyvalue',
+          variables: -> { [{ key: 'variable', value: 'value' }] }
+        }
+      }
+    end
+
+    with_them do
+      subject { expander.call(value, variables) }
+
+      it { is_expected.to eq(result) }
+    end
+  end
+
   describe '#expand' do
     context 'table tests' do
-      using RSpec::Parameterized::TableSyntax
-
-      where do
-        {
-          "no expansion": {
-            value: 'key',
-            result: 'key',
-            variables: []
-          },
-          "missing variable": {
-            value: 'key$variable',
-            result: 'key',
-            variables: []
-          },
-          "simple expansion": {
-            value: 'key$variable',
-            result: 'keyvalue',
-            variables: [
-              { key: 'variable', value: 'value' }
-            ]
-          },
-          "simple with hash of variables": {
-            value: 'key$variable',
-            result: 'keyvalue',
-            variables: {
-              'variable' => 'value'
+      it_behaves_like 'common variable expansion', described_class.method(:expand)
+
+      context 'with missing variables' do
+        using RSpec::Parameterized::TableSyntax
+
+        where do
+          {
+            "missing variable": {
+              value: 'key$variable',
+              result: 'key',
+              variables: []
+            },
+            "complex expansions with missing variable": {
+              value: 'key${variable}${variable2}',
+              result: 'keyvalue',
+              variables: [
+                { key: 'variable', value: 'value' }
+              ]
+            },
+            "complex expansions with missing variable for Windows": {
+              value: 'key%variable%%variable2%',
+              result: 'keyvalue',
+              variables: [
+                { key: 'variable', value: 'value' }
+              ]
             }
-          },
-          "complex expansion": {
-            value: 'key${variable}',
-            result: 'keyvalue',
-            variables: [
-              { key: 'variable', value: 'value' }
-            ]
-          },
-          "simple expansions": {
-            value: 'key$variable$variable2',
-            result: 'keyvalueresult',
-            variables: [
-              { key: 'variable', value: 'value' },
-              { key: 'variable2', value: 'result' }
-            ]
-          },
-          "complex expansions": {
-            value: 'key${variable}${variable2}',
-            result: 'keyvalueresult',
-            variables: [
-              { key: 'variable', value: 'value' },
-              { key: 'variable2', value: 'result' }
-            ]
-          },
-          "complex expansions with missing variable": {
-            value: 'key${variable}${variable2}',
-            result: 'keyvalue',
-            variables: [
-              { key: 'variable', value: 'value' }
-            ]
-          },
-          "out-of-order expansion": {
-            value: 'key$variable2$variable',
-            result: 'keyresultvalue',
-            variables: [
-              { key: 'variable', value: 'value' },
-              { key: 'variable2', value: 'result' }
-            ]
-          },
-          "out-of-order complex expansion": {
-            value: 'key${variable2}${variable}',
-            result: 'keyresultvalue',
-            variables: [
-              { key: 'variable', value: 'value' },
-              { key: 'variable2', value: 'result' }
-            ]
-          },
-          "review-apps expansion": {
-            value: 'review/$CI_COMMIT_REF_NAME',
-            result: 'review/feature/add-review-apps',
-            variables: [
-              { key: 'CI_COMMIT_REF_NAME', value: 'feature/add-review-apps' }
-            ]
-          },
-          "do not lazily access variables when no expansion": {
-            value: 'key',
-            result: 'key',
-            variables: -> { raise NotImplementedError }
-          },
-          "lazily access variables": {
-            value: 'key$variable',
-            result: 'keyvalue',
-            variables: -> { [{ key: 'variable', value: 'value' }] }
           }
-        }
-      end
+        end
 
-      with_them do
-        subject { ExpandVariables.expand(value, variables) }
+        with_them do
+          subject { ExpandVariables.expand(value, variables) }
 
-        it { is_expected.to eq(result) }
+          it { is_expected.to eq(result) }
+        end
       end
     end
 
@@ -132,4 +158,70 @@ RSpec.describe ExpandVariables do
       end
     end
   end
+
+  describe '#expand_existing' do
+    context 'table tests' do
+      it_behaves_like 'common variable expansion', described_class.method(:expand_existing)
+
+      context 'with missing variables' do
+        using RSpec::Parameterized::TableSyntax
+
+        where do
+          {
+            "missing variable": {
+              value: 'key$variable',
+              result: 'key$variable',
+              variables: []
+            },
+            "complex expansions with missing variable": {
+              value: 'key${variable}${variable2}',
+              result: 'keyvalue${variable2}',
+              variables: [
+                { key: 'variable', value: 'value' }
+              ]
+            },
+            "complex expansions with missing variable for Windows": {
+              value: 'key%variable%%variable2%',
+              result: 'keyvalue%variable2%',
+              variables: [
+                { key: 'variable', value: 'value' }
+              ]
+            }
+          }
+        end
+
+        with_them do
+          subject { ExpandVariables.expand_existing(value, variables) }
+
+          it { is_expected.to eq(result) }
+        end
+      end
+    end
+
+    context 'lazily inits variables' do
+      let(:variables) { -> { [{ key: 'variable', value: 'result' }] } }
+
+      subject { described_class.expand_existing(value, variables) }
+
+      context 'when expanding variable' do
+        let(:value) { 'key$variable$variable2' }
+
+        it 'calls block at most once' do
+          expect(variables).to receive(:call).once.and_call_original
+
+          is_expected.to eq('keyresult$variable2')
+        end
+      end
+
+      context 'when no expansion is needed' do
+        let(:value) { 'key' }
+
+        it 'does not call block' do
+          expect(variables).not_to receive(:call)
+
+          is_expected.to eq('key')
+        end
+      end
+    end
+  end
 end

spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb

@@ -13,5 +13,41 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do
 
       subject { described_class.new(globs).satisfied_by?(pipeline, nil) }
     end
+
+    context 'when using variable expansion' do
+      let(:pipeline) { build(:ci_pipeline) }
+      let(:modified_paths) { ['helm/test.txt'] }
+      let(:globs) { ['$HELM_DIR/**/*'] }
+      let(:context) { double('context') }
+      let(:variables) { [] }
+
+      subject { described_class.new(globs).satisfied_by?(pipeline, context) }
+
+      before do
+        allow(pipeline).to receive(:modified_paths).and_return(modified_paths)
+        allow(context).to receive(:variables).and_return(variables)
+      end
+
+      context 'when context is nil' do
+        let(:context) {}
+
+        it { is_expected.to be_falsey }
+      end
+
+      context 'when context has the specified variables' do
+        let(:variables) do
+          [{ key: "HELM_DIR", value: "helm", public: true }]
+        end
+
+        it { is_expected.to be_truthy }
+      end
+
+      context 'when variable expansion does not match' do
+        let(:globs) { ['path/with/$in/it/*'] }
+        let(:modified_paths) { ['path/with/$in/it/file.txt'] }
+
+        it { is_expected.to be_truthy }
+      end
+    end
   end
 end

spec/services/ci/create_pipeline_service_spec.rb

@@ -1870,6 +1870,12 @@ RSpec.describe Ci::CreatePipelineService do
                 - changes:
                   - README.md
                   allow_failure: true
+
+            README:
+              script: "I use variables for changes!"
+              rules:
+                - changes:
+                  - $CI_JOB_NAME*
           EOY
         end
 
@@ -1879,10 +1885,10 @@ RSpec.describe Ci::CreatePipelineService do
               .to receive(:modified_paths).and_return(%w[README.md])
           end
 
-          it 'creates two jobs' do
+          it 'creates five jobs' do
             expect(pipeline).to be_persisted
             expect(build_names)
-              .to contain_exactly('regular-job', 'rules-job', 'delayed-job', 'negligible-job')
+              .to contain_exactly('regular-job', 'rules-job', 'delayed-job', 'negligible-job', 'README')
           end
 
           it 'sets when: for all jobs' do