Merge branch '332040-handle-ci-includes-exists' into 'master'

Add support for includes:rules:exists in CI configs

See merge request gitlab-org/gitlab!71745

doc/ci/yaml/includes.md

@@ -269,10 +269,11 @@ see this [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos).
 > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) GitLab 14.3.
 > - [Feature flag `ci_include_rules` removed](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.4.
 > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.4.
+> - [Support for `exists` keyword added](https://gitlab.com/gitlab-org/gitlab/-/issues/341511) in GitLab 14.5.
 
 You can use [`rules`](index.md#rules) with `include` to conditionally include other configuration files.
-You can only use [`if` rules](index.md#rulesif) in `include`, and only with [certain variables](#use-variables-with-include).
-`rules` keywords such as `changes` and `exists` are not supported.
+You can only use [`if` rules](index.md#rulesif) and [`exists` rules](index.md#rulesexists) in `include`, and only with
+[certain variables](#use-variables-with-include). `rules` keyword `changes` is not supported.
 
 ```yaml
 include:

doc/user/project/settings/index.md

@@ -177,7 +177,9 @@ audit trail:
 include:  # Execute individual project's configuration (if project contains .gitlab-ci.yml)
   project: '$CI_PROJECT_PATH'
   file: '$CI_CONFIG_PATH'
-  ref: '$CI_COMMIT_REF_NAME'  # Must be defined or MR pipelines always use the use default branch.
+  ref: '$CI_COMMIT_REF_NAME' # Must be defined or MR pipelines always use the use default branch
+  rules:
+    - exists: '$CI_CONFIG_PATH'
 ```
 
 ##### Ensure compliance jobs are always run

lib/gitlab/ci/build/context/base.rb

@@ -5,6 +5,8 @@ module Gitlab
     module Build
       module Context
         class Base
+          include Gitlab::Utils::StrongMemoize
+
           attr_reader :pipeline
 
           def initialize(pipeline)
@@ -15,6 +17,26 @@ module Gitlab
             raise NotImplementedError
           end
 
+          def project
+            pipeline.project
+          end
+
+          def sha
+            pipeline.sha
+          end
+
+          def top_level_worktree_paths
+            strong_memoize(:top_level_worktree_paths) do
+              project.repository.tree(sha).blobs.map(&:path)
+            end
+          end
+
+          def all_worktree_paths
+            strong_memoize(:all_worktree_paths) do
+              project.repository.ls_files(sha)
+            end
+          end
+
           protected
 
           def pipeline_attributes

lib/gitlab/ci/build/rules/rule/clause/exists.rb

@@ -15,19 +15,21 @@ module Gitlab
           @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?))
         end
 
-        def satisfied_by?(pipeline, context)
-          paths = worktree_paths(pipeline)
+        def satisfied_by?(_pipeline, context)
+          paths = worktree_paths(context)
 
           exact_matches?(paths) || pattern_matches?(paths)
         end
 
         private
 
-        def worktree_paths(pipeline)
+        def worktree_paths(context)
+          return unless context.project
+
           if @top_level_only
-            pipeline.top_level_worktree_paths
+            context.top_level_worktree_paths
           else
-            pipeline.all_worktree_paths
+            context.all_worktree_paths
           end
         end
 

lib/gitlab/ci/config/entry/include/rules/rule.rb

@@ -9,9 +9,9 @@ module Gitlab
             include ::Gitlab::Config::Entry::Validatable
             include ::Gitlab::Config::Entry::Attributable
 
-            ALLOWED_KEYS = %i[if].freeze
+            ALLOWED_KEYS = %i[if exists].freeze
 
-            attributes :if
+            attributes :if, :exists
 
             validations do
               validates :config, presence: true

lib/gitlab/ci/config/external/context.rb

@@ -5,6 +5,8 @@ module Gitlab
     class Config
       module External
         class Context
+          include Gitlab::Utils::StrongMemoize
+
           TimeoutError = Class.new(StandardError)
 
           attr_reader :project, :sha, :user, :parent_pipeline, :variables
@@ -22,6 +24,18 @@ module Gitlab
             yield self if block_given?
           end
 
+          def top_level_worktree_paths
+            strong_memoize(:top_level_worktree_paths) do
+              project.repository.tree(sha).blobs.map(&:path)
+            end
+          end
+
+          def all_worktree_paths
+            strong_memoize(:all_worktree_paths) do
+              project.repository.ls_files(sha)
+            end
+          end
+
           def mutate(attrs = {})
             self.class.new(**attrs) do |ctx|
               ctx.expandset = expandset

spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb

@@ -3,10 +3,8 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Exists do
-  describe '#satisfied_by?' do
-    let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.head_commit.sha) }
-
-    subject { described_class.new(globs).satisfied_by?(pipeline, nil) }
+  shared_examples 'an exists rule with a context' do
+    subject { described_class.new(globs).satisfied_by?(pipeline, context) }
 
     it_behaves_like 'a glob matching rule' do
       let(:project) { create(:project, :custom_repo, files: files) }
@@ -24,4 +22,26 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Exists do
       it { is_expected.to be_truthy }
     end
   end
+
+  describe '#satisfied_by?' do
+    let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.head_commit.sha) }
+
+    context 'when context is Build::Context::Build' do
+      it_behaves_like 'an exists rule with a context' do
+        let(:context) { Gitlab::Ci::Build::Context::Build.new(pipeline, sha: 'abc1234') }
+      end
+    end
+
+    context 'when context is Build::Context::Global' do
+      it_behaves_like 'an exists rule with a context' do
+        let(:context) { Gitlab::Ci::Build::Context::Global.new(pipeline, yaml_variables: {}) }
+      end
+    end
+
+    context 'when context is Config::External::Context' do
+      it_behaves_like 'an exists rule with a context' do
+        let(:context) { Gitlab::Ci::Config::External::Context.new(project: project, sha: project.repository.tree.sha) }
+      end
+    end
+  end
 end

spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb

@@ -5,7 +5,7 @@ require 'fast_spec_helper'
 RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules::Rule do
   let(:factory) do
     Gitlab::Config::Entry::Factory.new(described_class)
-      .value(config)
+                                  .value(config)
   end
 
   subject(:entry) { factory.create! }
@@ -25,6 +25,12 @@ RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules::Rule do
       it { is_expected.to be_valid }
     end
 
+    context 'when specifying an exists: clause' do
+      let(:config) { { exists: './this.md' } }
+
+      it { is_expected.to be_valid }
+    end
+
     context 'using a list of multiple expressions' do
       let(:config) { { if: ['$MY_VAR == "this"', '$YOUR_VAR == "that"'] } }
 
@@ -86,5 +92,13 @@ RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules::Rule do
         expect(subject).to eq(if: '$THIS || $THAT')
       end
     end
+
+    context 'when specifying an exists: clause' do
+      let(:config) { { exists: './test.md' } }
+
+      it 'returns the config' do
+        expect(subject).to eq(exists: './test.md')
+      end
+    end
   end
 end

spec/lib/gitlab/ci/config/external/processor_spec.rb

@@ -406,7 +406,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
     context 'when rules defined' do
       context 'when a rule is invalid' do
         let(:values) do
-          { include: [{ local: 'builds.yml', rules: [{ exists: ['$MY_VAR'] }] }] }
+          { include: [{ local: 'builds.yml', rules: [{ changes: ['$MY_VAR'] }] }] }
         end
 
         it 'raises IncludeError' do

spec/lib/gitlab/ci/config/external/rules_spec.rb

 # frozen_string_literal: true
 
-require 'fast_spec_helper'
+require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Config::External::Rules do
   let(:rule_hashes) {}
@@ -32,6 +32,26 @@ RSpec.describe Gitlab::Ci::Config::External::Rules do
       end
     end
 
+    context 'when there is a rule with exists' do
+      let(:project) { create(:project, :repository) }
+      let(:context) { double(project: project, sha: project.repository.tree.sha, top_level_worktree_paths: ['test.md']) }
+      let(:rule_hashes) { [{ exists: 'Dockerfile' }] }
+
+      context 'when the file does not exist' do
+        it { is_expected.to eq(false) }
+      end
+
+      context 'when the file exists' do
+        let(:context) { double(project: project, sha: project.repository.tree.sha, top_level_worktree_paths: ['Dockerfile']) }
+
+        before do
+          project.repository.create_file(project.owner, 'Dockerfile', "commit", message: 'test', branch_name: "master")
+        end
+
+        it { is_expected.to eq(true) }
+      end
+    end
+
     context 'when there is a rule with if and when' do
       let(:rule_hashes) { [{ if: '$MY_VAR == "hello"', when: 'on_success' }] }
 
@@ -41,12 +61,12 @@ RSpec.describe Gitlab::Ci::Config::External::Rules do
       end
     end
 
-    context 'when there is a rule with exists' do
-      let(:rule_hashes) { [{ exists: ['$MY_VAR'] }] }
+    context 'when there is a rule with changes' do
+      let(:rule_hashes) { [{ changes: ['$MY_VAR'] }] }
 
       it 'raises an error' do
         expect { result }.to raise_error(described_class::InvalidIncludeRulesError,
-                                         'invalid include rule: {:exists=>["$MY_VAR"]}')
+                                         'invalid include rule: {:changes=>["$MY_VAR"]}')
       end
     end
   end

spec/lib/gitlab/ci/config_spec.rb

@@ -597,7 +597,7 @@ RSpec.describe Gitlab::Ci::Config do
             job1: {
               script: ["echo 'hello from main file'"],
               variables: {
-                VARIABLE_DEFINED_IN_MAIN_FILE: 'some value'
+               VARIABLE_DEFINED_IN_MAIN_FILE: 'some value'
               }
             }
           })
@@ -727,30 +727,70 @@ RSpec.describe Gitlab::Ci::Config do
       end
     end
 
-    context "when an 'include' has rules with a project variable" do
-      let(:gitlab_ci_yml) do
-        <<~HEREDOC
-        include:
-          - local: #{local_location}
-            rules:
-              - if: $CI_PROJECT_ID == "#{project_id}"
-        image: ruby:2.7
-        HEREDOC
-      end
+    context "when an 'include' has rules" do
+      context "when the rule is an if" do
+        let(:gitlab_ci_yml) do
+          <<~HEREDOC
+          include:
+            - local: #{local_location}
+              rules:
+                - if: $CI_PROJECT_ID == "#{project_id}"
+          image: ruby:2.7
+          HEREDOC
+        end
 
-      context 'when the rules condition is satisfied' do
-        let(:project_id) { project.id }
+        context 'when the rules condition is satisfied' do
+          let(:project_id) { project.id }
 
-        it 'includes the file' do
-          expect(config.to_hash).to include(local_location_hash)
+          it 'includes the file' do
+            expect(config.to_hash).to include(local_location_hash)
+          end
+        end
+
+        context 'when the rules condition is satisfied' do
+          let(:project_id) { non_existing_record_id }
+
+          it 'does not include the file' do
+            expect(config.to_hash).not_to include(local_location_hash)
+          end
         end
       end
 
-      context 'when the rules condition is satisfied' do
-        let(:project_id) { non_existing_record_id }
+      context "when the rule is an exists" do
+        let(:gitlab_ci_yml) do
+          <<~HEREDOC
+          include:
+            - local: #{local_location}
+              rules:
+                - exists: "#{filename}"
+          image: ruby:2.7
+          HEREDOC
+        end
 
-        it 'does not include the file' do
-          expect(config.to_hash).not_to include(local_location_hash)
+        before do
+          project.repository.create_file(
+            project.creator,
+            'my_builds.yml',
+            local_file_content,
+            message: 'Add my_builds.yml',
+            branch_name: '12345'
+          )
+        end
+
+        context 'when the exists file does not exist' do
+          let(:filename) { 'not_a_real_file.md' }
+
+          it 'does not include the file' do
+            expect(config.to_hash).not_to include(local_location_hash)
+          end
+        end
+
+        context 'when the exists file does exist' do
+          let(:filename) { 'my_builds.yml' }
+
+          it 'does include the file' do
+            expect(config.to_hash).to include(local_location_hash)
+          end
         end
       end
     end