Merge branch '344937-expose-include-links' into 'master'

Collect included files on CI pipeline creation

See merge request gitlab-org/gitlab!81982

lib/gitlab/ci/config.rb

@@ -82,7 +82,13 @@ module Gitlab
       end
 
       def included_templates
-        @context.expandset.filter_map { |i| i[:template] }
+        @context.includes.filter_map { |i| i[:location] if i[:type] == :template }
+      end
+
+      def metadata
+        {
+          includes: @context.includes
+        }
       end
 
       private

lib/gitlab/ci/config/external/context.rb

@@ -70,16 +70,20 @@ module Gitlab
             }
           end
 
-          def mask_variables_from(location)
-            variables.reduce(location.dup) do |loc, variable|
+          def mask_variables_from(string)
+            variables.reduce(string.dup) do |str, variable|
               if variable[:masked]
-                Gitlab::Ci::MaskSecret.mask!(loc, variable[:value])
+                Gitlab::Ci::MaskSecret.mask!(str, variable[:value])
               else
-                loc
+                str
               end
             end
           end
 
+          def includes
+            expandset.map(&:metadata)
+          end
+
           protected
 
           attr_writer :expandset, :execution_deadline, :logger

lib/gitlab/ci/config/external/file/artifact.rb

@@ -28,6 +28,14 @@ module Gitlab
               end
             end
 
+            def metadata
+              super.merge(
+                type: :artifact,
+                location: masked_location,
+                extra: { job_name: masked_job_name }
+              )
+            end
+
             private
 
             def project
@@ -52,7 +60,7 @@ module Gitlab
               end
 
               unless artifact_job.present?
-                errors.push("Job `#{job_name}` not found in parent pipeline or does not have artifacts!")
+                errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!")
                 return false
               end
 
@@ -80,6 +88,12 @@ module Gitlab
                 parent_pipeline: context.parent_pipeline
               }
             end
+
+            def masked_job_name
+              strong_memoize(:masked_job_name) do
+                context.mask_variables_from(job_name)
+              end
+            end
           end
         end
       end

lib/gitlab/ci/config/external/file/base.rb

@@ -55,6 +55,21 @@ module Gitlab
               end
             end
 
+            def metadata
+              {
+                context_project: context.project&.full_path,
+                context_sha: context.sha
+              }
+            end
+
+            def eql?(other)
+              other.hash == hash
+            end
+
+            def hash
+              [params, context.project&.full_path, context.sha].hash
+            end
+
             protected
 
             def expanded_content_hash

lib/gitlab/ci/config/external/file/local.rb

@@ -19,6 +19,14 @@ module Gitlab
               strong_memoize(:content) { fetch_local_content }
             end
 
+            def metadata
+              super.merge(
+                type: :local,
+                location: masked_location,
+                extra: {}
+              )
+            end
+
             private
 
             def validate_content!

lib/gitlab/ci/config/external/file/project.rb

@@ -27,17 +27,25 @@ module Gitlab
               strong_memoize(:content) { fetch_local_content }
             end
 
+            def metadata
+              super.merge(
+                type: :file,
+                location: masked_location,
+                extra: { project: masked_project_name, ref: masked_ref_name }
+              )
+            end
+
             private
 
             def validate_content!
               if !can_access_local_content?
-                errors.push("Project `#{project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.")
+                errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.")
               elsif sha.nil?
-                errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!")
+                errors.push("Project `#{masked_project_name}` reference `#{masked_ref_name}` does not exist!")
               elsif content.nil?
-                errors.push("Project `#{project_name}` file `#{masked_location}` does not exist!")
+                errors.push("Project `#{masked_project_name}` file `#{masked_location}` does not exist!")
               elsif content.blank?
-                errors.push("Project `#{project_name}` file `#{masked_location}` is empty!")
+                errors.push("Project `#{masked_project_name}` file `#{masked_location}` is empty!")
               end
             end
 
@@ -76,6 +84,18 @@ module Gitlab
                 variables: context.variables
               }
             end
+
+            def masked_project_name
+              strong_memoize(:masked_project_name) do
+                context.mask_variables_from(project_name)
+              end
+            end
+
+            def masked_ref_name
+              strong_memoize(:masked_ref_name) do
+                context.mask_variables_from(ref_name)
+              end
+            end
           end
         end
       end

lib/gitlab/ci/config/external/file/remote.rb

@@ -18,6 +18,14 @@ module Gitlab
               strong_memoize(:content) { fetch_remote_content }
             end
 
+            def metadata
+              super.merge(
+                type: :remote,
+                location: masked_location,
+                extra: {}
+              )
+            end
+
             private
 
             def validate_location!

lib/gitlab/ci/config/external/file/template.rb

@@ -20,6 +20,14 @@ module Gitlab
               strong_memoize(:content) { fetch_template_content }
             end
 
+            def metadata
+              super.merge(
+                type: :template,
+                location: masked_location,
+                extra: {}
+              )
+            end
+
             private
 
             def validate_location!

lib/gitlab/ci/config/external/mapper.rb

@@ -48,7 +48,6 @@ module Gitlab
               .flat_map(&method(:expand_project_files))
               .flat_map(&method(:expand_wildcard_paths))
               .map(&method(:expand_variables))
-              .each(&method(:verify_duplicates!))
               .map(&method(:select_first_matching))
               .each(&method(:verify!))
           end
@@ -112,26 +111,6 @@ module Gitlab
             end
           end
 
-          def verify_duplicates!(location)
-            logger.instrument(:config_mapper_verify) do
-              verify_max_includes_and_add_location!(location)
-            end
-          end
-
-          def verify_max_includes_and_add_location!(location)
-            if expandset.count >= MAX_INCLUDES
-              raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!"
-            end
-
-            # Scope location to context to allow support of
-            # relative includes
-            scoped_location = location.merge(
-              context_project: context.project,
-              context_sha: context.sha)
-
-            expandset.add(scoped_location)
-          end
-
           def select_first_matching(location)
             logger.instrument(:config_mapper_select) do
               select_first_matching_without_instrumentation(location)
@@ -149,7 +128,15 @@ module Gitlab
           end
 
           def verify!(location_object)
+            verify_max_includes!
             location_object.validate!
+            expandset.add(location_object)
+          end
+
+          def verify_max_includes!
+            if expandset.count >= MAX_INCLUDES
+              raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!"
+            end
           end
 
           def expand_variables(data)

spec/lib/gitlab/ci/config/external/file/artifact_spec.rb

@@ -4,8 +4,9 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Config::External::File::Artifact do
   let(:parent_pipeline) { create(:ci_pipeline) }
+  let(:variables) {}
   let(:context) do
-    Gitlab::Ci::Config::External::Context.new(parent_pipeline: parent_pipeline)
+    Gitlab::Ci::Config::External::Context.new(variables: variables, parent_pipeline: parent_pipeline)
   end
 
   let(:external_file) { described_class.new(params, context) }
@@ -170,6 +171,58 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact do
           end
         end
       end
+
+      context 'when job is provided as a variable' do
+        let(:variables) do
+          Gitlab::Ci::Variables::Collection.new([
+            { key: 'VAR1', value: 'a_secret_variable_value', masked: true }
+          ])
+        end
+
+        let(:params) { { artifact: 'generated.yml', job: 'a_secret_variable_value' } }
+
+        context 'when job does not exist in the parent pipeline' do
+          let(:expected_error) do
+            'Job `xxxxxxxxxxxxxxxxxxxxxxx` not found in parent pipeline or does not have artifacts!'
+          end
+
+          it_behaves_like 'is invalid'
+        end
+      end
+    end
+  end
+
+  describe '#metadata' do
+    let(:params) { { artifact: 'generated.yml' } }
+
+    subject(:metadata) { external_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: nil,
+        context_sha: nil,
+        type: :artifact,
+        location: 'generated.yml',
+        extra: { job_name: nil }
+      )
+    }
+
+    context 'when job name includes a masked variable' do
+      let(:variables) do
+        Gitlab::Ci::Variables::Collection.new([{ key: 'VAR1', value: 'a_secret_variable_value', masked: true }])
+      end
+
+      let(:params) { { artifact: 'generated.yml', job: 'a_secret_variable_value' } }
+
+      it {
+        is_expected.to eq(
+          context_project: nil,
+          context_sha: nil,
+          type: :artifact,
+          location: 'generated.yml',
+          extra: { job_name: 'xxxxxxxxxxxxxxxxxxxxxxx' }
+        )
+      }
     end
   end
 end

spec/lib/gitlab/ci/config/external/file/base_spec.rb

@@ -112,4 +112,52 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base do
       end
     end
   end
+
+  describe '#metadata' do
+    let(:location) { 'some/file/config.yml' }
+
+    subject(:metadata) { file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: nil,
+        context_sha: 'HEAD'
+      )
+    }
+  end
+
+  describe '#eql?' do
+    let(:location) { 'some/file/config.yml' }
+
+    subject(:eql) { file.eql?(other_file) }
+
+    context 'when the other file has the same params' do
+      let(:other_file) { test_class.new(location, context) }
+
+      it { is_expected.to eq(true) }
+    end
+
+    context 'when the other file has not the same params' do
+      let(:other_file) { test_class.new('some/other/file', context) }
+
+      it { is_expected.to eq(false) }
+    end
+  end
+
+  describe '#hash' do
+    let(:location) { 'some/file/config.yml' }
+
+    subject(:filehash) { file.hash }
+
+    context 'with a project' do
+      let(:project) { create(:project) }
+      let(:context_params) { { project: project, sha: 'HEAD', variables: variables } }
+
+      it { is_expected.to eq([location, project.full_path, 'HEAD'].hash) }
+    end
+
+    context 'without a project' do
+      it { is_expected.to eq([location, nil, 'HEAD'].hash) }
+    end
+  end
 end

spec/lib/gitlab/ci/config/external/file/local_spec.rb

@@ -187,4 +187,20 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do
       end
     end
   end
+
+  describe '#metadata' do
+    let(:location) { '/lib/gitlab/ci/templates/existent-file.yml' }
+
+    subject(:metadata) { local_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: project.full_path,
+        context_sha: '12345',
+        type: :local,
+        location: location,
+        extra: {}
+      )
+    }
+  end
 end

spec/lib/gitlab/ci/config/external/file/project_spec.rb

@@ -160,6 +160,23 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do
         expect(project_file.error_message).to include('Included file `/invalid-file` does not have YAML extension!')
       end
     end
+
+    context 'when non-existing project is used with a masked variable' do
+      let(:variables) do
+        Gitlab::Ci::Variables::Collection.new([
+          { key: 'VAR1', value: 'a_secret_variable_value', masked: true }
+        ])
+      end
+
+      let(:params) do
+        { project: 'a_secret_variable_value', file: '/file.yml' }
+      end
+
+      it 'returns false with masked project name' do
+        expect(valid?).to be_falsy
+        expect(project_file.error_message).to include("Project `xxxxxxxxxxxxxxxxxxxxxxx` not found or access denied!")
+      end
+    end
   end
 
   describe '#expand_context' do
@@ -177,6 +194,45 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do
     end
   end
 
+  describe '#metadata' do
+    let(:params) do
+      { project: project.full_path, file: '/file.yml' }
+    end
+
+    subject(:metadata) { project_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: context_project.full_path,
+        context_sha: '12345',
+        type: :file,
+        location: '/file.yml',
+        extra: { project: project.full_path, ref: 'HEAD' }
+      )
+    }
+
+    context 'when project name and ref include masked variables' do
+      let(:variables) do
+        Gitlab::Ci::Variables::Collection.new([
+          { key: 'VAR1', value: 'a_secret_variable_value1', masked: true },
+          { key: 'VAR2', value: 'a_secret_variable_value2', masked: true }
+        ])
+      end
+
+      let(:params) { { project: 'a_secret_variable_value1', ref: 'a_secret_variable_value2', file: '/file.yml' } }
+
+      it {
+        is_expected.to eq(
+          context_project: context_project.full_path,
+          context_sha: '12345',
+          type: :file,
+          location: '/file.yml',
+          extra: { project: 'xxxxxxxxxxxxxxxxxxxxxxxx', ref: 'xxxxxxxxxxxxxxxxxxxxxxxx' }
+        )
+      }
+    end
+  end
+
   private
 
   def stub_project_blob(ref, path)

spec/lib/gitlab/ci/config/external/file/remote_spec.rb

@@ -199,4 +199,22 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote do
       is_expected.to be_empty
     end
   end
+
+  describe '#metadata' do
+    before do
+      stub_full_request(location).to_return(body: remote_file_content)
+    end
+
+    subject(:metadata) { remote_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: nil,
+        context_sha: '12345',
+        type: :remote,
+        location: 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.xxxxxxxxxxx.yml',
+        extra: {}
+      )
+    }
+  end
 end

spec/lib/gitlab/ci/config/external/file/template_spec.rb

@@ -114,4 +114,18 @@ RSpec.describe Gitlab::Ci::Config::External::File::Template do
       is_expected.to be_empty
     end
   end
+
+  describe '#metadata' do
+    subject(:metadata) { template_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: project.full_path,
+        context_sha: '12345',
+        type: :template,
+        location: template,
+        extra: {}
+      )
+    }
+  end
 end

spec/lib/gitlab/ci/config/external/mapper_spec.rb

@@ -21,6 +21,8 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
     HEREDOC
   end
 
+  subject(:mapper) { described_class.new(values, context) }
+
   before do
     stub_full_request(remote_url).to_return(body: file_content)
 
@@ -30,7 +32,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
   end
 
   describe '#process' do
-    subject { described_class.new(values, context).process }
+    subject(:process) { mapper.process }
 
     context "when single 'include' keyword is defined" do
       context 'when the string is a local file' do
@@ -189,7 +191,12 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
       end
 
       it 'does not raise an exception' do
-        expect { subject }.not_to raise_error
+        expect { process }.not_to raise_error
+      end
+
+      it 'has expanset with one' do
+        process
+        expect(mapper.expandset.size).to eq(1)
       end
     end
 
@@ -385,5 +392,27 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
         end
       end
     end
+
+    context "when locations are same after masking variables" do
+      let(:variables) do
+        Gitlab::Ci::Variables::Collection.new([
+          { 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file1', 'masked' => true },
+          { 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file2', 'masked' => true }
+        ])
+      end
+
+      let(:values) do
+        { include: [
+          { 'local' => 'hello/secret-file1.yml' },
+          { 'local' => 'hello/secret-file2.yml' }
+        ],
+        image: 'ruby:2.7' }
+      end
+
+      it 'has expanset with two' do
+        process
+        expect(mapper.expandset.size).to eq(2)
+      end
+    end
   end
 end

spec/lib/gitlab/ci/config/external/processor_spec.rb

@@ -22,7 +22,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
   end
 
   describe "#perform" do
-    subject { processor.perform }
+    subject(:perform) { processor.perform }
 
     context 'when no external files defined' do
       let(:values) { { image: 'image:1.0' } }
@@ -262,6 +262,18 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
 
           expect(process_obs_count).to eq(3)
         end
+
+        it 'stores includes' do
+          perform
+
+          expect(context.includes).to contain_exactly(
+            { type: :local, location: '/local/file.yml', extra: {}, context_project: project.full_path, context_sha: '12345' },
+            { type: :template, location: 'Ruby.gitlab-ci.yml', extra: {}, context_project: project.full_path, context_sha: '12345' },
+            { type: :remote, location: 'http://my.domain.com/config.yml', extra: {}, context_project: project.full_path, context_sha: '12345' },
+            { type: :file, location: '/templates/my-workflow.yml', extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, context_sha: '12345' },
+            { type: :local, location: '/templates/my-build.yml', extra: {}, context_project: another_project.full_path, context_sha: another_project.commit.sha }
+          )
+        end
       end
 
       context 'when user is reporter of another project' do
@@ -377,10 +389,19 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
         output = processor.perform
         expect(output.keys).to match_array([:image, :my_build, :my_test])
       end
+
+      it 'stores includes' do
+        perform
+
+        expect(context.includes).to contain_exactly(
+          { type: :file, location: '/templates/my-build.yml', extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, context_sha: '12345' },
+          { type: :file, location: '/templates/my-test.yml', extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, context_sha: '12345' }
+        )
+      end
     end
 
     context 'when local file path has wildcard' do
-      let_it_be(:project) { create(:project, :repository) }
+      let(:project) { create(:project, :repository) }
 
       let(:values) do
         { include: 'myfolder/*.yml', image: 'image:1.0' }
@@ -412,6 +433,15 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
         output = processor.perform
         expect(output.keys).to match_array([:image, :my_build, :my_test])
       end
+
+      it 'stores includes' do
+        perform
+
+        expect(context.includes).to contain_exactly(
+          { type: :local, location: 'myfolder/file1.yml', extra: {}, context_project: project.full_path, context_sha: '12345' },
+          { type: :local, location: 'myfolder/file2.yml', extra: {}, context_project: project.full_path, context_sha: '12345' }
+        )
+      end
     end
 
     context 'when rules defined' do

spec/lib/gitlab/ci/config_spec.rb

@@ -104,6 +104,26 @@ RSpec.describe Gitlab::Ci::Config do
     end
 
     it { is_expected.to contain_exactly('Jobs/Deploy.gitlab-ci.yml', 'Jobs/Build.gitlab-ci.yml') }
+
+    it 'stores includes' do
+      expect(config.metadata[:includes]).to contain_exactly(
+        { type: :template,
+          location: 'Jobs/Deploy.gitlab-ci.yml',
+          extra: {},
+          context_project: nil,
+          context_sha: nil },
+        { type: :template,
+          location: 'Jobs/Build.gitlab-ci.yml',
+          extra: {},
+          context_project: nil,
+          context_sha: nil },
+        { type: :remote,
+          location: 'https://example.com/gitlab-ci.yml',
+          extra: {},
+          context_project: nil,
+          context_sha: nil }
+      )
+    end
   end
 
   context 'when using extendable hash' do
@@ -403,6 +423,26 @@ RSpec.describe Gitlab::Ci::Config do
           end
         end
       end
+
+      it 'stores includes' do
+        expect(config.metadata[:includes]).to contain_exactly(
+          { type: :local,
+            location: local_location,
+            extra: {},
+            context_project: project.full_path,
+            context_sha: '12345' },
+          { type: :remote,
+            location: remote_location,
+            extra: {},
+            context_project: project.full_path,
+            context_sha: '12345' },
+          { type: :file,
+            location: '.gitlab-ci.yml',
+            extra: { project: main_project.full_path, ref: 'HEAD' },
+            context_project: project.full_path,
+            context_sha: '12345' }
+        )
+      end
     end
 
     context "when gitlab_ci.yml has invalid 'include' defined" do