Collect included files on CI pipeline creation

We want to show included files in the CI Linter page. To do that,
backend needs to send this information to frontend via the ciConfig
result.

In this commit, we only collect this data in a metadata method of
CI Config. In the next iteration, we'll pass this information to
the GraphQL endpoint.

lib/gitlab/ci/config.rb

@@ -82,7 +82,13 @@ module Gitlab
       end
 
       def included_templates
-        @context.expandset.filter_map { |i| i[:template] }
+        @context.includes.filter_map { |i| i[:location] if i[:type] == :template }
+      end
+
+      def metadata
+        {
+          includes: @context.includes
+        }
       end
 
       private

lib/gitlab/ci/config/external/context.rb

@@ -70,16 +70,20 @@ module Gitlab
             }
           end
 
-          def mask_variables_from(location)
-            variables.reduce(location.dup) do |loc, variable|
+          def mask_variables_from(string)
+            variables.reduce(string.dup) do |str, variable|
               if variable[:masked]
-                Gitlab::Ci::MaskSecret.mask!(loc, variable[:value])
+                Gitlab::Ci::MaskSecret.mask!(str, variable[:value])
               else
-                loc
+                str
               end
             end
           end
 
+          def includes
+            expandset.map(&:metadata)
+          end
+
           protected
 
           attr_writer :expandset, :execution_deadline, :logger

lib/gitlab/ci/config/external/file/artifact.rb

@@ -28,6 +28,14 @@ module Gitlab
               end
             end
 
+            def metadata
+              super.merge(
+                type: :artifact,
+                location: masked_location,
+                extra: { job_name: masked_job_name }
+              )
+            end
+
             private
 
             def project
@@ -52,7 +60,7 @@ module Gitlab
               end
 
               unless artifact_job.present?
-                errors.push("Job `#{job_name}` not found in parent pipeline or does not have artifacts!")
+                errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!")
                 return false
               end
 
@@ -80,6 +88,12 @@ module Gitlab
                 parent_pipeline: context.parent_pipeline
               }
             end
+
+            def masked_job_name
+              strong_memoize(:masked_job_name) do
+                context.mask_variables_from(job_name)
+              end
+            end
           end
         end
       end

lib/gitlab/ci/config/external/file/base.rb

@@ -55,6 +55,21 @@ module Gitlab
               end
             end
 
+            def metadata
+              {
+                context_project: context.project&.full_path,
+                context_sha: context.sha
+              }
+            end
+
+            def eql?(other)
+              other.hash == hash
+            end
+
+            def hash
+              [params, context.project&.full_path, context.sha].hash
+            end
+
             protected
 
             def expanded_content_hash

lib/gitlab/ci/config/external/file/local.rb

@@ -19,6 +19,14 @@ module Gitlab
               strong_memoize(:content) { fetch_local_content }
             end
 
+            def metadata
+              super.merge(
+                type: :local,
+                location: masked_location,
+                extra: {}
+              )
+            end
+
             private
 
             def validate_content!

lib/gitlab/ci/config/external/file/project.rb

@@ -27,17 +27,25 @@ module Gitlab
               strong_memoize(:content) { fetch_local_content }
             end
 
+            def metadata
+              super.merge(
+                type: :file,
+                location: masked_location,
+                extra: { project: masked_project_name, ref: masked_ref_name }
+              )
+            end
+
             private
 
             def validate_content!
               if !can_access_local_content?
-                errors.push("Project `#{project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.")
+                errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.")
               elsif sha.nil?
-                errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!")
+                errors.push("Project `#{masked_project_name}` reference `#{masked_ref_name}` does not exist!")
               elsif content.nil?
-                errors.push("Project `#{project_name}` file `#{masked_location}` does not exist!")
+                errors.push("Project `#{masked_project_name}` file `#{masked_location}` does not exist!")
               elsif content.blank?
-                errors.push("Project `#{project_name}` file `#{masked_location}` is empty!")
+                errors.push("Project `#{masked_project_name}` file `#{masked_location}` is empty!")
               end
             end
 
@@ -76,6 +84,18 @@ module Gitlab
                 variables: context.variables
               }
             end
+
+            def masked_project_name
+              strong_memoize(:masked_project_name) do
+                context.mask_variables_from(project_name)
+              end
+            end
+
+            def masked_ref_name
+              strong_memoize(:masked_ref_name) do
+                context.mask_variables_from(ref_name)
+              end
+            end
           end
         end
       end

lib/gitlab/ci/config/external/file/remote.rb

@@ -18,6 +18,14 @@ module Gitlab
               strong_memoize(:content) { fetch_remote_content }
             end
 
+            def metadata
+              super.merge(
+                type: :remote,
+                location: masked_location,
+                extra: {}
+              )
+            end
+
             private
 
             def validate_location!

lib/gitlab/ci/config/external/file/template.rb

@@ -20,6 +20,14 @@ module Gitlab
               strong_memoize(:content) { fetch_template_content }
             end
 
+            def metadata
+              super.merge(
+                type: :template,
+                location: masked_location,
+                extra: {}
+              )
+            end
+
             private
 
             def validate_location!

lib/gitlab/ci/config/external/mapper.rb

@@ -48,7 +48,6 @@ module Gitlab
               .flat_map(&method(:expand_project_files))
               .flat_map(&method(:expand_wildcard_paths))
               .map(&method(:expand_variables))
-              .each(&method(:verify_duplicates!))
               .map(&method(:select_first_matching))
               .each(&method(:verify!))
           end
@@ -112,26 +111,6 @@ module Gitlab
             end
           end
 
-          def verify_duplicates!(location)
-            logger.instrument(:config_mapper_verify) do
-              verify_max_includes_and_add_location!(location)
-            end
-          end
-
-          def verify_max_includes_and_add_location!(location)
-            if expandset.count >= MAX_INCLUDES
-              raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!"
-            end
-
-            # Scope location to context to allow support of
-            # relative includes
-            scoped_location = location.merge(
-              context_project: context.project,
-              context_sha: context.sha)
-
-            expandset.add(scoped_location)
-          end
-
           def select_first_matching(location)
             logger.instrument(:config_mapper_select) do
               select_first_matching_without_instrumentation(location)
@@ -149,7 +128,15 @@ module Gitlab
           end
 
           def verify!(location_object)
+            verify_max_includes!
             location_object.validate!
+            expandset.add(location_object)
+          end
+
+          def verify_max_includes!
+            if expandset.count >= MAX_INCLUDES
+              raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!"
+            end
           end
 
           def expand_variables(data)

spec/lib/gitlab/ci/config/external/file/artifact_spec.rb

@@ -4,8 +4,9 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Config::External::File::Artifact do
   let(:parent_pipeline) { create(:ci_pipeline) }
+  let(:variables) {}
   let(:context) do
-    Gitlab::Ci::Config::External::Context.new(parent_pipeline: parent_pipeline)
+    Gitlab::Ci::Config::External::Context.new(variables: variables, parent_pipeline: parent_pipeline)
   end
 
   let(:external_file) { described_class.new(params, context) }
@@ -170,6 +171,58 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact do
           end
         end
       end
+
+      context 'when job is provided as a variable' do
+        let(:variables) do
+          Gitlab::Ci::Variables::Collection.new([
+            { key: 'VAR1', value: 'a_secret_variable_value', masked: true }
+          ])
+        end
+
+        let(:params) { { artifact: 'generated.yml', job: 'a_secret_variable_value' } }
+
+        context 'when job does not exist in the parent pipeline' do
+          let(:expected_error) do
+            'Job `xxxxxxxxxxxxxxxxxxxxxxx` not found in parent pipeline or does not have artifacts!'
+          end
+
+          it_behaves_like 'is invalid'
+        end
+      end
+    end
+  end
+
+  describe '#metadata' do
+    let(:params) { { artifact: 'generated.yml' } }
+
+    subject(:metadata) { external_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: nil,
+        context_sha: nil,
+        type: :artifact,
+        location: 'generated.yml',
+        extra: { job_name: nil }
+      )
+    }
+
+    context 'when job name includes a masked variable' do
+      let(:variables) do
+        Gitlab::Ci::Variables::Collection.new([{ key: 'VAR1', value: 'a_secret_variable_value', masked: true }])
+      end
+
+      let(:params) { { artifact: 'generated.yml', job: 'a_secret_variable_value' } }
+
+      it {
+        is_expected.to eq(
+          context_project: nil,
+          context_sha: nil,
+          type: :artifact,
+          location: 'generated.yml',
+          extra: { job_name: 'xxxxxxxxxxxxxxxxxxxxxxx' }
+        )
+      }
     end
   end
 end

spec/lib/gitlab/ci/config/external/file/base_spec.rb

@@ -112,4 +112,52 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base do
       end
     end
   end
+
+  describe '#metadata' do
+    let(:location) { 'some/file/config.yml' }
+
+    subject(:metadata) { file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: nil,
+        context_sha: 'HEAD'
+      )
+    }
+  end
+
+  describe '#eql?' do
+    let(:location) { 'some/file/config.yml' }
+
+    subject(:eql) { file.eql?(other_file) }
+
+    context 'when the other file has the same params' do
+      let(:other_file) { test_class.new(location, context) }
+
+      it { is_expected.to eq(true) }
+    end
+
+    context 'when the other file has not the same params' do
+      let(:other_file) { test_class.new('some/other/file', context) }
+
+      it { is_expected.to eq(false) }
+    end
+  end
+
+  describe '#hash' do
+    let(:location) { 'some/file/config.yml' }
+
+    subject(:filehash) { file.hash }
+
+    context 'with a project' do
+      let(:project) { create(:project) }
+      let(:context_params) { { project: project, sha: 'HEAD', variables: variables } }
+
+      it { is_expected.to eq([location, project.full_path, 'HEAD'].hash) }
+    end
+
+    context 'without a project' do
+      it { is_expected.to eq([location, nil, 'HEAD'].hash) }
+    end
+  end
 end

spec/lib/gitlab/ci/config/external/file/local_spec.rb

@@ -187,4 +187,20 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do
       end
     end
   end
+
+  describe '#metadata' do
+    let(:location) { '/lib/gitlab/ci/templates/existent-file.yml' }
+
+    subject(:metadata) { local_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: project.full_path,
+        context_sha: '12345',
+        type: :local,
+        location: location,
+        extra: {}
+      )
+    }
+  end
 end

spec/lib/gitlab/ci/config/external/file/project_spec.rb

@@ -160,6 +160,23 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do
         expect(project_file.error_message).to include('Included file `/invalid-file` does not have YAML extension!')
       end
     end
+
+    context 'when non-existing project is used with a masked variable' do
+      let(:variables) do
+        Gitlab::Ci::Variables::Collection.new([
+          { key: 'VAR1', value: 'a_secret_variable_value', masked: true }
+        ])
+      end
+
+      let(:params) do
+        { project: 'a_secret_variable_value', file: '/file.yml' }
+      end
+
+      it 'returns false with masked project name' do
+        expect(valid?).to be_falsy
+        expect(project_file.error_message).to include("Project `xxxxxxxxxxxxxxxxxxxxxxx` not found or access denied!")
+      end
+    end
   end
 
   describe '#expand_context' do
@@ -177,6 +194,45 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do
     end
   end
 
+  describe '#metadata' do
+    let(:params) do
+      { project: project.full_path, file: '/file.yml' }
+    end
+
+    subject(:metadata) { project_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: context_project.full_path,
+        context_sha: '12345',
+        type: :file,
+        location: '/file.yml',
+        extra: { project: project.full_path, ref: 'HEAD' }
+      )
+    }
+
+    context 'when project name and ref include masked variables' do
+      let(:variables) do
+        Gitlab::Ci::Variables::Collection.new([
+          { key: 'VAR1', value: 'a_secret_variable_value1', masked: true },
+          { key: 'VAR2', value: 'a_secret_variable_value2', masked: true }
+        ])
+      end
+
+      let(:params) { { project: 'a_secret_variable_value1', ref: 'a_secret_variable_value2', file: '/file.yml' } }
+
+      it {
+        is_expected.to eq(
+          context_project: context_project.full_path,
+          context_sha: '12345',
+          type: :file,
+          location: '/file.yml',
+          extra: { project: 'xxxxxxxxxxxxxxxxxxxxxxxx', ref: 'xxxxxxxxxxxxxxxxxxxxxxxx' }
+        )
+      }
+    end
+  end
+
   private
 
   def stub_project_blob(ref, path)

spec/lib/gitlab/ci/config/external/file/remote_spec.rb

@@ -199,4 +199,22 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote do
       is_expected.to be_empty
     end
   end
+
+  describe '#metadata' do
+    before do
+      stub_full_request(location).to_return(body: remote_file_content)
+    end
+
+    subject(:metadata) { remote_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: nil,
+        context_sha: '12345',
+        type: :remote,
+        location: 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.xxxxxxxxxxx.yml',
+        extra: {}
+      )
+    }
+  end
 end

spec/lib/gitlab/ci/config/external/file/template_spec.rb

@@ -114,4 +114,18 @@ RSpec.describe Gitlab::Ci::Config::External::File::Template do
       is_expected.to be_empty
     end
   end
+
+  describe '#metadata' do
+    subject(:metadata) { template_file.metadata }
+
+    it {
+      is_expected.to eq(
+        context_project: project.full_path,
+        context_sha: '12345',
+        type: :template,
+        location: template,
+        extra: {}
+      )
+    }
+  end
 end

spec/lib/gitlab/ci/config/external/mapper_spec.rb

@@ -21,6 +21,8 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
     HEREDOC
   end
 
+  subject(:mapper) { described_class.new(values, context) }
+
   before do
     stub_full_request(remote_url).to_return(body: file_content)
 
@@ -30,7 +32,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
   end
 
   describe '#process' do
-    subject { described_class.new(values, context).process }
+    subject(:process) { mapper.process }
 
     context "when single 'include' keyword is defined" do
       context 'when the string is a local file' do
@@ -189,7 +191,12 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
       end
 
       it 'does not raise an exception' do
-        expect { subject }.not_to raise_error
+        expect { process }.not_to raise_error
+      end
+
+      it 'has expanset with one' do
+        process
+        expect(mapper.expandset.size).to eq(1)
       end
     end
 
@@ -385,5 +392,27 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
         end
       end
     end
+
+    context "when locations are same after masking variables" do
+      let(:variables) do
+        Gitlab::Ci::Variables::Collection.new([
+          { 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file1', 'masked' => true },
+          { 'key' => 'GITLAB_TOKEN', 'value' => 'secret-file2', 'masked' => true }
+        ])
+      end
+
+      let(:values) do
+        { include: [
+          { 'local' => 'hello/secret-file1.yml' },
+          { 'local' => 'hello/secret-file2.yml' }
+        ],
+        image: 'ruby:2.7' }
+      end
+
+      it 'has expanset with two' do
+        process
+        expect(mapper.expandset.size).to eq(2)
+      end
+    end
   end
 end

spec/lib/gitlab/ci/config/external/processor_spec.rb

@@ -22,7 +22,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
   end
 
   describe "#perform" do
-    subject { processor.perform }
+    subject(:perform) { processor.perform }
 
     context 'when no external files defined' do
       let(:values) { { image: 'image:1.0' } }
@@ -262,6 +262,18 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
 
           expect(process_obs_count).to eq(3)
         end
+
+        it 'stores includes' do
+          perform
+
+          expect(context.includes).to contain_exactly(
+            { type: :local, location: '/local/file.yml', extra: {}, context_project: project.full_path, context_sha: '12345' },
+            { type: :template, location: 'Ruby.gitlab-ci.yml', extra: {}, context_project: project.full_path, context_sha: '12345' },
+            { type: :remote, location: 'http://my.domain.com/config.yml', extra: {}, context_project: project.full_path, context_sha: '12345' },
+            { type: :file, location: '/templates/my-workflow.yml', extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, context_sha: '12345' },
+            { type: :local, location: '/templates/my-build.yml', extra: {}, context_project: another_project.full_path, context_sha: another_project.commit.sha }
+          )
+        end
       end
 
       context 'when user is reporter of another project' do
@@ -377,10 +389,19 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
         output = processor.perform
         expect(output.keys).to match_array([:image, :my_build, :my_test])
       end
+
+      it 'stores includes' do
+        perform
+
+        expect(context.includes).to contain_exactly(
+          { type: :file, location: '/templates/my-build.yml', extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, context_sha: '12345' },
+          { type: :file, location: '/templates/my-test.yml', extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, context_sha: '12345' }
+        )
+      end
     end
 
     context 'when local file path has wildcard' do
-      let_it_be(:project) { create(:project, :repository) }
+      let(:project) { create(:project, :repository) }
 
       let(:values) do
         { include: 'myfolder/*.yml', image: 'image:1.0' }
@@ -412,6 +433,15 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
         output = processor.perform
         expect(output.keys).to match_array([:image, :my_build, :my_test])
       end
+
+      it 'stores includes' do
+        perform
+
+        expect(context.includes).to contain_exactly(
+          { type: :local, location: 'myfolder/file1.yml', extra: {}, context_project: project.full_path, context_sha: '12345' },
+          { type: :local, location: 'myfolder/file2.yml', extra: {}, context_project: project.full_path, context_sha: '12345' }
+        )
+      end
     end
 
     context 'when rules defined' do

spec/lib/gitlab/ci/config_spec.rb

@@ -104,6 +104,26 @@ RSpec.describe Gitlab::Ci::Config do
     end
 
     it { is_expected.to contain_exactly('Jobs/Deploy.gitlab-ci.yml', 'Jobs/Build.gitlab-ci.yml') }
+
+    it 'stores includes' do
+      expect(config.metadata[:includes]).to contain_exactly(
+        { type: :template,
+          location: 'Jobs/Deploy.gitlab-ci.yml',
+          extra: {},
+          context_project: nil,
+          context_sha: nil },
+        { type: :template,
+          location: 'Jobs/Build.gitlab-ci.yml',
+          extra: {},
+          context_project: nil,
+          context_sha: nil },
+        { type: :remote,
+          location: 'https://example.com/gitlab-ci.yml',
+          extra: {},
+          context_project: nil,
+          context_sha: nil }
+      )
+    end
   end
 
   context 'when using extendable hash' do
@@ -403,6 +423,26 @@ RSpec.describe Gitlab::Ci::Config do
           end
         end
       end
+
+      it 'stores includes' do
+        expect(config.metadata[:includes]).to contain_exactly(
+          { type: :local,
+            location: local_location,
+            extra: {},
+            context_project: project.full_path,
+            context_sha: '12345' },
+          { type: :remote,
+            location: remote_location,
+            extra: {},
+            context_project: project.full_path,
+            context_sha: '12345' },
+          { type: :file,
+            location: '.gitlab-ci.yml',
+            extra: { project: main_project.full_path, ref: 'HEAD' },
+            context_project: project.full_path,
+            context_sha: '12345' }
+        )
+      end
     end
 
     context "when gitlab_ci.yml has invalid 'include' defined" do