Merge branch 'sk/334415-add-secret-detection' into 'master'

Update security policies pipeline processor to support secret detection

See merge request gitlab-org/gitlab!67223

ee/app/models/security/orchestration_policy_configuration.rb

@@ -16,6 +16,7 @@ module Security
       schedule: 'schedule'
     }.freeze
 
+    SCAN_TYPES = %w[dast secret_detection].freeze
     ON_DEMAND_SCANS = %w[dast].freeze
     AVAILABLE_POLICY_TYPES = %i{scan_execution_policy}.freeze
 
@@ -40,6 +41,10 @@ module Security
       self.exists?(security_policy_management_project_id: project_id)
     end
 
+    def self.valid_scan_type?(scan_type)
+      SCAN_TYPES.include?(scan_type)
+    end
+
     def enabled?
       ::Feature.enabled?(:security_orchestration_policies_configuration, project)
     end
@@ -69,10 +74,11 @@ module Security
     end
 
     def on_demand_scan_actions(ref)
-      active_policies
-        .select { |policy| applicable_for_ref?(policy, ref) }
-        .flat_map { |policy| policy[:actions] }
-        .select { |action| action[:scan].in?(ON_DEMAND_SCANS) }
+      active_policies_scan_actions(ref).select { |action| action[:scan].in?(ON_DEMAND_SCANS) }
+    end
+
+    def pipeline_scan_actions(ref)
+      active_policies_scan_actions(ref).reject { |action| action[:scan].in?(ON_DEMAND_SCANS) }
     end
 
     def active_policy_names_with_dast_site_profile(profile_name)
@@ -136,6 +142,12 @@ module Security
       end
     end
 
+    def active_policies_scan_actions(ref)
+      active_policies
+        .select { |policy| applicable_for_ref?(policy, ref) }
+        .flat_map { |policy| policy[:actions] }
+    end
+
     def policy_blob
       strong_memoize(:policy_blob) do
         policy_repo.blob_data_at(default_branch_or_main, POLICY_PATH)

ee/app/services/security/security_orchestration_policies/ci_configuration_service.rb

+# frozen_string_literal: true
+
+module Security
+  module SecurityOrchestrationPolicies
+    class CiConfigurationService
+      SCAN_TEMPLATES = {
+        'secret_detection' => 'Jobs/Secret-Detection'
+      }.freeze
+
+      def execute(action, ci_variables)
+        case action[:scan]
+        when 'secret_detection'
+          secret_detection_configuration(ci_variables)
+        else
+          error_script('Invalid Scan type')
+        end
+      end
+
+      private
+
+      def scan_template(scan_type)
+        template = ::TemplateFinder.build(:gitlab_ci_ymls, nil, name: SCAN_TEMPLATES[scan_type]).execute
+        Gitlab::Config::Loader::Yaml.new(template.content).load!
+      end
+
+      def secret_detection_configuration(ci_variables)
+        ci_configuration = scan_template('secret_detection')
+
+        ci_configuration[:secret_detection]
+          .merge(ci_configuration[:'.secret-analyzer'])
+          .deep_merge(variables: ci_configuration[:variables].deep_merge(ci_variables).compact)
+          .except(:extends)
+      end
+
+      def error_script(error_message)
+        {
+          'script' => "echo \"Error during Scan execution: #{error_message}\" && false",
+          'allow_failure' => true
+        }
+      end
+    end
+  end
+end

ee/app/services/security/security_orchestration_policies/scan_pipeline_service.rb

+# frozen_string_literal: true
+
+module Security
+  module SecurityOrchestrationPolicies
+    class ScanPipelineService
+      SCAN_VARIABLES = {
+        secret_detection: {
+          'SECRET_DETECTION_HISTORIC_SCAN' => 'false',
+          'SECRET_DETECTION_DISABLED' => nil
+        }
+      }.freeze
+
+      def execute(actions)
+        actions.map.with_index do |action, index|
+          valid_scan_type?(action[:scan]) ? prepare_policy_configuration(action, index) : {}
+        end.reduce({}, :merge)
+      end
+
+      private
+
+      def valid_scan_type?(scan_type)
+        ::Security::OrchestrationPolicyConfiguration.valid_scan_type?(scan_type)
+      end
+
+      def prepare_policy_configuration(action, index)
+        {
+          "#{action[:scan].dasherize}-#{index}" => scan_configuration(action)
+        }.deep_symbolize_keys
+      end
+
+      def scan_configuration(action)
+        ::Security::SecurityOrchestrationPolicies::CiConfigurationService.new.execute(action, SCAN_VARIABLES[action[:scan].to_sym])
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb

@@ -19,7 +19,9 @@ module Gitlab
             return @config unless security_orchestration_policy_configuration.policy_configuration_valid?
             return @config unless extend_configuration?
 
-            merged_config = @config.deep_merge(on_demand_scans_template)
+            merged_config = @config
+                              .deep_merge(on_demand_scans_template)
+                              .deep_merge(pipeline_scan_template)
             observe_processing_duration(Time.current - @start)
 
             merged_config
@@ -37,6 +39,11 @@ module Gitlab
               .execute(security_orchestration_policy_configuration.on_demand_scan_actions(@ref))
           end
 
+          def pipeline_scan_template
+            ::Security::SecurityOrchestrationPolicies::ScanPipelineService
+              .new.execute(security_orchestration_policy_configuration.pipeline_scan_actions(@ref))
+          end
+
           def observe_processing_duration(duration)
             ::Gitlab::Ci::Pipeline::Metrics
               .pipeline_security_orchestration_policy_processing_duration_histogram

ee/spec/lib/gitlab/ci/config/security_orchestration_policies/processor_spec.rb

@@ -31,6 +31,7 @@ RSpec.describe Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor do
          - scan: dast
            site_profile: Site Profile
            scanner_profile: Scanner Profile
+         - scan: secret_detection
     EOS
   end
 
@@ -48,6 +49,12 @@ RSpec.describe Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor do
     end
   end
 
+  shared_examples 'with different scan type' do
+    it 'extends config with additional jobs' do
+      expect(subject).to include(expected_configuration)
+    end
+  end
+
   shared_examples 'when policy is invalid' do
     let_it_be(:policy_yml) do
       <<-EOS
@@ -116,10 +123,7 @@ RSpec.describe Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor do
 
         context 'when DAST profiles are not found' do
           it 'does not modify the config' do
-            expect(subject).to eq(
-              image: 'ruby:3.0.1',
-              'dast-on-demand-0': { allow_failure: true, script: 'echo "Error during On-Demand Scan execution: Dast site profile was not provided" && false' }
-            )
+            expect(subject[:'dast-on-demand-0']).to eq({ allow_failure: true, script: 'echo "Error during On-Demand Scan execution: Dast site profile was not provided" && false' })
           end
         end
 
@@ -130,41 +134,74 @@ RSpec.describe Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor do
           let_it_be(:dast_scanner_profile) { create(:dast_scanner_profile, project: project, name: 'Scanner Profile') }
           let_it_be(:dast_site_profile) { create(:dast_site_profile, project: project, name: 'Site Profile') }
 
-          let(:expected_configuration) do
-            {
-              image: 'ruby:3.0.1',
-              'dast-on-demand-0': {
-                stage: 'dast',
-                image: {
-                  name: '$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION'
-                },
-                variables: {
-                  DAST_VERSION: 2,
-                  SECURE_ANALYZERS_PREFIX: secure_analyzers_prefix,
-                  GIT_STRATEGY: 'none'
-                },
-                allow_failure: true,
-                script: ['/analyze'],
-                artifacts: {
-                  reports: {
-                    dast: 'gl-dast-report.json'
+          it_behaves_like 'with different scan type' do
+            let(:expected_configuration) do
+              {
+                image: 'ruby:3.0.1',
+                'dast-on-demand-0': {
+                  stage: 'dast',
+                  image: {
+                    name: '$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION'
+                  },
+                  variables: {
+                    DAST_VERSION: 2,
+                    SECURE_ANALYZERS_PREFIX: secure_analyzers_prefix,
+                    GIT_STRATEGY: 'none'
+                  },
+                  allow_failure: true,
+                  script: ['/analyze'],
+                  artifacts: {
+                    reports: {
+                      dast: 'gl-dast-report.json'
+                    }
+                  },
+                  dast_configuration: {
+                    site_profile: dast_site_profile.name,
+                    scanner_profile: dast_scanner_profile.name
                   }
-                },
-                dast_configuration: {
-                  site_profile: dast_site_profile.name,
-                  scanner_profile: dast_scanner_profile.name
                 }
               }
-            }
-          end
-
-          it 'extends config with additional jobs' do
-            expect(subject).to include(expected_configuration)
+            end
           end
 
           it_behaves_like 'with pipeline source applicable for CI'
           it_behaves_like 'when policy is invalid'
         end
+
+        context 'when scan type is secret_detection' do
+          it_behaves_like 'with different scan type' do
+            let(:expected_configuration) do
+              {
+                'secret-detection-0': {
+                  rules: [{ if: '$SECRET_DETECTION_DISABLED', when: 'never' }, { if: '$CI_COMMIT_BRANCH' }],
+                  script:
+                    ['if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi',
+                     'if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi',
+                     'git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME',
+                     'git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt',
+                     'export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt',
+                     '/analyzer run',
+                     'rm "$CI_COMMIT_SHA"_commit_list.txt'],
+                  stage: 'test',
+                  image: '$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION',
+                  services: [],
+                  allow_failure: true,
+                  artifacts: {
+                    reports: {
+                      secret_detection: 'gl-secret-detection-report.json'
+                    }
+                  },
+                  variables: {
+                    SECURE_ANALYZERS_PREFIX: 'registry.gitlab.com/gitlab-org/security-products/analyzers',
+                    SECRETS_ANALYZER_VERSION: '3',
+                    SECRET_DETECTION_EXCLUDED_PATHS: '',
+                    SECRET_DETECTION_HISTORIC_SCAN: 'false'
+                  }
+                }
+              }
+            end
+          end
+        end
       end
     end
   end

ee/spec/models/security/orchestration_policy_configuration_spec.rb

@@ -11,6 +11,27 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
 
   let(:default_branch) { security_policy_management_project.default_branch }
   let(:repository) { instance_double(Repository, root_ref: 'master') }
+  let(:policy_yaml) do
+    <<-EOS
+        scan_execution_policy:
+        - name: Run DAST in every pipeline
+          description: This policy enforces to run DAST for every pipeline within the project
+          enabled: true
+          rules:
+          - type: pipeline
+            branches:
+            - "production"
+          actions:
+          - scan: dast
+            site_profile: Site Profile
+            scanner_profile: Scanner Profile
+    EOS
+  end
+
+  before do
+    allow(security_policy_management_project).to receive(:repository).and_return(repository)
+    allow(repository).to receive(:blob_data_at).with(default_branch, Security::OrchestrationPolicyConfiguration::POLICY_PATH).and_return(policy_yaml)
+  end
 
   describe 'associations' do
     it { is_expected.to belong_to(:project).inverse_of(:security_orchestration_policy_configuration) }
@@ -65,6 +86,16 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
     end
   end
 
+  describe '.valid_scan_type?' do
+    it 'returns true when scan type is valid' do
+      expect(described_class.valid_scan_type?('secret_detection')).to be_truthy
+    end
+
+    it 'returns false when scan type is invalid' do
+      expect(described_class.valid_scan_type?('invalid')).to be_falsey
+    end
+  end
+
   describe '#enabled?' do
     subject { security_orchestration_policy_configuration.enabled? }
 
@@ -88,11 +119,6 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
   describe '#policy_configuration_exists?' do
     subject { security_orchestration_policy_configuration.policy_configuration_exists? }
 
-    before do
-      allow(security_policy_management_project).to receive(:repository).and_return(repository)
-      allow(repository).to receive(:blob_data_at).with(default_branch, Security::OrchestrationPolicyConfiguration::POLICY_PATH).and_return(policy_yaml)
-    end
-
     context 'when file is missing' do
       let(:policy_yaml) { nil }
 
@@ -100,23 +126,6 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
     end
 
     context 'when policy is present' do
-      let(:policy_yaml) do
-        <<-EOS
-        scan_execution_policy:
-        - name: Run DAST in every pipeline
-          description: This policy enforces to run DAST for every pipeline within the project
-          enabled: true
-          rules:
-          - type: pipeline
-            branches:
-            - "production"
-          actions:
-          - scan: dast
-            site_profile: Site Profile
-            scanner_profile: Scanner Profile
-        EOS
-      end
-
       it { is_expected.to eq(true) }
     end
   end
@@ -124,29 +133,7 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
   describe '#policy_hash' do
     subject { security_orchestration_policy_configuration.policy_hash }
 
-    before do
-      allow(security_policy_management_project).to receive(:repository).and_return(repository)
-      allow(repository).to receive(:blob_data_at).with(default_branch, Security::OrchestrationPolicyConfiguration::POLICY_PATH).and_return(policy_yaml)
-    end
-
     context 'when policy is present' do
-      let(:policy_yaml) do
-        <<-EOS
-        scan_execution_policy:
-        - name: Run DAST in every pipeline
-          description: This policy enforces to run DAST for every pipeline within the project
-          enabled: true
-          rules:
-          - type: pipeline
-            branches:
-            - "production"
-          actions:
-          - scan: dast
-            site_profile: Site Profile
-            scanner_profile: Scanner Profile
-        EOS
-      end
-
       it { expect(subject.dig(:scan_execution_policy, 0, :name)).to eq('Run DAST in every pipeline') }
     end
 
@@ -200,11 +187,6 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
   describe '#policy_configuration_valid?' do
     subject { security_orchestration_policy_configuration.policy_configuration_valid? }
 
-    before do
-      allow(security_policy_management_project).to receive(:repository).and_return(repository)
-      allow(repository).to receive(:blob_data_at).with(default_branch, Security::OrchestrationPolicyConfiguration::POLICY_PATH).and_return(policy_yaml)
-    end
-
     context 'when file is invalid' do
       let(:policy_yaml) do
         <<-EOS
@@ -226,23 +208,6 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
     end
 
     context 'when file is valid' do
-      let(:policy_yaml) do
-        <<-EOS
-        scan_execution_policy:
-        - name: Run DAST in every pipeline
-          description: This policy enforces to run DAST for every pipeline within the project
-          enabled: true
-          rules:
-          - type: pipeline
-            branches:
-            - "production"
-          actions:
-          - scan: dast
-            site_profile: Site Profile
-            scanner_profile: Scanner Profile
-        EOS
-      end
-
       it { is_expected.to eq(true) }
     end
 
@@ -430,11 +395,6 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
       security_orchestration_policy_configuration.on_demand_scan_actions(ref)
     end
 
-    before do
-      allow(security_policy_management_project).to receive(:repository).and_return(repository)
-      allow(repository).to receive(:blob_data_at).with(default_branch, Security::OrchestrationPolicyConfiguration::POLICY_PATH).and_return(policy_yaml)
-    end
-
     context 'when ref is branch' do
       let(:ref) { 'refs/heads/release/123' }
 
@@ -450,6 +410,64 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
     end
   end
 
+  describe '#pipeline_scan_actions' do
+    let(:policy_yaml) do
+      <<-EOS
+      scan_execution_policy:
+        - name: Run DAST and Secret Detection in every pipeline
+          enabled: true
+          rules:
+          - type: pipeline
+            branches:
+            - "production"
+          actions:
+          - scan: dast
+            site_profile: Site Profile 2
+            scanner_profile: Scanner Profile 2
+          - scan: secret_detection
+        - name: Run DAST in every pipeline
+          enabled: true
+          rules:
+          - type: pipeline
+            branches:
+            - "production"
+          actions:
+          - scan: dast
+            site_profile: Site Profile
+            scanner_profile: Scanner Profile
+        - name: Run Secret Detection for all branches
+          enabled: true
+          rules:
+          - type: pipeline
+            branches:
+            - "*"
+          actions:
+          - scan: secret_detection
+        - name: Scheduled scan
+          enabled: true
+          rules:
+          - type: schedule
+            cadence: '*/15 * * * *'
+            branches:
+            - "*"
+          actions:
+          - scan: secret_detection
+      EOS
+    end
+
+    let(:expected_actions) do
+      [{ scan: 'secret_detection' }, { scan: 'secret_detection' }]
+    end
+
+    subject(:pipeline_scan_actions) do
+      security_orchestration_policy_configuration.pipeline_scan_actions('refs/heads/production')
+    end
+
+    it 'returns only actions for pipeline scans applicable for branch' do
+      expect(pipeline_scan_actions).to eq(expected_actions)
+    end
+  end
+
   describe '#active_policy_names_with_dast_site_profile' do
     let(:policy_yaml) do
       <<-EOS
@@ -471,11 +489,6 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
       EOS
     end
 
-    before do
-      allow(security_policy_management_project).to receive(:repository).and_return(repository)
-      allow(repository).to receive(:blob_data_at).with(default_branch, Security::OrchestrationPolicyConfiguration::POLICY_PATH).and_return(policy_yaml)
-    end
-
     it 'returns list of policy names where site profile is referenced' do
       expect( security_orchestration_policy_configuration.active_policy_names_with_dast_site_profile('Site Profile')).to contain_exactly('Run DAST in every pipeline')
     end

ee/spec/services/security/security_orchestration_policies/ci_configuration_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Security::SecurityOrchestrationPolicies::CiConfigurationService do
+  describe '#execute' do
+    let_it_be(:service) { described_class.new }
+    let_it_be(:ci_variables) do
+      { 'SECRET_DETECTION_HISTORIC_SCAN' => 'false', 'SECRET_DETECTION_DISABLED' => nil }
+    end
+
+    subject { service.execute(action, ci_variables) }
+
+    shared_examples 'with template name for scan type' do
+      it 'fetches template content using ::TemplateFinder' do
+        expect(::TemplateFinder).to receive(:build).with(:gitlab_ci_ymls, nil, name: template_name).and_call_original
+
+        subject
+      end
+    end
+
+    context 'when scan type is secret_detection' do
+      context 'when action is valid' do
+        let_it_be(:action) { { scan: 'secret_detection' } }
+        let_it_be(:template_name) { 'Jobs/Secret-Detection' }
+
+        it_behaves_like 'with template name for scan type'
+
+        it 'returns prepared CI configuration with Secret Detection scans' do
+          expected_configuration = {
+            rules: [{ if: '$SECRET_DETECTION_DISABLED', when: 'never' }, { if: '$CI_COMMIT_BRANCH' }],
+            script:
+              ['if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi',
+               'if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi',
+               'git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME',
+               'git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt',
+               'export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt',
+               '/analyzer run',
+               'rm "$CI_COMMIT_SHA"_commit_list.txt'],
+            stage: 'test',
+            image: '$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION',
+            services: [],
+            allow_failure: true,
+            artifacts: {
+              reports: {
+                secret_detection: 'gl-secret-detection-report.json'
+              }
+            },
+            variables: {
+              SECURE_ANALYZERS_PREFIX: 'registry.gitlab.com/gitlab-org/security-products/analyzers',
+              SECRETS_ANALYZER_VERSION: '3',
+              SECRET_DETECTION_EXCLUDED_PATHS: '',
+              SECRET_DETECTION_HISTORIC_SCAN: 'false'
+            }
+          }
+
+          expect(subject.deep_symbolize_keys).to eq(expected_configuration)
+        end
+      end
+
+      context 'when action is invalid' do
+        let_it_be(:action) { { scan: 'invalid_type' } }
+
+        it 'returns prepared CI configuration with error script' do
+          expected_configuration = {
+            'allow_failure' => true,
+            'script' => "echo \"Error during Scan execution: Invalid Scan type\" && false"
+          }
+
+          expect(subject).to eq(expected_configuration)
+        end
+      end
+    end
+  end
+end

ee/spec/services/security/security_orchestration_policies/scan_pipeline_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Security::SecurityOrchestrationPolicies::ScanPipelineService do
+  describe '#execute' do
+    let_it_be(:service) { described_class.new }
+
+    subject { service.execute(actions) }
+
+    shared_examples 'creates scan jobs' do |times, job_names|
+      it 'returns created jobs' do
+        expect(::Security::SecurityOrchestrationPolicies::CiConfigurationService).to receive(:new).exactly(times).times.and_call_original
+
+        expect(subject.keys).to eq(job_names)
+      end
+    end
+
+    context 'when there is an invalid action' do
+      let(:actions) { [{ scan: 'invalid' }] }
+
+      it 'does not create scan job' do
+        expect(::Security::SecurityOrchestrationPolicies::CiConfigurationService).not_to receive(:new)
+
+        expect(subject.keys).to eq([])
+      end
+    end
+
+    context 'when there is only one action' do
+      let(:actions) { [{ scan: 'secret_detection' }] }
+
+      it_behaves_like 'creates scan jobs', 1, [:'secret-detection-0']
+    end
+
+    context 'when there are multiple actions' do
+      let(:actions) do
+        [
+          { scan: 'secret_detection' },
+          { scan: 'dast', scanner_profile: 'Scanner Profile', site_profile: 'Site Profile' }
+        ]
+      end
+
+      it_behaves_like 'creates scan jobs', 2, [:'secret-detection-0', :'dast-1']
+    end
+
+    context 'when there are valid and invalid actions' do
+      let(:actions) do
+        [
+          { scan: 'secret_detection' },
+          { scan: 'invalid' }
+        ]
+      end
+
+      it_behaves_like 'creates scan jobs', 1, [:'secret-detection-0']
+    end
+  end
+end