Merge branch 'count-security-jobs' into 'master'

Count security jobs

See merge request gitlab-org/gitlab!39481

ee/changelogs/unreleased/count-security-jobs.yml

+---
+title: Count security jobs
+merge_request: 39481
+author:
+type: added

ee/lib/ee/gitlab/usage_data.rb

@@ -317,6 +317,7 @@ module EE
           end
 
           results.merge!(count_secure_pipelines(time_period))
+          results.merge!(count_secure_jobs(time_period))
 
           results[:"#{prefix}unique_users_all_secure_scanners"] = distinct_count(::Ci::Build.where(name: SECURE_PRODUCT_TYPES.keys).where(time_period), :user_id)
 
@@ -334,6 +335,20 @@ module EE
         # rubocop:disable CodeReuse/ActiveRecord
         # rubocop: disable UsageData/LargeTable
         # rubocop: disable UsageData/DistinctCountByLargeForeignKey
+        def count_secure_jobs(time_period)
+          start = ::Security::Scan.minimum(:build_id)
+          finish = ::Security::Scan.maximum(:build_id)
+
+          {}.tap do |secure_jobs|
+            ::Security::Scan.scan_types.each do |name, scan_type|
+              secure_jobs["#{name}_scans".to_sym] = count(::Security::Scan.joins(:build)
+                .where(scan_type: scan_type)
+                .merge(::CommitStatus.latest.success)
+                .where(time_period), :build_id, start: start, finish: finish)
+            end
+          end
+        end
+
         def count_secure_pipelines(time_period)
           return {} if time_period.blank?
 

ee/spec/lib/ee/gitlab/usage_data_spec.rb

@@ -499,11 +499,17 @@ RSpec.describe Gitlab::UsageData do
         user_sast_jobs: 1,
         user_secret_detection_jobs: 1,
         sast_pipeline: 0,
+        sast_scans: 0,
         dependency_scanning_pipeline: 0,
+        dependency_scanning_scans: 0,
         container_scanning_pipeline: 0,
+        container_scanning_scans: 0,
         dast_pipeline: 0,
+        dast_scans: 0,
         secret_detection_pipeline: 0,
+        secret_detection_scans: 0,
         coverage_fuzzing_pipeline: 0,
+        coverage_fuzzing_scans: 0,
         user_unique_users_all_secure_scanners: 1
       )
     end
@@ -511,6 +517,7 @@ RSpec.describe Gitlab::UsageData do
     it 'counts pipelines that have security jobs' do
       for_defined_days_back do
         ds_build = create(:ci_build, name: 'retirejs', user: user, status: 'success')
+        ds_bundler_audit_build = create(:ci_build, :failed, user: user, name: 'retirejs')
         ds_bundler_build = create(:ci_build, name: 'bundler-audit', user: user, commit_id: ds_build.pipeline.id, status: 'success')
         secret_detection_build = create(:ci_build, name: 'secret', user: user, commit_id: ds_build.pipeline.id, status: 'success')
         cs_build = create(:ci_build, name: 'klar', user: user, status: 'success')
@@ -520,6 +527,7 @@ RSpec.describe Gitlab::UsageData do
         create(:security_scan, build: secret_detection_build, scan_type: 'secret_detection')
         create(:security_scan, build: cs_build, scan_type: 'container_scanning')
         create(:security_scan, build: sast_build, scan_type: 'sast')
+        create(:security_scan, build: ds_bundler_audit_build, scan_type: 'dependency_scanning')
       end
 
       expect(described_class.usage_activity_by_stage_secure({})).to include(
@@ -530,7 +538,13 @@ RSpec.describe Gitlab::UsageData do
         user_license_management_jobs: 1,
         user_sast_jobs: 1,
         user_secret_detection_jobs: 1,
-        user_unique_users_all_secure_scanners: 1
+        user_unique_users_all_secure_scanners: 1,
+        sast_scans: 0,
+        dependency_scanning_scans: 4,
+        container_scanning_scans: 2,
+        dast_scans: 0,
+        secret_detection_scans: 2,
+        coverage_fuzzing_scans: 0
       )
 
       expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include(
@@ -547,7 +561,13 @@ RSpec.describe Gitlab::UsageData do
         dast_pipeline: 0,
         secret_detection_pipeline: 1,
         coverage_fuzzing_pipeline: 0,
-        user_unique_users_all_secure_scanners: 1
+        user_unique_users_all_secure_scanners: 1,
+        sast_scans: 0,
+        dependency_scanning_scans: 2,
+        container_scanning_scans: 1,
+        dast_scans: 0,
+        secret_detection_scans: 1,
+        coverage_fuzzing_scans: 0
       )
     end
 
@@ -568,11 +588,17 @@ RSpec.describe Gitlab::UsageData do
         user_sast_jobs: 2,
         user_secret_detection_jobs: 1,
         sast_pipeline: 0,
+        sast_scans: 0,
         dependency_scanning_pipeline: 0,
+        dependency_scanning_scans: 0,
         container_scanning_pipeline: 0,
+        container_scanning_scans: 0,
         dast_pipeline: 0,
+        dast_scans: 0,
         secret_detection_pipeline: 0,
+        secret_detection_scans: 0,
         coverage_fuzzing_pipeline: 0,
+        coverage_fuzzing_scans: 0,
         user_unique_users_all_secure_scanners: 3
       )
     end
@@ -592,14 +618,50 @@ RSpec.describe Gitlab::UsageData do
         user_sast_jobs: 1,
         user_secret_detection_jobs: 1,
         sast_pipeline: 0,
+        sast_scans: 0,
         dependency_scanning_pipeline: 0,
+        dependency_scanning_scans: 0,
         container_scanning_pipeline: 0,
+        container_scanning_scans: 0,
         dast_pipeline: 0,
+        dast_scans: 0,
         secret_detection_pipeline: 0,
+        secret_detection_scans: 0,
         coverage_fuzzing_pipeline: 0,
+        coverage_fuzzing_scans: 0,
         user_unique_users_all_secure_scanners: 1
       )
     end
+
+    it 'has to resort to 0 for counting license scan' do
+      allow(Gitlab::Database::BatchCount).to receive(:batch_distinct_count).and_raise(ActiveRecord::StatementInvalid)
+      allow(Gitlab::Database::BatchCount).to receive(:batch_count).and_raise(ActiveRecord::StatementInvalid)
+      allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid)
+
+      expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq(
+        user_preferences_group_overview_security_dashboard: -1,
+        user_container_scanning_jobs: -1,
+        user_coverage_fuzzing_jobs: -1,
+        user_dast_jobs: -1,
+        user_dependency_scanning_jobs: -1,
+        user_license_management_jobs: -1,
+        user_sast_jobs: -1,
+        user_secret_detection_jobs: -1,
+        sast_pipeline: -1,
+        sast_scans: -1,
+        dependency_scanning_pipeline: -1,
+        dependency_scanning_scans: -1,
+        container_scanning_pipeline: -1,
+        container_scanning_scans: -1,
+        dast_pipeline: -1,
+        dast_scans: -1,
+        secret_detection_pipeline: -1,
+        secret_detection_scans: -1,
+        coverage_fuzzing_pipeline: -1,
+        coverage_fuzzing_scans: -1,
+        user_unique_users_all_secure_scanners: -1
+      )
+    end
   end
 
   describe 'usage_activity_by_stage_verify' do