Merge branch 'telemetry-for-secret-detection' into 'master'

Telemetry for secret detection

See merge request gitlab-org/gitlab!31785

db/migrate/20200512195442_update_index_on_name_type_eq_ci_build_to_ci_builds.rb

+# frozen_string_literal: true
+
+class UpdateIndexOnNameTypeEqCiBuildToCiBuilds < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  OLD_INDEX_NAME = 'index_ci_builds_on_name_and_security_type_eq_ci_build'
+  NEW_INDEX_NAME = 'index_security_ci_builds_on_name_and_id'
+
+  OLD_CLAUSE = "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text,
+                ('dast'::character varying)::text,
+                ('dependency_scanning'::character varying)::text,
+                ('license_management'::character varying)::text,
+                ('sast'::character varying)::text,
+                ('license_scanning'::character varying)::text])) AND ((type)::text = 'Ci::Build'::text)"
+
+  NEW_CLAUSE = "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text,
+                ('dast'::character varying)::text,
+                ('dependency_scanning'::character varying)::text,
+                ('license_management'::character varying)::text,
+                ('sast'::character varying)::text,
+                ('secret_detection'::character varying)::text,
+                ('license_scanning'::character varying)::text])) AND ((type)::text = 'Ci::Build'::text)"
+
+  def up
+    add_concurrent_index :ci_builds, [:name, :id], name: NEW_INDEX_NAME, where: NEW_CLAUSE
+    remove_concurrent_index_by_name :ci_builds, OLD_INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :ci_builds, [:name, :id], name: OLD_INDEX_NAME, where: OLD_CLAUSE
+    remove_concurrent_index_by_name :ci_builds, NEW_INDEX_NAME
+  end
+end

db/structure.sql

@@ -9235,8 +9235,6 @@ CREATE INDEX index_ci_builds_on_commit_id_and_type_and_name_and_ref ON public.ci
 
 CREATE INDEX index_ci_builds_on_commit_id_and_type_and_ref ON public.ci_builds USING btree (commit_id, type, ref);
 
-CREATE INDEX index_ci_builds_on_name_and_security_type_eq_ci_build ON public.ci_builds USING btree (name, id) WHERE (((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text, ('license_scanning'::character varying)::text])) AND ((type)::text = 'Ci::Build'::text));
-
 CREATE INDEX index_ci_builds_on_project_id_and_id ON public.ci_builds USING btree (project_id, id);
 
 CREATE INDEX index_ci_builds_on_project_id_and_name_and_ref ON public.ci_builds USING btree (project_id, name, ref) WHERE (((type)::text = 'Ci::Build'::text) AND ((status)::text = 'success'::text) AND ((retried = false) OR (retried IS NULL)));
@@ -10609,6 +10607,8 @@ CREATE UNIQUE INDEX index_scim_identities_on_user_id_and_group_id ON public.scim
 
 CREATE UNIQUE INDEX index_scim_oauth_access_tokens_on_group_id_and_token_encrypted ON public.scim_oauth_access_tokens USING btree (group_id, token_encrypted);
 
+CREATE INDEX index_security_ci_builds_on_name_and_id ON public.ci_builds USING btree (name, id) WHERE (((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text, ('secret_detection'::character varying)::text, ('license_scanning'::character varying)::text])) AND ((type)::text = 'Ci::Build'::text));
+
 CREATE INDEX index_self_managed_prometheus_alert_events_on_environment_id ON public.self_managed_prometheus_alert_events USING btree (environment_id);
 
 CREATE INDEX index_sent_notifications_on_noteable_type_noteable_id ON public.sent_notifications USING btree (noteable_id) WHERE ((noteable_type)::text = 'Issue'::text);
@@ -13903,6 +13903,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200512085150
 20200512160004
 20200512164334
+20200512195442
 20200513160930
 20200513171959
 20200513224143

ee/changelogs/unreleased/telemetry-for-secret-detection.yml

+---
+title: Update index_ci_builds_on_name_and_security_type_eq_ci_build index to support secret-detection
+merge_request: 31785
+author:
+type: performance

ee/lib/ee/gitlab/usage_data.rb

@@ -23,6 +23,9 @@ module EE
         },
         sast: {
           name: :sast_jobs
+        },
+        secret_detection: {
+          name: :secret_detection_jobs
         }
       }.freeze
 
@@ -345,7 +348,7 @@ module EE
         end
 
         # Currently too complicated and to get reliable counts for these stats:
-        # container_scanning_jobs, dast_jobs, dependency_scanning_jobs, license_management_jobs, sast_jobs
+        # container_scanning_jobs, dast_jobs, dependency_scanning_jobs, license_management_jobs, sast_jobs, secret_detection_jobs
         # Once https://gitlab.com/gitlab-org/gitlab/merge_requests/17568 is merged, this might be doable
         def usage_activity_by_stage_secure(time_period)
           prefix = 'user_'

ee/spec/lib/ee/gitlab/usage_data_spec.rb

@@ -24,6 +24,7 @@ describe Gitlab::UsageData do
       create(:ci_build, name: 'license_management', pipeline: pipeline)
       create(:ee_ci_build, name: 'license_scanning', pipeline: pipeline)
       create(:ci_build, name: 'sast', pipeline: pipeline)
+      create(:ci_build, name: 'secret_detection', pipeline: pipeline)
 
       create(:prometheus_alert, project: projects[0])
       create(:prometheus_alert, project: projects[0])
@@ -106,6 +107,7 @@ describe Gitlab::UsageData do
         projects_with_prometheus_alerts
         projects_with_tracing_enabled
         sast_jobs
+        secret_detection_jobs
         status_page_projects
         status_page_issues
         user_preferences_group_overview_details
@@ -136,6 +138,7 @@ describe Gitlab::UsageData do
       expect(count_data[:dependency_scanning_jobs]).to eq(1)
       expect(count_data[:license_management_jobs]).to eq(2)
       expect(count_data[:sast_jobs]).to eq(1)
+      expect(count_data[:secret_detection_jobs]).to eq(1)
     end
 
     it 'correctly shows failure for combined license management' do
@@ -584,6 +587,7 @@ describe Gitlab::UsageData do
                 create(:ci_build, name: 'dependency_scanning', user: user)
                 create(:ci_build, name: 'license_management', user: user)
                 create(:ci_build, name: 'sast', user: user)
+                create(:ci_build, name: 'secret_detection', user: user)
               end
             end
 
@@ -594,7 +598,8 @@ describe Gitlab::UsageData do
                 user_dast_jobs: 1,
                 user_dependency_scanning_jobs: 1,
                 user_license_management_jobs: 1,
-                user_sast_jobs: 1
+                user_sast_jobs: 1,
+                user_secret_detection_jobs: 1
               )
             end
 
@@ -609,7 +614,8 @@ describe Gitlab::UsageData do
                 user_dast_jobs: 1,
                 user_dependency_scanning_jobs: 1,
                 user_license_management_jobs: 2,
-                user_sast_jobs: 1
+                user_sast_jobs: 1,
+                user_secret_detection_jobs: 1
               )
             end
 
@@ -623,7 +629,8 @@ describe Gitlab::UsageData do
                 user_dast_jobs: -1,
                 user_dependency_scanning_jobs: -1,
                 user_license_management_jobs: -1,
-                user_sast_jobs: -1
+                user_sast_jobs: -1,
+                user_secret_detection_jobs: -1
               )
             end
           end