Track YAML-less Auto DevOps inclusions separately

In https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51391 we added
tracking for include:template. Unfortunately, YAML-less Auto DevOps also
uses include:template for Auto-DevOps.gitab-ci.yml, making it currently
impossible to differentiate .gitlab-ci.yml template usage from implicit
usage in our numbers.

This changes it so that pipelines from the auto_devops_source config
source (i.e. YAML-free Auto DevOps pipelines) are counted separately.

changelogs/unreleased/count-implicit-auto-devops-inclusions-separately.yml

+---
+title: Track YAML-less Auto DevOps inclusions separately
+merge_request: 53383
+author:
+type: changed

lib/gitlab/ci/pipeline/chain/template_usage.rb

@@ -19,7 +19,7 @@ module Gitlab
 
           def track_event(template)
             Gitlab::UsageDataCounters::CiTemplateUniqueCounter
-              .track_unique_project_event(project_id: pipeline.project_id, template: template)
+              .track_unique_project_event(project_id: pipeline.project_id, template: template, config_source: pipeline.config_source)
           end
 
           def included_templates

lib/gitlab/usage_data_counters/ci_template_unique_counter.rb

@@ -4,6 +4,7 @@ module Gitlab::UsageDataCounters
   class CiTemplateUniqueCounter
     REDIS_SLOT = 'ci_templates'.freeze
 
+    # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_`
     TEMPLATE_TO_EVENT = {
       '5-Minute-Production-App.gitlab-ci.yml' => '5_min_production_app',
       'Auto-DevOps.gitlab-ci.yml' => 'auto_devops',
@@ -18,19 +19,21 @@ module Gitlab::UsageDataCounters
     }.freeze
 
     class << self
-      def track_unique_project_event(project_id:, template:)
+      def track_unique_project_event(project_id:, template:, config_source:)
         return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml)
 
-        if event = unique_project_event(template)
+        if event = unique_project_event(template, config_source)
           Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id)
         end
       end
 
       private
 
-      def unique_project_event(template)
+      def unique_project_event(template, config_source)
         if name = TEMPLATE_TO_EVENT[template]
-          "p_#{REDIS_SLOT}_#{name}"
+          prefix = 'implicit_' if config_source.to_s == 'auto_devops_source'
+
+          "p_#{REDIS_SLOT}_#{prefix}#{name}"
         end
       end
     end

lib/gitlab/usage_data_counters/known_events/ci_templates.yml

+# Implicit Auto DevOps pipeline events
+- name: p_ci_templates_implicit_auto_devops
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_implicit_auto_devops_build
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_implicit_auto_devops_deploy
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_implicit_security_sast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_implicit_security_secret_detection
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+# Explicit include:template pipeline events
+- name: p_ci_templates_5_min_production_app
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_auto_devops
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_aws_cf_deploy_ec2
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_aws_deploy_ecs
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_auto_devops_build
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_auto_devops_deploy
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_auto_devops_deploy_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_security_sast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_security_secret_detection
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+
+- name: p_ci_templates_terraform_base_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects

lib/gitlab/usage_data_counters/known_events/common.yml

@@ -466,57 +466,6 @@
   redis_slot: terraform
   aggregation: weekly
   feature_flag: usage_data_p_terraform_state_api_unique_users
-# CI templates
-- name: p_ci_templates_5_min_production_app
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_auto_devops
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_aws_cf_deploy_ec2
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_aws_deploy_ecs
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_auto_devops_build
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_auto_devops_deploy
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_auto_devops_deploy_latest
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_security_sast
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_security_secret_detection
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
-- name: p_ci_templates_terraform_base_latest
-  category: ci_templates
-  redis_slot: ci_templates
-  aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 # Pipeline Authoring
 - name: o_pipeline_authoring_unique_users_committing_ciconfigfile
   category: pipeline_authoring

spec/lib/gitlab/ci/pipeline/chain/template_usage_spec.rb

@@ -27,7 +27,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::TemplateUsage do
       %w(Template-1 Template-2).each do |expected_template|
         expect(Gitlab::UsageDataCounters::CiTemplateUniqueCounter).to(
           receive(:track_unique_project_event)
-            .with(project_id: project.id, template: expected_template)
+            .with(project_id: project.id, template: expected_template, config_source: pipeline.config_source)
         )
       end
 

spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb

@@ -3,28 +3,88 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::UsageDataCounters::CiTemplateUniqueCounter do
-  let(:project_id) { 1 }
-
   describe '.track_unique_project_event' do
-    described_class::TEMPLATE_TO_EVENT.keys.each do |template|
-      context "when given template #{template}" do
-        it_behaves_like 'tracking unique hll events' do
-          subject(:request) { described_class.track_unique_project_event(project_id: project_id, template: template) }
+    using RSpec::Parameterized::TableSyntax
+
+    where(:template, :config_source, :expected_event) do
+      # Implicit Auto DevOps usage
+      'Auto-DevOps.gitlab-ci.yml'                     | :auto_devops_source | 'p_ci_templates_implicit_auto_devops'
+      'Jobs/Build.gitlab-ci.yml'                      | :auto_devops_source | 'p_ci_templates_implicit_auto_devops_build'
+      'Jobs/Deploy.gitlab-ci.yml'                     | :auto_devops_source | 'p_ci_templates_implicit_auto_devops_deploy'
+      'Security/SAST.gitlab-ci.yml'                   | :auto_devops_source | 'p_ci_templates_implicit_security_sast'
+      'Security/Secret-Detection.gitlab-ci.yml'       | :auto_devops_source | 'p_ci_templates_implicit_security_secret_detection'
+      # Explicit include:template usage
+      '5-Minute-Production-App.gitlab-ci.yml'         | :repository_source  | 'p_ci_templates_5_min_production_app'
+      'Auto-DevOps.gitlab-ci.yml'                     | :repository_source  | 'p_ci_templates_auto_devops'
+      'AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml' | :repository_source  | 'p_ci_templates_aws_cf_deploy_ec2'
+      'AWS/Deploy-ECS.gitlab-ci.yml'                  | :repository_source  | 'p_ci_templates_aws_deploy_ecs'
+      'Jobs/Build.gitlab-ci.yml'                      | :repository_source  | 'p_ci_templates_auto_devops_build'
+      'Jobs/Deploy.gitlab-ci.yml'                     | :repository_source  | 'p_ci_templates_auto_devops_deploy'
+      'Jobs/Deploy.latest.gitlab-ci.yml'              | :repository_source  | 'p_ci_templates_auto_devops_deploy_latest'
+      'Security/SAST.gitlab-ci.yml'                   | :repository_source  | 'p_ci_templates_security_sast'
+      'Security/Secret-Detection.gitlab-ci.yml'       | :repository_source  | 'p_ci_templates_security_secret_detection'
+      'Terraform/Base.latest.gitlab-ci.yml'           | :repository_source  | 'p_ci_templates_terraform_base_latest'
+    end
+
+    with_them do
+      it_behaves_like 'tracking unique hll events' do
+        subject(:request) { described_class.track_unique_project_event(project_id: project_id, template: template, config_source: config_source) }
+
+        let(:project_id) { 1 }
+        let(:target_id) { expected_event }
+        let(:expected_type) { instance_of(Integer) }
+      end
+    end
+
+    context 'known_events coverage tests' do
+      let(:project_id) { 1 }
+      let(:config_source) { :repository_source }
 
-          let(:target_id) { "p_ci_templates_#{described_class::TEMPLATE_TO_EVENT[template]}" }
-          let(:expected_type) { instance_of(Integer) }
+      # These tests help guard against missing "explicit" events in known_events/ci_templates.yml
+      context 'explicit include:template events' do
+        described_class::TEMPLATE_TO_EVENT.keys.each do |template|
+          it "does not raise error for #{template}" do
+            expect do
+              described_class.track_unique_project_event(project_id: project_id, template: template, config_source: config_source)
+            end.not_to raise_error
+          end
+        end
+      end
+
+      # This test is to help guard against missing "implicit" events in known_events/ci_templates.yml
+      it 'does not raise error for any template in an implicit Auto DevOps pipeline' do
+        project = create(:project, :auto_devops)
+        pipeline = double(project: project)
+        command = double
+        result = Gitlab::Ci::YamlProcessor.new(
+          Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops.new(pipeline, command).content,
+          project: project,
+          user: double,
+          sha: double
+        ).execute
+
+        config_source = :auto_devops_source
+
+        result.included_templates.each do |template|
+          expect do
+            described_class.track_unique_project_event(project_id: project.id, template: template, config_source: config_source)
+          end.not_to raise_error
         end
       end
     end
 
-    it 'does not track templates outside of TEMPLATE_TO_EVENT' do
-      expect(Gitlab::UsageDataCounters::HLLRedisCounter).not_to(
-        receive(:track_event)
-      )
+    context 'templates outside of TEMPLATE_TO_EVENT' do
+      let(:project_id) { 1 }
+      let(:config_source) { :repository_source }
+
       Dir.glob(File.join('lib', 'gitlab', 'ci', 'templates', '**'), base: Rails.root) do |template|
         next if described_class::TEMPLATE_TO_EVENT.key?(template)
 
-        described_class.track_unique_project_event(project_id: 1, template: template)
+        it "does not track #{template}" do
+          expect(Gitlab::UsageDataCounters::HLLRedisCounter).not_to(receive(:track_event))
+
+          described_class.track_unique_project_event(project_id: project_id, template: template, config_source: config_source)
+        end
       end
     end
   end