Simplify builds queuing metrics and add feature flag

This commit adds a feature flag to incrementally roll out new metrics
for builds queuing. It also simplifies them a bit and improves
performance.

app/services/ci/register_job_service.rb

@@ -14,6 +14,8 @@ module Ci
     end
 
     def execute(params = {})
+      @metrics.increment_queue_operation(:queue_attempt)
+
       @metrics.observe_queue_time do
         process_queue(params)
       end
@@ -23,8 +25,6 @@ module Ci
 
     # rubocop: disable CodeReuse/ActiveRecord
     def process_queue(params)
-      @metrics.increment_queue_operation(:queue_attempt)
-
       builds =
         if runner.instance_type?
           builds_for_shared_runner
@@ -47,13 +47,13 @@ module Ci
         builds = builds.queued_before(params[:job_age].seconds.ago)
       end
 
-      @metrics.observe_queue_depth(:depth, builds.to_a.size)
+      @metrics.observe_queue_depth(:initialized, -> { builds.to_a.size })
 
       valid = true
       depth = 0
 
-      builds.each_with_index do |build, index|
-        depth = index
+      builds.each do |build|
+        depth += 1
         @metrics.increment_queue_operation(:queue_iteration)
 
         result = process_build(build, params)
@@ -61,7 +61,7 @@ module Ci
 
         if result.valid?
           @metrics.register_success(result.build)
-          @metrics.observe_queue_depth(:effective, depth)
+          @metrics.observe_queue_depth(:effective, -> { depth })
 
           return result # rubocop:disable Cop/AvoidReturnFromBlocks
         else
@@ -72,7 +72,7 @@ module Ci
       end
 
       @metrics.increment_queue_operation(:queue_conflict) unless valid
-      @metrics.observe_queue_depth(:ineffective, depth) unless valid
+      @metrics.observe_queue_depth(:ineffective, -> { depth }) unless valid
       @metrics.register_failure
 
       Result.new(nil, nil, valid)
@@ -138,11 +138,11 @@ module Ci
       failure_reason, _ = pre_assign_runner_checks.find { |_, check| check.call(build, params) }
 
       if failure_reason
-        @metrics.increment_queue_operation(:runner_checks_failed)
+        @metrics.increment_queue_operation(:runner_pre_assign_checks_failed)
 
         build.drop!(failure_reason)
       else
-        @metrics.increment_queue_operation(:runner_checks_success)
+        @metrics.increment_queue_operation(:runner_pre_assign_checks_success)
 
         build.run!
       end

config/feature_flags/development/gitlab_ci_builds_queuing_metrics.yml

+---
+name: gitlab_ci_builds_queuing_metrics
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54909
+rollout_issue_url: 
+milestone: '13.10'
+type: development
+group: group::continuous integration
+default_enabled: false

lib/gitlab/ci/queue/metrics.rb

@@ -23,15 +23,15 @@ module Gitlab
           :queue_attempt,
           :queue_conflict,
           :queue_iteration,
-          :runner_checks_failed,
-          :runner_checks_success
-        ].freeze
+          :runner_pre_assign_checks_failed,
+          :runner_pre_assign_checks_success
+        ].to_set.freeze
 
         QUEUE_DEPTH_HISTOGRAMS = [
-          :depth,
+          :initialized,
           :effective,
           :ineffective
-        ].freeze
+        ].to_set.freeze
 
         attr_reader :runner
 
@@ -70,19 +70,23 @@ module Gitlab
         # rubocop: enable CodeReuse/ActiveRecord
 
         def increment_queue_operation(operation)
-          unless OPERATION_COUNTERS.include?(operation)
-            raise ArgumentError, "unknown queue operation: #{operation}" unless Rails.env.production?
+          return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
+
+          if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation)
+            raise ArgumentError, "unknown queue operation: #{operation}"
           end
 
           self.class.queue_operations_total.increment(operation: operation)
         end
 
-        def observe_queue_depth(queue, size)
-          unless QUEUE_DEPTH_HISTOGRAMS.include?(queue)
-            raise ArgumentError, "unknown queue depth label: #{queue}" unless Rails.env.production?
+        def observe_queue_depth(queue, size_proc)
+          return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
+
+          if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue)
+            raise ArgumentError, "unknown queue depth label: #{queue}"
           end
 
-          self.class.queue_depth_size_total.observe({ queue: queue }, size.to_f)
+          self.class.queue_depth_size_total.observe({ queue: queue }, size_proc.call.to_f)
         end
 
         def observe_queue_time
@@ -90,6 +94,8 @@ module Gitlab
 
           result = yield
 
+          return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
+
           seconds = ::Gitlab::Metrics::System.monotonic_time - start_time
           self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f)