Add build trace chunks migration duration histogram

This commit adds another metric improving observability of cloud native
build logs. We are adding a histogram recording a duration of the
operation that migrates build trace chunks to object storage.

app/services/ci/update_build_state_service.rb

@@ -75,6 +75,7 @@ module Ci
 
       unless live_chunks_pending?
         metrics.increment_trace_operation(operation: :finalized)
+        metrics.observe_migration_duration(pending_state_seconds)
       end
 
       ::Gitlab::Ci::Trace::Checksum.new(build).then do |checksum|
@@ -130,7 +131,15 @@ module Ci
     end
 
     def pending_state_outdated?
-      Time.current - pending_state.created_at > ACCEPT_TIMEOUT
+      pending_state_duration > ACCEPT_TIMEOUT
+    end
+
+    def pending_state_duration
+      Time.current - pending_state.created_at
+    end
+
+    def pending_state_seconds
+      pending_state_duration.seconds
     end
 
     def build_state

lib/gitlab/ci/trace/metrics.rb

@@ -33,6 +33,10 @@ module Gitlab
           self.class.trace_bytes.increment({}, size.to_i)
         end
 
+        def observe_migration_duration(seconds)
+          self.class.finalize_histogram.observe({}, seconds.to_f)
+        end
+
         def self.trace_operations
           strong_memoize(:trace_operations) do
             name = :gitlab_ci_trace_operations_total
@@ -50,6 +54,17 @@ module Gitlab
             Gitlab::Metrics.counter(name, comment)
           end
         end
+
+        def self.finalize_histogram
+          strong_memoize(:finalize_histogram) do
+            name = :gitlab_ci_trace_finalize_duration_seconds
+            comment = 'Duration of build trace chunks migration to object storage'
+            buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 10.0, 30.0, 60.0, 300.0]
+            labels = {}
+
+            ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+          end
+        end
       end
     end
   end

spec/services/ci/update_build_state_service_spec.rb

@@ -131,6 +131,18 @@ RSpec.describe Ci::UpdateBuildStateService do
           .with(operation: :finalized)
       end
 
+      it 'records migration duration in a histogram' do
+        freeze_time do
+          create(:ci_build_pending_state, build: build, created_at: 0.5.seconds.ago)
+
+          execute_with_stubbed_metrics!
+        end
+
+        expect(metrics)
+          .to have_received(:observe_migration_duration)
+          .with(0.5)
+      end
+
       context 'when trace checksum is not valid' do
         it 'increments invalid trace metric' do
           execute_with_stubbed_metrics!