Merge branch 'feature/gb/invalid-build-trace-chunks-logs' into 'master'

Log detected invalid build trace chunks

See merge request gitlab-org/gitlab!44409

app/services/ci/update_build_state_service.rb

@@ -6,6 +6,7 @@ module Ci
     include ::Gitlab::ExclusiveLeaseHelpers
 
     Result = Struct.new(:status, :backoff, keyword_init: true)
+    InvalidTraceError = Class.new(StandardError)
 
     ACCEPT_TIMEOUT = 5.minutes.freeze
 
@@ -76,8 +77,20 @@ module Ci
         metrics.increment_trace_operation(operation: :finalized)
       end
 
-      unless ::Gitlab::Ci::Trace::Checksum.new(build).valid?
-        metrics.increment_trace_operation(operation: :invalid)
+      ::Gitlab::Ci::Trace::Checksum.new(build).then do |checksum|
+        unless checksum.valid?
+          metrics.increment_trace_operation(operation: :invalid)
+
+          next unless log_invalid_chunks?
+
+          ::Gitlab::ErrorTracking.log_exception(InvalidTraceError.new,
+            project_path: build.project.full_path,
+            build_id: build.id,
+            state_crc32: checksum.state_crc32,
+            chunks_crc32: checksum.chunks_crc32,
+            chunks_count: checksum.chunks_count
+          )
+        end
       end
     end
 
@@ -182,5 +195,9 @@ module Ci
     def chunks_migration_enabled?
       ::Gitlab::Ci::Features.accept_trace?(build.project)
     end
+
+    def log_invalid_chunks?
+      ::Gitlab::Ci::Features.log_invalid_trace_chunks?(build.project)
+    end
   end
 end

config/feature_flags/ops/ci_trace_log_invalid_chunks.yml

+---
+name: ci_trace_log_invalid_chunks
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44409
+rollout_issue_url:
+type: ops
+group: group::continuous integration
+default_enabled: false

lib/gitlab/ci/features.rb

@@ -59,6 +59,10 @@ module Gitlab
           ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: false)
       end
 
+      def self.log_invalid_trace_chunks?(project)
+        ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false)
+      end
+
       def self.new_artifact_file_reader_enabled?(project)
         ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: true)
       end

lib/gitlab/ci/trace/checksum.rb

@@ -30,12 +30,14 @@ module Gitlab
         end
 
         def state_crc32
-          strong_memoize(:crc32) { build.pending_state&.crc32 }
+          strong_memoize(:state_crc32) { build.pending_state&.crc32 }
         end
 
         def chunks_crc32
-          trace_chunks.reduce(0) do |crc32, chunk|
-            Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk))
+          strong_memoize(:chunks_crc32) do
+            trace_chunks.reduce(0) do |crc32, chunk|
+              Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk))
+            end
           end
         end
 
@@ -62,6 +64,10 @@ module Gitlab
           end
         end
 
+        def chunks_count
+          trace_chunks.to_a.size
+        end
+
         private
 
         def chunk_size(chunk)