Merge branch '259619-store-md5-checksum-for-archived-traces' into 'master'

Store MD5 checksum for archived job traces

See merge request gitlab-org/gitlab!71184

app/models/ci/build_trace_metadata.rb

@@ -37,8 +37,10 @@ module Ci
       increment!(:archival_attempts, touch: :last_archival_attempt_at)
     end
 
-    def track_archival!(trace_artifact_id)
-      update!(trace_artifact_id: trace_artifact_id, archived_at: Time.current)
+    def track_archival!(trace_artifact_id, checksum)
+      update!(trace_artifact_id: trace_artifact_id,
+        checksum: checksum,
+        archived_at: Time.current)
     end
 
     def archival_attempts_message

lib/gitlab/ci/trace.rb

@@ -236,35 +236,7 @@ module Gitlab
       end
 
       def archive_stream!(stream)
-        clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
-          create_build_trace!(job, clone_path)
-        end
-      end
-
-      def clone_file!(src_stream, temp_dir)
-        FileUtils.mkdir_p(temp_dir)
-        Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path|
-          temp_path = File.join(dir_path, "job.log")
-          FileUtils.touch(temp_path)
-          size = IO.copy_stream(src_stream, temp_path)
-          raise ArchiveError, 'Failed to copy stream' unless size == src_stream.size
-
-          yield(temp_path)
-        end
-      end
-
-      def create_build_trace!(job, path)
-        File.open(path) do |stream|
-          # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration
-          # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307
-          trace_artifact = job.create_job_artifacts_trace!(
-            project: job.project,
-            file_type: :trace,
-            file: stream,
-            file_sha256: self.class.sha256_hexdigest(path))
-
-          trace_metadata.track_archival!(trace_artifact.id)
-        end
+        ::Gitlab::Ci::Trace::Archive.new(job, trace_metadata).execute!(stream)
       end
 
       def trace_metadata

lib/gitlab/ci/trace/archive.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Trace
+      class Archive
+        include ::Gitlab::Utils::StrongMemoize
+        include Checksummable
+
+        def initialize(job, trace_metadata)
+          @job = job
+          @trace_metadata = trace_metadata
+        end
+
+        def execute!(stream)
+          clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
+            md5_checksum    = self.class.md5_hexdigest(clone_path)
+            sha256_checksum = self.class.sha256_hexdigest(clone_path)
+
+            job.transaction do
+              trace_artifact = create_build_trace!(clone_path, sha256_checksum)
+              trace_metadata.track_archival!(trace_artifact.id, md5_checksum)
+            end
+          end
+        end
+
+        private
+
+        attr_reader :job, :trace_metadata
+
+        def clone_file!(src_stream, temp_dir)
+          FileUtils.mkdir_p(temp_dir)
+          Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path|
+            temp_path = File.join(dir_path, "job.log")
+            FileUtils.touch(temp_path)
+            size = IO.copy_stream(src_stream, temp_path)
+            raise ::Gitlab::Ci::Trace::ArchiveError, 'Failed to copy stream' unless size == src_stream.size
+
+            yield(temp_path)
+          end
+        end
+
+        def create_build_trace!(path, file_sha256)
+          File.open(path) do |stream|
+            # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration
+            # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307
+            job.create_job_artifacts_trace!(
+              project: job.project,
+              file_type: :trace,
+              file: stream,
+              file_sha256: file_sha256)
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/trace/archive_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Trace::Archive do
+  let_it_be(:job) { create(:ci_build, :success, :trace_live) }
+  let_it_be(:trace_metadata) { create(:ci_build_trace_metadata, build: job) }
+  let_it_be(:src_checksum) do
+    job.trace.read { |stream| Digest::MD5.hexdigest(stream.raw) }
+  end
+
+  describe '#execute' do
+    subject { described_class.new(job, trace_metadata) }
+
+    it 'computes and assigns checksum' do
+      Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
+        expect { subject.execute!(stream) }.to change { Ci::JobArtifact.count }.by(1)
+      end
+
+      expect(trace_metadata.checksum).to eq(src_checksum)
+      expect(trace_metadata.trace_artifact).to eq(job.job_artifacts_trace)
+    end
+  end
+end

spec/models/ci/build_trace_metadata_spec.rb

@@ -88,14 +88,16 @@ RSpec.describe Ci::BuildTraceMetadata do
   describe '#track_archival!' do
     let(:trace_artifact) { create(:ci_job_artifact) }
     let(:metadata) { create(:ci_build_trace_metadata) }
+    let(:checksum) { SecureRandom.hex }
 
     it 'stores the artifact id and timestamp' do
       expect(metadata.trace_artifact_id).to be_nil
 
-      metadata.track_archival!(trace_artifact.id)
+      metadata.track_archival!(trace_artifact.id, checksum)
       metadata.reload
 
       expect(metadata.trace_artifact_id).to eq(trace_artifact.id)
+      expect(metadata.checksum).to eq(checksum)
       expect(metadata.archived_at).to be_like_time(Time.current)
     end
   end