Transfer job archives after creation

app/models/ci/job_artifact.rb

 module Ci
   class JobArtifact < ActiveRecord::Base
+    include AfterCommitQueue
     extend Gitlab::Ci::Model
 
     belongs_to :project

app/models/lfs_object.rb

 class LfsObject < ActiveRecord::Base
   prepend EE::LfsObject
+  include AfterCommitQueue
 
   has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :projects, through: :lfs_objects_projects

app/uploaders/job_artifact_uploader.rb

 class JobArtifactUploader < ObjectStoreUploader
   storage_options Gitlab.config.artifacts
+  after :store, :schedule_migration_to_object_storage
 
   def self.local_store_path
     Gitlab.config.artifacts.path

changelogs/unreleased-ee/zj-auto-upload-job-artifacts.yml

+---
+title: Transfer job archives to object storage after creation
+merge_request:
+author:
+type: added

config/gitlab.yml.example

@@ -771,6 +771,7 @@ test:
     object_store:
       enabled: false
       remote_directory: artifacts # The bucket name
+      background_upload: false
       connection:
         provider: AWS # Only AWS supported at the moment
         aws_access_key_id: AWS_ACCESS_KEY_ID

ee/app/uploaders/object_store_uploader.rb

@@ -116,9 +116,14 @@ class ObjectStoreUploader < CarrierWave::Uploader::Base
     end
   end
 
-  def schedule_migration_to_object_storage(new_file)
+  def schedule_migration_to_object_storage(*args)
     if self.class.object_store_enabled? && licensed? && file_storage?
-      ObjectStorageUploadWorker.perform_async(self.class.name, model.class.name, mounted_as, model.id)
+      uploader = self
+      mount_field = mounted_as
+
+      model.run_after_commit do
+        ObjectStorageUploadWorker.perform_async(uploader.class.name, self.class.name, mount_field, id)
+      end
     end
   end
 

ee/app/workers/object_storage_upload_worker.rb

 class ObjectStorageUploadWorker
   include ApplicationWorker
 
+  sidekiq_options retry: 5
+
   def perform(uploader_class_name, subject_class_name, file_field, subject_id)
     uploader_class = uploader_class_name.constantize
     subject_class = subject_class_name.constantize
@@ -8,7 +10,9 @@ class ObjectStorageUploadWorker
     return unless uploader_class.object_store_enabled?
     return unless uploader_class.background_upload_enabled?
 
-    subject = subject_class.find(subject_id)
+    subject = subject_class.find_by(id: subject_id)
+    return unless subject
+
     file = subject.public_send(file_field) # rubocop:disable GitlabSecurity/PublicSend
 
     return unless file.licensed?

spec/workers/object_storage_upload_worker_spec.rb → spec/ee/workers/object_storage_upload_worker_spec.rb

@@ -60,7 +60,7 @@ describe ObjectStorageUploadWorker do
 
       context 'and remote storage is defined' do
         before do
-          stub_artifacts_object_storage
+          stub_artifacts_object_storage(background_upload: true)
         end
 
         it "migrates file to remote storage" do
@@ -94,7 +94,7 @@ describe ObjectStorageUploadWorker do
 
       context 'and remote storage is defined' do
         before do
-          stub_artifacts_object_storage
+          stub_artifacts_object_storage(background_upload: true)
         end
 
         it "migrates file to remote storage" do

spec/models/ci/job_artifact_spec.rb

@@ -12,6 +12,44 @@ describe Ci::JobArtifact do
   it { is_expected.to respond_to(:created_at) }
   it { is_expected.to respond_to(:updated_at) }
 
+  describe 'callbacks' do
+    subject { create(:ci_job_artifact, :archive) }
+
+    describe '#schedule_migration_to_object_storage' do
+      context 'when object storage is disabled' do
+        it 'does not schedule the migration' do
+          expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
+
+          subject
+        end
+      end
+
+      context 'when object storage is enabled' do
+        before do
+          stub_artifacts_object_storage
+        end
+
+        it 'schedules the model for migration' do
+          expect(ObjectStorageUploadWorker).to receive(:perform_async).with('JobArtifactUploader', described_class.name, :file, kind_of(Numeric))
+
+          subject
+        end
+      end
+
+      context 'when object storage is unlicensed' do
+        before do
+          stub_artifacts_object_storage(licensed: false)
+        end
+
+        it 'does not schedule the migration' do
+          expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
+
+          subject
+        end
+      end
+    end
+  end
+
   describe '#set_size' do
     it 'sets the size' do
       expect(artifact.size).to eq(106365)

spec/requests/api/runner_spec.rb

@@ -1085,6 +1085,8 @@ describe API::Runner do
             let(:stored_artifacts_size) { job.reload.artifacts_size }
 
             before do
+              stub_artifacts_object_storage(enabled: false)
+
               post(api("/jobs/#{job.id}/artifacts"), post_data, headers_with_token)
             end
 
@@ -1154,13 +1156,6 @@ describe API::Runner do
 
         context 'when job has artifacts' do
           let(:job) { create(:ci_build) }
-          let(:store) { JobArtifactUploader::LOCAL_STORE }
-
-          before do
-            create(:ci_job_artifact, :archive, file_store: store, job: job)
-
-            download_artifact
-          end
 
           context 'when using job token' do
             context 'when artifacts are stored locally' do
@@ -1170,6 +1165,11 @@ describe API::Runner do
               end
 
               it 'download artifacts' do
+                stub_artifacts_object_storage(enabled: false)
+                create(:ci_job_artifact, :archive, job: job)
+
+                download_artifact
+
                 expect(response).to have_gitlab_http_status(200)
                 expect(response.headers).to include download_headers
               end
@@ -1180,6 +1180,10 @@ describe API::Runner do
               let!(:job) { create(:ci_build) }
 
               it 'download artifacts' do
+                create(:ci_job_artifact, :archive, :remote_store, job: job)
+
+                download_artifact
+
                 expect(response).to have_gitlab_http_status(302)
               end
             end
@@ -1189,6 +1193,8 @@ describe API::Runner do
             let(:token) { job.project.runners_token }
 
             it 'responds with forbidden' do
+              download_artifact
+
               expect(response).to have_gitlab_http_status(403)
             end
           end

spec/support/stub_object_storage.rb

 module StubConfiguration
-  def stub_object_storage_uploader(config:, uploader:, remote_directory:, enabled: true, licensed: true)
+  def stub_object_storage_uploader(config:, uploader:, remote_directory:, enabled: true, licensed: true, background_upload: nil)
     Fog.mock!
 
     allow(config).to receive(:enabled) { enabled }
+    allow(config).to receive(:background_upload) { background_upload } if background_upload
 
     stub_licensed_features(object_storage: licensed) unless licensed == :skip