Directly refer application code from migration code

app/models/ci/build.rb

@@ -55,6 +55,11 @@ module Ci
       where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)',
         '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive)
     end
+
+    scope :without_archived_trace, ->() do
+      where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace)
+    end
+
     scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) }
     scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
     scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) }

db/post_migrate/20180529152628_archive_legacy_traces.rb

@@ -2,7 +2,7 @@ class ArchiveLegacyTraces < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
   DOWNTIME = false
-  BATCH_SIZE = 10_000
+  BATCH_SIZE = 5000
   BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces'
 
   disable_ddl_transaction!
@@ -14,25 +14,14 @@ class ArchiveLegacyTraces < ActiveRecord::Migration
 
     scope :finished, -> { where(status: [:success, :failed, :canceled]) }
 
-    scope :without_new_traces, ->() do
-      where('NOT EXISTS (?)',
-        ::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+    scope :without_archived_trace, -> do
+      where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)')
     end
   end
 
-  class JobArtifact < ActiveRecord::Base
-    self.table_name = 'ci_job_artifacts'
-
-    enum file_type: {
-      archive: 1,
-      metadata: 2,
-      trace: 3
-    }
-  end
-
   def up
     queue_background_migration_jobs_by_range_at_intervals(
-      ::ArchiveLegacyTraces::Build.finished.without_new_traces,
+      ::ArchiveLegacyTraces::Build.finished.without_archived_trace,
       BACKGROUND_MIGRATION_CLASS,
       5.minutes,
       batch_size: BATCH_SIZE)

lib/gitlab/background_migration/archive_legacy_traces.rb

@@ -5,73 +5,18 @@
 module Gitlab
   module BackgroundMigration
     class ArchiveLegacyTraces
-      class Build < ActiveRecord::Base
-        include ::HasStatus
-
-        self.table_name = 'ci_builds'
-        self.inheritance_column = :_type_disabled # Disable STI
-
-        belongs_to :project, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Project'
-        has_one :job_artifacts_trace, -> () { where(file_type: ArchiveLegacyTraces::JobArtifact.file_types[:trace]) }, class_name: 'ArchiveLegacyTraces::JobArtifact', foreign_key: :job_id
-        has_many :trace_chunks, foreign_key: :build_id, class_name: 'ArchiveLegacyTraces::BuildTraceChunk'
-    
-        scope :finished, -> { where(status: [:success, :failed, :canceled]) }
-
-        scope :without_new_traces, ->() do
-          finished.where('NOT EXISTS (?)',
-            BackgroundMigration::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
-        end
-
-        def trace
-          ::Gitlab::Ci::Trace.new(self)
-        end
-
-        def trace=(data)
-          raise NotImplementedError
-        end
-    
-        def old_trace
-          read_attribute(:trace)
-        end
-    
-        def erase_old_trace!
-          update_column(:trace, nil)
-        end
-      end
-
-      class JobArtifact < ActiveRecord::Base
-        self.table_name = 'ci_job_artifacts'
-
-        belongs_to :build
-        belongs_to :project
-
-        mount_uploader :file, JobArtifactUploader
-
-        enum file_type: {
-          archive: 1,
-          metadata: 2,
-          trace: 3
-        }
-      end
-
-      class BuildTraceChunk < ActiveRecord::Base
-        self.table_name = 'ci_build_trace_chunks'
-
-        belongs_to :build
-      end
-
-      class Project < ActiveRecord::Base
-        self.table_name = 'projects'
-
-        has_many :builds, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Build'
-      end
-
       def perform(start_id, stop_id)
-        BackgroundMigration::ArchiveLegacyTraces::Build
-          .finished
-          .without_new_traces
-          .where(id: (start_id..stop_id)).find_each do |build|
-            build.trace.archive!
+        # This background migrations directly refer ::Ci::Build model which is defined in application code.
+        # In general, migration code should be isolated as much as possible in order to be idempotent.
+        # However, `archive!` logic is too complicated to be replicated. So we chose a way to refer ::Ci::Build directly
+        # and we don't change the `archive!` logic until 11.1
+        ::Ci::Build.finished.without_archived_trace
+          .where(id: start_id..stop_id).find_each do |build|
+            begin
+              build.trace.archive!
+            rescue => e
+              Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}"
+            end
           end
       end
     end

lib/tasks/gitlab/traces.rake

@@ -8,9 +8,7 @@ namespace :gitlab do
       logger = Logger.new(STDOUT)
       logger.info('Archiving legacy traces')
 
-      Ci::Build.finished
-        .where('NOT EXISTS (?)',
-          Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+      Ci::Build.finished.without_archived_trace
         .order(id: :asc)
         .find_in_batches(batch_size: 1000) do |jobs|
         job_ids = jobs.map { |job| [job.id] }

spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb

@@ -14,7 +14,7 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2
     @legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path,
       build.created_at.utc.strftime("%Y_%m"),
       build.project_id.to_s)
-    
+
     FileUtils.mkdir_p(@legacy_trace_dir)
 
     @legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log")