Reschedule Delete Orphaned Deployments BG migration

This commit reschedules the initial BG migration
with a smaller batch size to correct the failured
batches by statement timeouts.

Changelog: other

db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb

@@ -11,13 +11,12 @@ class ScheduleDeleteOrphanedDeployments < ActiveRecord::Migration[6.1]
   disable_ddl_transaction!
 
   def up
-    queue_background_migration_jobs_by_range_at_intervals(
-      define_batchable_model('deployments'),
-      MIGRATION,
-      DELAY_INTERVAL,
-      batch_size: BATCH_SIZE,
-      track_jobs: true
-    )
+    # no-op.
+    # This background migration is rescheduled in 20210722010101_cleanup_delete_orphaned_deployments_background_migration.rb
+    # with a smaller batch size, because the initial attempt caused
+    # 80 failures out of 1639 batches (faiulre rate is 4.88%) due to statement timeouts,
+    # that takes approx. 1 hour to perform a cleanup/sync migration.
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/335071#note_618380503 for more information.
   end
 
   def down

db/post_migrate/20210722010101_reschedule_delete_orphaned_deployments.rb

+# frozen_string_literal: true
+
+class RescheduleDeleteOrphanedDeployments < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  MIGRATION = 'DeleteOrphanedDeployments'
+  BATCH_SIZE = 10_000
+  DELAY_INTERVAL = 2.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    Gitlab::BackgroundMigration.steal(MIGRATION)
+    Gitlab::Database::BackgroundMigrationJob.for_migration_class(MIGRATION).delete_all
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      define_batchable_model('deployments'),
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20210722010101

+bbd39849499d16f92a5129506a87a6b253f209200bcb3a63c2432862c4b78aae
\ No newline at end of file

spec/migrations/schedule_delete_orphaned_deployments_spec.rb → spec/migrations/reschedule_delete_orphaned_deployments_spec.rb

@@ -4,7 +4,7 @@ require 'spec_helper'
 
 require_migration!
 
-RSpec.describe ScheduleDeleteOrphanedDeployments, :sidekiq, schema: 20210617161348 do
+RSpec.describe RescheduleDeleteOrphanedDeployments, :sidekiq, schema: 20210617161348 do
   let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
   let!(:project) { table(:projects).create!(namespace_id: namespace.id) }
   let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) }
@@ -22,6 +22,31 @@ RSpec.describe ScheduleDeleteOrphanedDeployments, :sidekiq, schema: 202106171613
     stub_const("#{described_class}::BATCH_SIZE", 1)
   end
 
+  it 'steal existing background migration jobs' do
+    expect(Gitlab::BackgroundMigration).to receive(:steal).with('DeleteOrphanedDeployments')
+
+    migrate!
+  end
+
+  it 'cleans up background migration jobs tracking records' do
+    old_successful_job = background_migration_jobs.create!(
+      class_name: 'DeleteOrphanedDeployments',
+      status: Gitlab::Database::BackgroundMigrationJob.statuses[:succeeded],
+      arguments: [table(:deployments).minimum(:id), table(:deployments).minimum(:id)]
+    )
+
+    old_pending_job = background_migration_jobs.create!(
+      class_name: 'DeleteOrphanedDeployments',
+      status: Gitlab::Database::BackgroundMigrationJob.statuses[:pending],
+      arguments: [table(:deployments).maximum(:id), table(:deployments).maximum(:id)]
+    )
+
+    migrate!
+
+    expect { old_successful_job.reload }.to raise_error(ActiveRecord::RecordNotFound)
+    expect { old_pending_job.reload }.to raise_error(ActiveRecord::RecordNotFound)
+  end
+
   it 'schedules DeleteOrphanedDeployments background jobs' do
     Sidekiq::Testing.fake! do
       freeze_time do