Allow to put migrations on hold for some time

Changelog: added

db/migrate/20220401110443_add_on_hold_until_column_for_batched_migration.rb

+# frozen_string_literal: true
+
+class AddOnHoldUntilColumnForBatchedMigration < Gitlab::Database::Migration[1.0]
+  def change
+    add_column :batched_background_migrations, :on_hold_until, :timestamptz,
+      comment: 'execution of this migration is on hold until this time'
+  end
+end

db/schema_migrations/20220401110443

+c35e6149ee37321cd446284a2f0405378f3e34e3c9d11b7d128e9afa6c83281d
\ No newline at end of file

db/structure.sql

@@ -11716,6 +11716,7 @@ CREATE TABLE batched_background_migrations (
     pause_ms integer DEFAULT 100 NOT NULL,
     max_batch_size integer,
     started_at timestamp with time zone,
+    on_hold_until timestamp with time zone,
     CONSTRAINT check_5bb0382d6f CHECK ((char_length(column_name) <= 63)),
     CONSTRAINT check_6b6a06254a CHECK ((char_length(table_name) <= 63)),
     CONSTRAINT check_batch_size_in_range CHECK ((batch_size >= sub_batch_size)),
@@ -11726,6 +11727,8 @@ CREATE TABLE batched_background_migrations (
     CONSTRAINT check_positive_sub_batch_size CHECK ((sub_batch_size > 0))
 );
 
+COMMENT ON COLUMN batched_background_migrations.on_hold_until IS 'execution of this migration is on hold until this time';
+
 CREATE SEQUENCE batched_background_migrations_id_seq
     START WITH 1
     INCREMENT BY 1

lib/gitlab/database/background_migration/batched_migration.rb

@@ -24,6 +24,10 @@ module Gitlab
 
         scope :queue_order, -> { order(id: :asc) }
         scope :queued, -> { with_statuses(:active, :paused) }
+
+        # on_hold_until is a temporary runtime status which puts execution "on hold"
+        scope :executable, -> { with_status(:active).where('on_hold_until IS NULL OR on_hold_until < NOW()') }
+
         scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do
           where(job_class_name: job_class_name, table_name: table_name, column_name: column_name)
             .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals
@@ -72,7 +76,7 @@ module Gitlab
         end
 
         def self.active_migration
-          with_status(:active).queue_order.first
+          executable.queue_order.first
         end
 
         def self.successful_rows_counts(migrations)
@@ -178,6 +182,10 @@ module Gitlab
           BatchOptimizer.new(self).optimize!
         end
 
+        def hold!(until_time: 10.minutes.from_now)
+          update!(on_hold_until: until_time)
+        end
+
         private
 
         def validate_batched_jobs_status

spec/lib/gitlab/database/background_migration/batched_migration_spec.rb

@@ -79,12 +79,23 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m
 
   describe '.active_migration' do
     let!(:migration1) { create(:batched_background_migration, :finished) }
-    let!(:migration2) { create(:batched_background_migration, :active) }
-    let!(:migration3) { create(:batched_background_migration, :active) }
 
-    it 'returns the first active migration according to queue order' do
-      expect(described_class.active_migration).to eq(migration2)
-      create(:batched_background_migration_job, :succeeded, batched_migration: migration1, batch_size: 1000)
+    context 'without migrations on hold' do
+      let!(:migration2) { create(:batched_background_migration, :active) }
+      let!(:migration3) { create(:batched_background_migration, :active) }
+
+      it 'returns the first active migration according to queue order' do
+        expect(described_class.active_migration).to eq(migration2)
+      end
+    end
+
+    context 'with migrations are on hold' do
+      let!(:migration2) { create(:batched_background_migration, :active, on_hold_until: 10.minutes.from_now) }
+      let!(:migration3) { create(:batched_background_migration, :active, on_hold_until: 2.minutes.ago) }
+
+      it 'returns the first active migration that is not on hold according to queue order' do
+        expect(described_class.active_migration).to eq(migration3)
+      end
     end
   end
 
@@ -518,6 +529,20 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m
     end
   end
 
+  describe '#hold!', :freeze_time do
+    subject { create(:batched_background_migration) }
+
+    let(:time) { 5.minutes.from_now }
+
+    it 'updates on_hold_until property' do
+      expect { subject.hold!(until_time: time) }.to change { subject.on_hold_until }.from(nil).to(time)
+    end
+
+    it 'defaults to 10 minutes' do
+      expect { subject.hold! }.to change { subject.on_hold_until }.from(nil).to(10.minutes.from_now)
+    end
+  end
+
   describe '.for_configuration' do
     let!(:migration) do
       create(