Allow small variance when running migration jobs

When running batched background migrations, allow a small variance in
the `interval_elapsed?` check, to prevent issues where the cron skips a
job execution due to missing the next execution window by only several
seconds.

Without the variance, a job scheduled to run every 2 minutes will often
run closer to every 3 minutes in practice.

app/workers/database/batched_background_migration_worker.rb

@@ -10,6 +10,7 @@ module Database
 
     LEASE_TIMEOUT_MULTIPLIER = 3
     MINIMUM_LEASE_TIMEOUT = 10.minutes.freeze
+    INTERVAL_VARIANCE = 5.seconds.freeze
 
     def perform
       return unless Feature.enabled?(:execute_batched_migrations_on_schedule, type: :ops) && active_migration
@@ -22,7 +23,7 @@ module Database
         # models don't inherit from ApplicationRecord
         active_migration.reload # rubocop:disable Cop/ActiveRecordAssociationReload
 
-        run_active_migration if active_migration.active? && active_migration.interval_elapsed?
+        run_active_migration if active_migration.active? && active_migration.interval_elapsed?(variance: INTERVAL_VARIANCE)
       end
     end
 

lib/gitlab/database/background_migration/batched_migration.rb

@@ -27,8 +27,11 @@ module Gitlab
           active.queue_order.first
         end
 
-        def interval_elapsed?
-          last_job.nil? || last_job.created_at <= Time.current - interval
+        def interval_elapsed?(variance: 0)
+          return true unless last_job
+
+          interval_with_variance = interval - variance
+          last_job.created_at <= Time.current - interval_with_variance
         end
 
         def create_batched_job!(min, max)

spec/lib/gitlab/database/background_migration/batched_migration_spec.rb

@@ -87,6 +87,34 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m
           end
         end
       end
+
+      context 'when an interval variance is given' do
+        let(:variance) { 2.seconds }
+
+        context 'when the last job is less than an interval with variance old' do
+          it 'returns false' do
+            freeze_time do
+              create(:batched_background_migration_job,
+                batched_migration: batched_migration,
+                created_at: Time.current - 1.minute - 57.seconds)
+
+              expect(batched_migration.interval_elapsed?(variance: variance)).to eq(false)
+            end
+          end
+        end
+
+        context 'when the last job is more than an interval with variance old' do
+          it 'returns true' do
+            freeze_time do
+              create(:batched_background_migration_job,
+                batched_migration: batched_migration,
+                created_at: Time.current - 1.minute - 58.seconds)
+
+              expect(batched_migration.interval_elapsed?(variance: variance)).to eq(true)
+            end
+          end
+        end
+      end
     end
   end
 

spec/workers/database/batched_background_migration_worker_spec.rb

@@ -40,12 +40,13 @@ RSpec.describe Database::BatchedBackgroundMigrationWorker, '#perform', :clean_gi
       let(:lease_timeout) { 15.minutes }
       let(:lease_key) { 'batched_background_migration_worker' }
       let(:migration) { build(:batched_background_migration, :active, interval: job_interval) }
+      let(:interval_variance) { described_class::INTERVAL_VARIANCE }
 
       before do
         allow(Gitlab::Database::BackgroundMigration::BatchedMigration).to receive(:active_migration)
           .and_return(migration)
 
-        allow(migration).to receive(:interval_elapsed?).and_return(true)
+        allow(migration).to receive(:interval_elapsed?).with(variance: interval_variance).and_return(true)
         allow(migration).to receive(:reload)
       end
 
@@ -66,7 +67,7 @@ RSpec.describe Database::BatchedBackgroundMigrationWorker, '#perform', :clean_gi
         it 'does not run the migration' do
           expect_to_obtain_exclusive_lease(lease_key, timeout: lease_timeout)
 
-          expect(migration).to receive(:interval_elapsed?).and_return(false)
+          expect(migration).to receive(:interval_elapsed?).with(variance: interval_variance).and_return(false)
 
           expect(worker).not_to receive(:run_active_migration)