Apply retry at time to verification_failed_batch

ee/lib/gitlab/geo/verification_state.rb

@@ -35,6 +35,7 @@ module Gitlab
         scope :checksummed, -> { where.not(verification_checksum: nil) }
         scope :not_checksummed, -> { where(verification_checksum: nil) }
         scope :verification_timed_out, -> { verification_started.where("verification_started_at < ?", VERIFICATION_TIMEOUT.ago) }
+        scope :retry_due, -> { where(arel_table[:verification_retry_at].eq(nil).or(arel_table[:verification_retry_at].lt(Time.current))) }
         scope :needs_verification, -> { with_verification_state(:verification_pending, :verification_failed) }
         # rubocop:enable CodeReuse/ActiveRecord
 
@@ -124,7 +125,7 @@ module Gitlab
 
         # Overridden by Geo::VerifiableRegistry
         def verification_failed_batch_relation(batch_size:)
-          verification_failed.order(Gitlab::Database.nulls_first_order(:verification_retry_at)).limit(batch_size) # rubocop:disable CodeReuse/ActiveRecord
+          verification_failed.retry_due.order(Gitlab::Database.nulls_first_order(:verification_retry_at)).limit(batch_size) # rubocop:disable CodeReuse/ActiveRecord
         end
 
         # @return [Integer] number of records that need verification

ee/spec/lib/gitlab/geo/verification_state_spec.rb

@@ -90,10 +90,15 @@ RSpec.describe Gitlab::Geo::VerificationState do
     let(:other_failed_ids) { other_failed_records.map { |result| result['id'] } }
 
     before do
-      subject.verification_started!
+      subject.verification_started
       subject.verification_failed_with_message!('foo')
     end
 
+    context 'with a failed record with retry due' do
+      before do
+        subject.update!(verification_retry_at: 1.minute.ago)
+      end
+
       it 'returns IDs of rows pending verification' do
         expect(subject.class.verification_failed_batch(batch_size: 3)).to include(subject.id)
       end
@@ -131,6 +136,15 @@ RSpec.describe Gitlab::Geo::VerificationState do
       end
     end
 
+    context 'when verification_retry_at is in the future' do
+      it 'does not return the row' do
+        subject.update!(verification_retry_at: 1.minute.from_now)
+
+        expect(subject.class.verification_failed_batch(batch_size: 3)).not_to include(subject.id)
+      end
+    end
+  end
+
   describe '.fail_verification_timeouts' do
     before do
       subject.verification_started!

ee/spec/support/shared_examples/models/geo_verifiable_registry_shared_examples.rb

@@ -42,7 +42,12 @@ RSpec.shared_examples 'a Geo verifiable registry' do
       subject.verification_failed_with_message!('foo')
     end
 
-    it 'returns IDs of rows which are synced and failed verification' do
+    context 'with a failed record with retry due' do
+      before do
+        subject.update!(verification_retry_at: 1.minute.ago)
+      end
+
+      it 'returns IDs of rows which are synced and have failed verification' do
         expect(described_class.verification_failed_batch(batch_size: 4)).to match_array([subject.model_record_id])
       end
 
@@ -67,6 +72,15 @@ RSpec.shared_examples 'a Geo verifiable registry' do
       end
     end
 
+    context 'when verification_retry_at is in the future' do
+      it 'does not return the row which failed verification' do
+        subject.update!(verification_retry_at: 1.minute.from_now)
+
+        expect(subject.class.verification_failed_batch(batch_size: 4)).not_to include(subject.model_record_id)
+      end
+    end
+  end
+
   describe '.needs_verification_count' do
     before do
       subject.save!