Geo Framework: Persistent failures block

Due to lacking order by retry_at some items
could stuck in the queue of being processed

ee/app/models/concerns/geo/replicable_registry.rb

@@ -36,7 +36,7 @@ module Geo::ReplicableRegistry
     include ::Delay
 
     scope :failed, -> { with_state(:failed) }
-    scope :needs_sync_again, -> { failed.retry_due }
+    scope :needs_sync_again, -> { failed.retry_due.order(Gitlab::Database.nulls_first_order(:retry_at)) }
     scope :never_attempted_sync, -> { pending.where(last_synced_at: nil) }
     scope :ordered, -> { order(:id) }
     scope :pending, -> { with_state(:pending) }

ee/changelogs/unreleased/293972-geo-framework-persistent-failures-block-slow-down-retries-of-tr.yml

+---
+title: 'Geo: Prioritize resyncing new failures over persistent failures'
+merge_request: 54585
+author:
+type: fixed

ee/spec/support/shared_examples/models/geo_framework_registry_shared_examples.rb

@@ -41,6 +41,15 @@ RSpec.shared_examples 'a Geo framework registry' do
         expect(result).to include(failed_item2)
         expect(result).not_to include(failed_item1)
       end
+
+      it 'orders records according to retry_at' do
+        failed_item1.update!(retry_at: 2.days.ago)
+        failed_item2.update!(retry_at: 4.days.ago)
+
+        result = described_class.find_registries_needs_sync_again(batch_size: 10)
+
+        expect(result.first).to eq failed_item2
+      end
     end
   end
 end