Reschedule UpdateAllMirrorsWorker if needed

This adds rescheduling logic to the end of UpdateAllMirrorsWorker job.
We wait for a few seconds before we check capacity again and - if some
jobs already completed - we schedule another run of
UpdateAllMirrorsWorker.

ee/app/workers/repository_update_mirror_worker.rb

@@ -35,8 +35,10 @@ class RepositoryUpdateMirrorWorker
     fail_mirror(project, ex.message)
     raise UpdateError, "#{ex.class}: #{ex.message}"
   ensure
-    if !lease.exists? && lease.try_obtain && Gitlab::Mirror.reschedule_immediately?
-      UpdateAllMirrorsWorker.perform_async
+    unless Feature.enabled?(:update_all_mirrors_worker_rescheduling)
+      if !lease.exists? && lease.try_obtain && Gitlab::Mirror.reschedule_immediately?
+        UpdateAllMirrorsWorker.perform_async
+      end
     end
   end
 

ee/app/workers/update_all_mirrors_worker.rb

@@ -7,11 +7,26 @@ class UpdateAllMirrorsWorker
   LEASE_TIMEOUT = 5.minutes
   SCHEDULE_WAIT_TIMEOUT = 4.minutes
   LEASE_KEY = 'update_all_mirrors'.freeze
+  RESCHEDULE_WAIT = 10.seconds
 
   def perform
-    with_lease do
+    scheduling_ran = with_lease do
       schedule_mirrors!
     end
+
+    # If we didn't get the lease, exit early
+    return unless scheduling_ran
+
+    if Feature.enabled?(:update_all_mirrors_worker_rescheduling)
+      # Wait to give some jobs a chance to complete
+      Kernel.sleep(RESCHEDULE_WAIT)
+
+      # If there's capacity left now (some jobs completed),
+      # reschedule this job to enqueue more work.
+      #
+      # This is in addition to the regular (cron-like) scheduling of this job.
+      reschedule_if_capacity_left
+    end
   end
 
   # rubocop: disable CodeReuse/ActiveRecord
@@ -46,10 +61,18 @@ class UpdateAllMirrorsWorker
 
   private
 
+  def reschedule_if_capacity_left
+    return unless Gitlab::Mirror.reschedule_immediately?
+
+    UpdateAllMirrorsWorker.perform_async
+  end
+
   def with_lease
     if lease_uuid = try_obtain_lease
       yield
     end
+
+    lease_uuid
   ensure
     cancel_lease(lease_uuid) if lease_uuid
   end

ee/spec/workers/repository_update_mirror_worker_spec.rb

@@ -54,6 +54,8 @@ describe RepositoryUpdateMirrorWorker do
     context 'reschedule mirrors' do
       before do
         allow_any_instance_of(Projects::UpdateMirrorService).to receive(:execute).and_return(status: :success)
+
+        stub_feature_flags(update_all_mirrors_worker_rescheduling: false)
       end
 
       context 'when we obtain the lease' do

ee/spec/workers/update_all_mirrors_worker_spec.rb

@@ -23,6 +23,34 @@ describe UpdateAllMirrorsWorker do
 
       worker.perform
     end
+
+    context 'with update_all_mirrors_worker_rescheduling feature' do
+      before do
+        stub_feature_flags(update_all_mirrors_worker_rescheduling: true)
+      end
+
+      it 'sleeps a bit after scheduling mirrors' do
+        expect(Kernel).to receive(:sleep).with(described_class::RESCHEDULE_WAIT)
+
+        worker.perform
+      end
+
+      it 'reschedules the job if capacity is left' do
+        allow(Gitlab::Mirror).to receive(:reschedule_immediately?).and_return(true)
+
+        expect(described_class).to receive(:perform_async)
+
+        worker.perform
+      end
+
+      it 'does not reschedule the job if no capacity left' do
+        allow(Gitlab::Mirror).to receive(:reschedule_immediately?).and_return(false)
+
+        expect(described_class).not_to receive(:perform_async)
+
+        worker.perform
+      end
+    end
   end
 
   describe '#schedule_mirrors!' do