Avoid backfill worker to stuck when clone take longer than run time

[ci skip]

app/services/geo/repository_backfill_service.rb

 module Geo
   class RepositoryBackfillService
-    attr_reader :project
+    attr_reader :project, :backfill_lease
 
-    LEASE_TIMEOUT = 8.hours.freeze
+    LEASE_TIMEOUT    = 8.hours.freeze
+    LEASE_KEY_PREFIX = 'repository_backfill_service'.freeze
 
-    def initialize(project)
-      @project = project
+    def initialize(project_id, backfill_lease)
+      @project = Project.find(project_id)
+      @backfill_lease = backfill_lease
     end
 
     def execute
@@ -16,7 +18,6 @@ module Geo
           registry.last_repository_synced_at = started_at
           registry.last_repository_successful_sync_at = finished_at if finished_at
           registry.save
-
           log('Finished repository sync')
         end
       end
@@ -58,22 +59,19 @@ module Geo
     def try_obtain_lease
       log('Trying to obtain lease to sync repository')
 
-      uuid = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
+      repository_lease = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
 
-      log('Could not obtain lease to sync repository') and return unless uuid
+      log('Could not obtain lease to sync repository') and return unless repository_lease
 
       yield
 
-      log('Releasing lease to sync repository')
-      release_lease(uuid)
-    end
-
-    def release_lease(uuid)
-      Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+      log('Releasing leases to sync repository')
+      Gitlab::ExclusiveLease.cancel(lease_key, repository_lease)
+      Gitlab::ExclusiveLease.cancel(LEASE_KEY_PREFIX, backfill_lease)
     end
 
     def lease_key
-      @key ||= "repository_backfill_service:#{project.id}"
+      @key ||= "#{LEASE_KEY_PREFIX}:#{project.id}"
     end
 
     def primary_ssh_path_prefix

app/workers/geo_backfill_worker.rb

@@ -2,8 +2,7 @@ class GeoBackfillWorker
   include Sidekiq::Worker
   include CronjobQueue
 
-  LEASE_TIMEOUT = 8.hours.freeze
-  RUN_TIME      = 5.minutes.to_i.freeze
+  RUN_TIME = 5.minutes.to_i.freeze
 
   def perform
     start = Time.now
@@ -18,8 +17,8 @@ class GeoBackfillWorker
       project = Project.find(project_id)
       next if project.repository_exists?
 
-      try_obtain_lease do
-        Geo::RepositoryBackfillService.new(project).execute
+      try_obtain_lease do |lease|
+        GeoSingleRepositoryBackfillWorker.new.perform(project_id, lease)
       end
     end
 
@@ -37,21 +36,19 @@ class GeoBackfillWorker
   end
 
   def try_obtain_lease
-    uuid = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
+    lease = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
 
-    return unless uuid
+    return unless lease
 
-    yield
-
-    release_lease(uuid)
+    yield lease
   end
 
-  def release_lease(uuid)
-    Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+  def lease_key
+    Geo::RepositoryBackfillService::LEASE_KEY_PREFIX
   end
 
-  def lease_key
-    'repository_backfill_service'
+  def lease_timeout
+    Geo::RepositoryBackfillService::LEASE_TIMEOUT
   end
 
   def node_enabled?

app/workers/geo_single_repository_backfill_worker.rb

+class GeoSingleRepositoryBackfillWorker
+  include Sidekiq::Worker
+  include ::GeoDynamicBackoff
+  include GeoQueue
+
+  def perform(project_id, lease)
+    Geo::RepositoryBackfillService.new(project_id, lease).execute
+  end
+end