Improves handling of stuck projects

app/models/project.rb

@@ -360,7 +360,7 @@ class Project < ActiveRecord::Base
     end
 
     event :import_fail do
-      transition [:scheduled, :started] => :failed
+      transition [:scheduled, :started, :failed, :finished] => :failed
     end
 
     state :scheduled
@@ -532,6 +532,7 @@ class Project < ActiveRecord::Base
       end
 
     if job_id
+      update(import_jid: job_id)
       Rails.logger.info "Import job started for #{full_path} with job ID #{job_id}"
     else
       Rails.logger.error "Import job failed to start for #{full_path}"
@@ -543,6 +544,7 @@ class Project < ActiveRecord::Base
       ProjectCacheWorker.perform_async(self.id)
     end
 
+    update(import_error: nil)
     remove_import_data
   end
 

app/workers/repository_fork_worker.rb

@@ -5,12 +5,16 @@ class RepositoryForkWorker
   include Gitlab::ShellAdapter
   include DedicatedSidekiqQueue
 
+  sidekiq_options status_expiration: StuckImportJobsWorker::FORK_EXPIRATION
+
   def perform(project_id, forked_from_repository_storage_path, source_path, target_path)
     Gitlab::Metrics.add_event(:fork_repository,
                               source_path: source_path,
                               target_path: target_path)
 
     project = Project.find(project_id)
+
+    raise ForkError, "Project was in inconsistent state: #{project.import_status}" unless project.import_scheduled?
     project.import_start
 
     result = gitlab_shell.fork_repository(forked_from_repository_storage_path, source_path,

app/workers/repository_import_worker.rb

@@ -12,14 +12,13 @@ class RepositoryImportWorker
     @project = Project.find(project_id)
     @current_user = @project.creator
 
+    raise ImportError, "Project was in inconsistent state: #{project.import_status}" unless project.import_scheduled?
     project.import_start
 
     Gitlab::Metrics.add_event(:import_repository,
                               import_url: @project.import_url,
                               path: @project.full_path)
 
-    project.update_columns(import_jid: self.jid, import_error: nil)
-
     result = Projects::ImportService.new(project, current_user).execute
     raise ImportError, result[:message] if result[:status] == :error
 

app/workers/repository_update_mirror_worker.rb

 class RepositoryUpdateMirrorWorker
   UpdateError = Class.new(StandardError)
-  UpdateAlreadyInProgressError = Class.new(StandardError)
 
   include Sidekiq::Worker
   include Gitlab::ShellAdapter
   include DedicatedSidekiqQueue
 
   # Retry not neccessary. It will try again at the next update interval.
-  sidekiq_options retry: false
+  sidekiq_options retry: false, status_expiration: StuckImportJobsWorker::MIRROR_EXPIRATION
 
   attr_accessor :project, :repository, :current_user
 
   def perform(project_id)
     project = Project.find(project_id)
 
-    raise UpdateAlreadyInProgressError if project.import_started?
+    raise UpdateError, "Project was in inconsistent state: #{project.import_status}" unless project.import_scheduled?
     start_mirror(project)
 
     @current_user = project.mirror_user || project.creator
@@ -23,8 +22,6 @@ class RepositoryUpdateMirrorWorker
     raise UpdateError, result[:message] if result[:status] == :error
 
     finish_mirror(project)
-  rescue UpdateAlreadyInProgressError
-    raise
   rescue UpdateError => ex
     fail_mirror(project, ex.message)
     raise

app/workers/stuck_import_jobs_worker.rb

@@ -3,6 +3,8 @@ class StuckImportJobsWorker
   include CronjobQueue
 
   IMPORT_EXPIRATION = 15.hours.to_i
+  FORK_EXPIRATION = 15.hours.to_i
+  MIRROR_EXPIRATION = 20.minutes.to_i
 
   def perform
     stuck_projects.find_in_batches(batch_size: 500) do |group|
@@ -12,9 +14,11 @@ class StuckImportJobsWorker
       completed_jids = Gitlab::SidekiqStatus.completed_jids(jids)
 
       if completed_jids.any?
-        completed_ids = group.select { |project| completed_jids.include?(project.import_jid) }.map(&:id)
+        group.each do |project|
+          project.mark_import_as_failed(error_message) if completed_jids.include(project.import_jid)
+        end
 
-        fail_batch!(completed_jids, completed_ids)
+        Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.join(', ')}")
       end
     end
   end
@@ -22,13 +26,7 @@ class StuckImportJobsWorker
   private
 
   def stuck_projects
-    Project.select('id, import_jid').with_import_status(:started).where.not(import_jid: nil)
-  end
-
-  def fail_batch!(completed_jids, completed_ids)
-    Project.where(id: completed_ids).update_all(import_status: 'failed', import_error: error_message)
-
-    Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.join(', ')}")
+    Project.with_import_status(:started).where.not(import_jid: nil)
   end
 
   def error_message

ee/app/models/ee/project.rb

@@ -216,7 +216,15 @@ module EE
         ::Gitlab::Mirror.increment_metric(:mirrors_scheduled, 'Mirrors scheduled count')
         Rails.logger.info("Mirror update for #{full_path} was scheduled.")
 
-        RepositoryUpdateMirrorWorker.perform_async(self.id)
+        job_id = RepositoryUpdateMirrorWorker.perform_async(self.id)
+
+        if job_id
+          update(import_jid: job_id)
+
+          Rails.logger.info("Mirror job created for #{full_path} with job ID #{job_id}.")
+        else
+          Rails.logger.error("Mirror job failed to create for #{full_path}.")
+        end
       end
     end