Before updating all mirrors, clear out any that appear to be stuck

Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/247

app/workers/update_all_mirrors_worker.rb

@@ -2,6 +2,19 @@ class UpdateAllMirrorsWorker
   include Sidekiq::Worker
 
   def perform
+    fail_stuck_mirrors!
+
     Project.mirror.each(&:update_mirror)
   end
+
+  def fail_stuck_mirrors!
+    stuck = Project.mirror.
+      with_import_status(:started).
+      where('mirror_last_update_at < ?', 1.day.ago)
+
+    stuck.find_each(batch_size: 50) do |project|
+      project.import_fail
+      project.update_attribute(:import_error, 'The mirror update took too long to complete.')
+    end
+  end
 end

spec/factories/projects.rb

@@ -61,6 +61,23 @@ FactoryGirl.define do
     trait :private do
       visibility_level Gitlab::VisibilityLevel::PRIVATE
     end
+
+    trait :import_started do
+      import_url FFaker::Internet.uri('http')
+      import_status :started
+    end
+
+    trait :import_finished do
+      import_started
+      import_status :finished
+    end
+
+    trait :mirror do
+      import_started
+
+      mirror true
+      mirror_user_id { creator_id }
+    end
   end
 
   # Project with empty repository

spec/workers/update_all_mirrors_worker_spec.rb

+require 'rails_helper'
+
+describe UpdateAllMirrorsWorker do
+  describe '#perform' do
+    it 'fails stuck mirrors' do
+      worker = described_class.new
+
+      expect(worker).to receive(:fail_stuck_mirrors!)
+
+      worker.perform
+    end
+
+    it 'updates all mirrored Projects' do
+      create(:empty_project, :mirror)
+      create(:empty_project)
+
+      expect_any_instance_of(Project).to receive(:update_mirror).once
+
+      described_class.new.perform
+    end
+  end
+
+  describe '#fail_stuck_mirrors!' do
+    it 'ignores records that are not mirrors' do
+      create(:empty_project, :import_started, mirror_last_update_at: 3.days.ago)
+
+      expect_any_instance_of(Project).not_to receive(:import_fail)
+
+      perform
+    end
+
+    it 'ignores records without in-progress import' do
+      create(:empty_project, :mirror, :import_finished, mirror_last_update_at: 3.days.ago)
+
+      expect_any_instance_of(Project).not_to receive(:import_fail)
+
+      perform
+    end
+
+    it 'ignores records with recently updated mirrors' do
+      create(:empty_project, :mirror, mirror_last_update_at: Time.now)
+
+      expect_any_instance_of(Project).not_to receive(:import_fail)
+
+      perform
+    end
+
+    it 'transitions stuck mirrors to a failed state' do
+      project = create(:empty_project, :mirror, mirror_last_update_at: 3.days.ago)
+
+      perform
+      project.reload
+
+      expect(project).to be_import_failed
+    end
+
+    it 'updates the import_error message' do
+      project = create(:empty_project, :mirror, mirror_last_update_at: 3.days.ago)
+
+      perform
+      project.reload
+
+      expect(project.import_error).to eq 'The mirror update took too long to complete.'
+    end
+
+    def perform
+      described_class.new.fail_stuck_mirrors!
+    end
+  end
+end