Merge branch 'tc-fix-retrying-same-repos' into 'master'

Geo::RepositySyncWorker should attempt to sync all projects

Closes #3259

See merge request !2796

app/workers/geo/repository_sync_worker.rb

@@ -24,7 +24,7 @@ module Geo
 
     def find_project_ids_not_synced
       current_node.projects
-                  .where.not(id: Geo::ProjectRegistry.synced.pluck(:project_id))
+                  .where.not(id: Geo::ProjectRegistry.pluck(:project_id))
                   .order(last_repository_updated_at: :desc)
                   .limit(db_retrieve_batch_size)
                   .pluck(:id)

changelogs/unreleased-ee/tc-fix-retrying-same-repos.yml

+---
+title: Fix Geo::RepositorySyncWorker so attempts to sync all projects if some are failing
+merge_request: 2796
+author:
+type: fixed

spec/factories/projects.rb

@@ -190,6 +190,10 @@ FactoryGirl.define do
       end
     end
 
+    trait :random_last_repository_updated_at do
+      last_repository_updated_at { rand(1.year).seconds.ago }
+    end
+
     trait(:wiki_enabled)            { wiki_access_level ProjectFeature::ENABLED }
     trait(:wiki_disabled)           { wiki_access_level ProjectFeature::DISABLED }
     trait(:wiki_private)            { wiki_access_level ProjectFeature::PRIVATE }

spec/workers/geo/repository_sync_worker_spec.rb

@@ -90,5 +90,31 @@ describe Geo::RepositorySyncWorker do
         subject.perform
       end
     end
+
+    context 'all repositories fail' do
+      let!(:project_list) { create_list(:project, 4, :random_last_repository_updated_at) }
+
+      before do
+        allow_any_instance_of(described_class).to receive(:db_retrieve_batch_size).and_return(2) # Must be >1 because of the Geo::BaseSchedulerWorker#interleave
+        allow_any_instance_of(described_class).to receive(:max_capacity).and_return(3) # Must be more than db_retrieve_batch_size
+        allow_any_instance_of(Project).to receive(:ensure_repository).and_raise(Gitlab::Shell::Error.new('foo'))
+        allow_any_instance_of(Geo::ProjectSyncWorker).to receive(:sync_wiki?).and_return(false)
+        allow_any_instance_of(Geo::RepositorySyncService).to receive(:expire_repository_caches)
+      end
+
+      it 'tries to sync every project' do
+        project_list.each do |project|
+          expect(Geo::ProjectSyncWorker)
+            .to receive(:perform_in)
+              .with(anything, project.id, anything)
+              .at_least(:once)
+              .and_call_original
+        end
+
+        3.times do
+          Sidekiq::Testing.inline! { subject.perform }
+        end
+      end
+    end
   end
 end