Geo::RepositySyncWorker should attempt to sync all projects

If some projects fail to sync, Geo::RepositySyncWorker should not take
the same set of projects over and over again. But try to sync all of
them.

Closes gitlab-org/gitlab-ee#3259

app/workers/geo/repository_sync_worker.rb

@@ -24,7 +24,7 @@ module Geo
 
     def find_project_ids_not_synced
       current_node.projects
-                  .where.not(id: Geo::ProjectRegistry.synced.pluck(:project_id))
+                  .where.not(id: Geo::ProjectRegistry.pluck(:project_id))
                   .order(last_repository_updated_at: :desc)
                   .limit(db_retrieve_batch_size)
                   .pluck(:id)

changelogs/unreleased-ee/tc-fix-retrying-same-repos.yml

+---
+title: Fix Geo::RepositorySyncWorker so attempts to sync all projects if some are failing
+merge_request: 2796
+author:
+type: fixed

spec/factories/projects.rb

@@ -190,6 +190,10 @@ FactoryGirl.define do
       end
     end
 
+    trait :random_last_repository_updated_at do
+      last_repository_updated_at { rand(1.year).seconds.ago }
+    end
+
     trait(:wiki_enabled)            { wiki_access_level ProjectFeature::ENABLED }
     trait(:wiki_disabled)           { wiki_access_level ProjectFeature::DISABLED }
     trait(:wiki_private)            { wiki_access_level ProjectFeature::PRIVATE }

spec/workers/geo/repository_sync_worker_spec.rb

@@ -90,5 +90,29 @@ describe Geo::RepositorySyncWorker do
         subject.perform
       end
     end
+
+    context 'all repositories fail' do
+      before do
+        allow_any_instance_of(described_class).to receive(:db_retrieve_batch_size).and_return(4)
+        allow_any_instance_of(described_class).to receive(:max_capacity).and_return(5)
+        allow_any_instance_of(Project).to receive(:ensure_repository).and_raise(Gitlab::Shell::Error.new('foo'))
+        allow_any_instance_of(Geo::ProjectSyncWorker).to receive(:sync_wiki?).and_return(false)
+        allow_any_instance_of(Geo::RepositorySyncService).to receive(:expire_repository_caches)
+        allow_any_instance_of(Geo::RepositorySyncService).to receive(:try_obtain_lease) { |&arg| arg.call }
+
+        create_list(:project, 20, :random_last_repository_updated_at)
+      end
+
+      it 'attempts to sync them all' do
+        Sidekiq::Testing.inline! do
+          10.times do
+            subject.perform
+            break if Geo::ProjectRegistry.count == Project.count
+          end
+        end
+
+        expect(Geo::ProjectRegistry.count).to eq(Project.count)
+      end
+    end
   end
 end