Handle projects that needs re-verification

ee/app/workers/geo/repository_verification/primary/shard_worker.rb

@@ -52,7 +52,7 @@ module Geo
           remaining_capacity = db_retrieve_batch_size - resources.size
           return resources if remaining_capacity.zero?
 
-          resources + find_failed_project_ids(batch_size: remaining_capacity)
+          resources + find_project_ids_to_reverify(batch_size: remaining_capacity)
         end
 
         # rubocop: disable CodeReuse/ActiveRecord
@@ -67,6 +67,13 @@ module Geo
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
+        def find_project_ids_to_reverify(batch_size:)
+          failed_project_ids = find_failed_project_ids(batch_size: batch_size)
+          reverifiable_project_ids = find_reverifiable_projects_ids(batch_size: batch_size)
+
+          take_batch(failed_project_ids, reverifiable_project_ids, batch_size: batch_size)
+        end
+
         def find_failed_project_ids(batch_size:)
           repositories_ids = find_failed_repositories_ids(batch_size: batch_size)
           wiki_ids = find_failed_wiki_ids(batch_size: batch_size)
@@ -85,6 +92,24 @@ module Geo
           finder.find_failed_wikis(batch_size: batch_size).pluck(:id)
         end
         # rubocop: enable CodeReuse/ActiveRecord
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def find_reverifiable_projects_ids(batch_size:)
+          return [] unless reverification_enabled?
+
+          jitter   = (minimum_reverification_interval.seconds * rand(15)) / 100
+          interval = minimum_reverification_interval.ago + jitter.seconds
+
+          repository_ids = finder.find_reverifiable_repositories(interval: interval, batch_size: batch_size).pluck(:id)
+          wiki_ids = finder.find_reverifiable_wikis(interval: interval, batch_size: batch_size).pluck(:id)
+
+          take_batch(repository_ids, wiki_ids, batch_size: batch_size)
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        def minimum_reverification_interval
+          ::Gitlab::Geo.current_node.minimum_reverification_interval.days
+        end
       end
     end
   end

ee/spec/factories/geo_nodes.rb

@@ -12,6 +12,8 @@ FactoryBot.define do
 
     trait :primary do
       primary true
+      minimum_reverification_interval 7
+
       url do
         uri = URI.parse("http://#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.relative_url_root}")
         uri.port = Gitlab.config.gitlab.port

ee/spec/workers/geo/repository_verification/primary/shard_worker_spec.rb

@@ -91,6 +91,28 @@ describe Geo::RepositoryVerification::Primary::ShardWorker, :postgresql, :clean_
       subject.perform(shard_name)
     end
 
+    it 'performs Geo::RepositoryVerification::Primary::SingleWorker for projects where repository should be reverified' do
+      project_to_be_reverified = create(:project)
+
+      create(:repository_state, :repository_verified, :wiki_verified,
+        project: project_to_be_reverified, last_repository_verification_ran_at: 10.days.ago)
+
+      expect(primary_singleworker).to receive(:perform_async).with(project_to_be_reverified.id)
+
+      subject.perform(shard_name)
+    end
+
+    it 'performs Geo::RepositoryVerification::Primary::SingleWorker for projects where wiki should be reverified' do
+      project_to_be_reverified = create(:project)
+
+      create(:repository_state, :repository_verified, :wiki_verified,
+        project: project_to_be_reverified, last_wiki_verification_ran_at: 10.days.ago)
+
+      expect(primary_singleworker).to receive(:perform_async).with(project_to_be_reverified.id)
+
+      subject.perform(shard_name)
+    end
+
     it 'does not perform Geo::RepositoryVerification::Primary::SingleWorker when shard becomes unhealthy' do
       create(:project)
 
@@ -172,6 +194,8 @@ describe Geo::RepositoryVerification::Primary::ShardWorker, :postgresql, :clean_
       let(:project_wiki_unverified) { create(:repository_state).project }
       let(:project_repo_failed_wiki_verified) { create(:repository_state, :repository_failed, :wiki_verified).project }
       let(:project_repo_verified_wiki_failed) { create(:repository_state, :repository_verified, :wiki_failed).project }
+      let(:project_repo_reverify) { create(:repository_state, :repository_verified, :wiki_verified, last_repository_verification_ran_at: 10.days.ago).project }
+      let(:project_wiki_reverify) { create(:repository_state, :repository_verified, :wiki_verified, last_wiki_verification_ran_at: 10.days.ago).project }
 
       it 'handles multiple batches of projects needing verification' do
         expect(primary_singleworker).to receive(:perform_async).with(project_repo_unverified.id).once.and_call_original
@@ -182,7 +206,7 @@ describe Geo::RepositoryVerification::Primary::ShardWorker, :postgresql, :clean_
         end
       end
 
-      it 'handles multiple batches of projects needing verification, including failed repos' do
+      it 'handles multiple batches of projects needing verification' do
         expect(primary_singleworker).to receive(:perform_async).with(project_repo_unverified.id).once.and_call_original
         expect(primary_singleworker).to receive(:perform_async).with(project_wiki_unverified.id).once.and_call_original
         expect(primary_singleworker).to receive(:perform_async).with(project_repo_verified.id).once.and_call_original
@@ -190,8 +214,10 @@ describe Geo::RepositoryVerification::Primary::ShardWorker, :postgresql, :clean_
         expect(primary_singleworker).to receive(:perform_async).with(project_both_failed.id).once.and_call_original
         expect(primary_singleworker).to receive(:perform_async).with(project_repo_failed_wiki_verified.id).once.and_call_original
         expect(primary_singleworker).to receive(:perform_async).with(project_repo_verified_wiki_failed.id).once.and_call_original
+        expect(primary_singleworker).to receive(:perform_async).with(project_repo_reverify.id).once.and_call_original
+        expect(primary_singleworker).to receive(:perform_async).with(project_wiki_reverify.id).once.and_call_original
 
-        8.times do
+        10.times do
           Sidekiq::Testing.inline! { subject.perform(shard_name) }
         end
       end