Merge branch 'fix/geo-failed-repo-count' into 'master'

Fix in-progress repository syncs counting as failed

Closes #4053

See merge request gitlab-org/gitlab-ee!3424

app/models/geo/project_registry.rb

@@ -6,11 +6,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
   scope :dirty, -> { where(arel_table[:resync_repository].eq(true).or(arel_table[:resync_wiki].eq(true))) }
 
   def self.failed
-    repository_sync_failed = arel_table[:last_repository_synced_at].not_eq(nil)
-      .and(arel_table[:last_repository_successful_sync_at].eq(nil))
-
-    wiki_sync_failed = arel_table[:last_wiki_synced_at].not_eq(nil)
-      .and(arel_table[:last_wiki_successful_sync_at].eq(nil))
+    repository_sync_failed = arel_table[:repository_retry_count].gt(0)
+    wiki_sync_failed = arel_table[:wiki_retry_count].gt(0)
 
     where(repository_sync_failed.or(wiki_sync_failed))
   end

app/services/geo/repository_sync_service.rb

@@ -30,10 +30,12 @@ module Geo
            Gitlab::Git::RepositoryMirroring::RemoteError,
            Geo::EmptyCloneUrlPrefixError => e
       log_error('Error syncing repository', e)
+      registry.increment!(:repository_retry_count)
     rescue Gitlab::Git::Repository::NoRepository => e
       log_error('Invalid repository', e)
       log_info('Setting force_to_redownload flag')
-      registry.update(force_to_redownload_repository: true)
+      registry.update(force_to_redownload_repository: true,
+                      repository_retry_count: retry_count + 1)
       log_info('Expiring caches')
       project.repository.after_create
     ensure
@@ -52,5 +54,9 @@ module Geo
     def repository
       project.repository
     end
+
+    def retry_count
+      registry.public_send("#{type}_retry_count") || -1 # rubocop:disable GitlabSecurity/PublicSend
+    end
   end
 end

app/services/geo/wiki_sync_service.rb

@@ -33,9 +33,11 @@ module Geo
            ProjectWiki::CouldNotCreateWikiError,
            Geo::EmptyCloneUrlPrefixError => e
       log_error('Error syncing wiki repository', e)
+      registry.increment!(:wiki_retry_count)
     rescue Gitlab::Git::Repository::NoRepository => e
       log_error('Invalid wiki', e)
-      registry.update(force_to_redownload_wiki: true)
+      registry.update(force_to_redownload_wiki: true,
+                      repository_retry_count: retry_count + 1)
     ensure
       clean_up_temporary_repository if redownload
     end
@@ -47,5 +49,9 @@ module Geo
     def repository
       project.wiki.repository
     end
+
+    def retry_count
+      registry.public_send("#{type}_retry_count") || -1 # rubocop:disable GitlabSecurity/PublicSend
+    end
   end
 end

changelogs/unreleased-ee/fix-geo-failed-repo-count.yml

+---
+title: Fix in-progress repository syncs counting as failed
+merge_request: 3424
+author:
+type: fixed

spec/factories/geo/project_registry.rb

@@ -39,6 +39,7 @@ FactoryGirl.define do
       last_wiki_successful_sync_at nil
       resync_repository true
       resync_wiki true
+      repository_retry_count 1
     end
 
     trait :repository_sync_failed do
@@ -48,6 +49,12 @@ FactoryGirl.define do
       last_wiki_successful_sync_at  { 5.days.ago }
       resync_repository true
       resync_wiki false
+      repository_retry_count 1
+    end
+
+    trait :repository_syncing do
+      repository_sync_failed
+      repository_retry_count 0
     end
 
     trait :wiki_sync_failed do
@@ -57,6 +64,12 @@ FactoryGirl.define do
       last_wiki_successful_sync_at nil
       resync_repository false
       resync_wiki true
+      wiki_retry_count 2
+    end
+
+    trait :wiki_syncing do
+      wiki_sync_failed
+      wiki_retry_count 0
     end
   end
 end

spec/models/geo/project_registry_spec.rb

@@ -21,6 +21,9 @@ describe Geo::ProjectRegistry do
     it 'returns projects where last attempt to sync failed' do
       create(:geo_project_registry, :synced)
       create(:geo_project_registry, :synced, :dirty)
+      create(:geo_project_registry, :repository_syncing)
+      create(:geo_project_registry, :wiki_syncing)
+
       repository_sync_failed = create(:geo_project_registry, :repository_sync_failed)
       wiki_sync_failed = create(:geo_project_registry, :wiki_sync_failed)
 

spec/models/geo_node_status_spec.rb

@@ -200,6 +200,8 @@ describe GeoNodeStatus, :geo do
     before do
       create(:geo_project_registry, :sync_failed, project: project_1)
       create(:geo_project_registry, :sync_failed, project: project_3)
+      create(:geo_project_registry, :repository_syncing, project: project_4)
+      create(:geo_project_registry, :wiki_syncing)
     end
 
     it 'returns the right number of failed repos with no group restrictions' do