Merge branch '6142-geo-sync-disabled-wikis' into 'master'

Resolve "Geo: sync disabled wikis"

Closes #7161 and #6142

See merge request gitlab-org/gitlab-ee!6420

db/schema.rb

@@ -1176,6 +1176,7 @@ ActiveRecord::Schema.define(version: 20180807153545) do
     t.binary "storage_configuration_digest"
     t.integer "repositories_retrying_verification_count"
     t.integer "wikis_retrying_verification_count"
+    t.integer "projects_count"
   end
 
   add_index "geo_node_statuses", ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true, using: :btree

doc/api/geo_nodes.md

@@ -178,6 +178,7 @@ Example response:
     "job_artifacts_synced_missing_on_primary_count": 0,
     "job_artifacts_synced_in_percentage": "0.00%",
     "repositories_count": 41,
+    "projects_count": 41,
     "repositories_failed_count": nil,
     "repositories_synced_count": nil,
     "repositories_synced_in_percentage": "0.00%",
@@ -242,6 +243,7 @@ Example response:
     "job_artifacts_synced_missing_on_primary_count": 0,
     "job_artifacts_synced_in_percentage": "50.00%",
     "repositories_count": 41,
+    "projects_count": 41,
     "repositories_failed_count": 1,
     "repositories_synced_count": 40,
     "repositories_synced_in_percentage": "97.56%",
@@ -283,6 +285,8 @@ Example response:
 ]
 ```
 
+Note: fields `wikis_count` and `repositories_count` are deprecated and will be deleted soon. Please use `projects_count` instead.
+
 ## Retrieve status about a specific Geo node
 
 ```
@@ -319,6 +323,7 @@ Example response:
   "job_artifacts_synced_missing_on_primary_count": 0,
   "job_artifacts_synced_in_percentage": "50.00%",
   "repositories_count": 41,
+  "projects_count": 41,
   "repositories_failed_count": 1,
   "repositories_synced_count": 40,
   "repositories_synced_in_percentage": "97.56%",
@@ -340,7 +345,9 @@ Example response:
 }
 ```
 
-Please note that the `health_status` parameter can only be in an "Healthy" or "Unhealthy" state, while the `health` parameter can be empty, "Healthy", or contain the actual error message.
+Note: The `health_status` parameter can only be in an "Healthy" or "Unhealthy" state, while the `health` parameter can be empty, "Healthy", or contain the actual error message.
+
+Note: Fields `wikis_count` and `repositories_count` are deprecated and will be deleted soon. Please use `projects_count` instead.
 
 ## Retrieve project sync or verification failures that occurred on the current node
 

ee/app/finders/geo/project_registry_finder.rb

 module Geo
   class ProjectRegistryFinder < RegistryFinder
-    def count_repositories
+    def count_projects
       current_node.projects.count
     end
 
-    def count_wikis
-      current_node.projects.with_wiki_enabled.count
-    end
-
     def count_synced_repositories
       relation =
         if selective_sync?
@@ -194,7 +190,7 @@ module Geo
 
     # @return [ActiveRecord::Relation<Geo::ProjectRegistry>]
     def fdw_find_synced_wikis
-      Geo::ProjectRegistry.synced_wikis.where(fdw_enabled_wikis)
+      Geo::ProjectRegistry.synced_wikis
     end
 
     # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
@@ -222,7 +218,7 @@ module Geo
 
     # @return [ActiveRecord::Relation<Geo::ProjectRegistry>]
     def fdw_find_verified_wikis
-      Geo::ProjectRegistry.verified_wikis.where(fdw_enabled_wikis)
+      Geo::ProjectRegistry.verified_wikis
     end
 
     def fdw_inner_join_repository_state
@@ -232,20 +228,6 @@ module Geo
         .join_sources
     end
 
-    def fdw_enabled_wikis
-      project_id_matcher =
-        Geo::Fdw::ProjectFeature.arel_table[:project_id]
-          .eq(Geo::ProjectRegistry.arel_table[:project_id])
-
-      # Only read the IDs of projects with disabled wikis from the remote database
-      Geo::Fdw::ProjectFeature
-        .where(wiki_access_level: [::ProjectFeature::DISABLED])
-        .where(project_id_matcher)
-        .select('1')
-        .exists
-        .not
-    end
-
     #
     # Legacy accessors (non FDW)
     #
@@ -290,7 +272,7 @@ module Geo
     # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of synced projects
     def legacy_find_synced_wikis
       legacy_inner_join_registry_ids(
-        current_node.projects.with_wiki_enabled,
+        current_node.projects,
           Geo::ProjectRegistry.synced_wikis.pluck(:project_id),
           Project
       )
@@ -304,7 +286,7 @@ module Geo
     # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of verified wikis
     def legacy_find_verified_wikis
       legacy_inner_join_registry_ids(
-        current_node.projects.with_wiki_enabled,
+        current_node.projects,
           Geo::ProjectRegistry.verified_wikis.pluck(:project_id),
           Project
       )

ee/app/models/geo/project_registry.rb

@@ -165,7 +165,7 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
   end
 
   def wiki_sync_due?(scheduled_time)
-    project.wiki_enabled? && (never_synced_wiki? || wiki_sync_needed?(scheduled_time))
+    never_synced_wiki? || wiki_sync_needed?(scheduled_time)
   end
 
   # Returns whether repository is pending verification check

ee/app/models/geo_node_status.rb

@@ -154,8 +154,7 @@ class GeoNodeStatus < ActiveRecord::Base
     latest_event = Geo::EventLog.latest_event
     self.last_event_id = latest_event&.id
     self.last_event_date = latest_event&.created_at
-    self.repositories_count = projects_finder.count_repositories
-    self.wikis_count = projects_finder.count_wikis
+    self.projects_count = projects_finder.count_projects
     self.lfs_objects_count = lfs_objects_finder.count_syncable
     self.job_artifacts_count = job_artifacts_finder.count_syncable
     self.attachments_count = attachments_finder.count_syncable
@@ -163,6 +162,11 @@ class GeoNodeStatus < ActiveRecord::Base
     self.storage_shards = StorageShard.all
     self.storage_configuration_digest = StorageShard.build_digest
 
+    # Backward compatibility. These are deprecated and not used normally. Only needed for smooth update
+    # when secondary node is outdated yet
+    self.repositories_count = projects_finder.count_projects
+    self.wikis_count = projects_finder.count_projects
+
     self.version = Gitlab::VERSION
     self.revision = Gitlab.revision
 
@@ -305,7 +309,7 @@ class GeoNodeStatus < ActiveRecord::Base
   end
 
   def repositories_synced_in_percentage
-    calc_percentage(repositories_count, repositories_synced_count)
+    calc_percentage(projects_count, repositories_synced_count)
   end
 
   def wikis_synced_in_percentage
@@ -313,7 +317,7 @@ class GeoNodeStatus < ActiveRecord::Base
   end
 
   def repositories_checksummed_in_percentage
-    calc_percentage(repositories_count, repositories_checksummed_count)
+    calc_percentage(projects_count, repositories_checksummed_count)
   end
 
   def wikis_checksummed_in_percentage
@@ -321,7 +325,7 @@ class GeoNodeStatus < ActiveRecord::Base
   end
 
   def repositories_verified_in_percentage
-    calc_percentage(repositories_count, repositories_verified_count)
+    calc_percentage(projects_count, repositories_verified_count)
   end
 
   def wikis_verified_in_percentage
@@ -329,7 +333,7 @@ class GeoNodeStatus < ActiveRecord::Base
   end
 
   def repositories_checked_in_percentage
-    calc_percentage(repositories_count, repositories_checked_count)
+    calc_percentage(projects_count, repositories_checked_count)
   end
 
   def lfs_objects_synced_in_percentage
@@ -359,6 +363,15 @@ class GeoNodeStatus < ActiveRecord::Base
     public_send(key) # rubocop:disable GitlabSecurity/PublicSend
   end
 
+  # This method is for backward compatibility only.
+  # During the app update the secondary node can be outdated, and it does not provide
+  # us with the projects_count which is already expected by updated primary node.
+  # So we just fallback to old repositories_count
+  def projects_count
+    projects_count_attr = read_attribute(:projects_count)
+    projects_count_attr.nil? ? repositories_count : projects_count_attr
+  end
+
   private
 
   def primary_storage_digest

ee/app/workers/geo/project_sync_worker.rb

@@ -27,31 +27,8 @@ module Geo
         return
       end
 
-      mark_disabled_wiki_as_synced(registry)
-
       Geo::RepositorySyncService.new(project).execute if registry.repository_sync_due?(scheduled_time)
       Geo::WikiSyncService.new(project).execute if registry.wiki_sync_due?(scheduled_time)
     end
-
-    private
-
-    def mark_disabled_wiki_as_synced(registry)
-      return if registry.project.wiki_enabled?
-
-      registry.last_wiki_sync_failure = nil
-      registry.resync_wiki = false
-      registry.wiki_retry_count = nil
-      registry.wiki_retry_at = nil
-      registry.force_to_redownload_wiki = false
-
-      if registry.changed? || registry.last_wiki_synced_at.nil? || registry.last_wiki_successful_sync_at.nil?
-        now = DateTime.now
-        registry.last_wiki_synced_at = now
-        registry.last_wiki_successful_sync_at = now
-
-        success = registry.save
-        log_info("#{success ? 'Successfully marked' : 'Failed to mark'} disabled wiki as synced", registry_id: registry.id, project_id: registry.project_id)
-      end
-    end
   end
 end

ee/changelogs/unreleased/6142-geo-sync-disabled-wikis.yml

+---
+title: 'Geo: sync disabled wikis'
+merge_request: 6420
+author:
+type: fixed

ee/db/migrate/20180803041220_add_projects_count_to_geo_node_statuses.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddProjectsCountToGeoNodeStatuses < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :geo_node_statuses, :projects_count, :integer
+  end
+end

ee/lib/ee/api/entities.rb

@@ -315,14 +315,16 @@ module EE
           number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
         end
 
-        expose :repositories_count
+        expose :projects_count
+
+        expose :repositories_count # Deprecated
         expose :repositories_failed_count
         expose :repositories_synced_count
         expose :repositories_synced_in_percentage do |node|
           number_to_percentage(node.repositories_synced_in_percentage, precision: 2)
         end
 
-        expose :wikis_count
+        expose :wikis_count  # Deprecated
         expose :wikis_failed_count
         expose :wikis_synced_count
         expose :wikis_synced_in_percentage do |node|

ee/lib/tasks/geo.rake

@@ -249,13 +249,13 @@ namespace :geo do
 
     print 'Repositories: '.rjust(COLUMN_WIDTH)
     show_failed_value(current_node_status.repositories_failed_count)
-    print "#{current_node_status.repositories_synced_count}/#{current_node_status.repositories_count} "
+    print "#{current_node_status.repositories_synced_count}/#{current_node_status.projects_count} "
     puts using_percentage(current_node_status.repositories_synced_in_percentage)
 
     if Gitlab::Geo.repository_verification_enabled?
       print 'Verified Repositories: '.rjust(COLUMN_WIDTH)
       show_failed_value(current_node_status.repositories_verification_failed_count)
-      print "#{current_node_status.repositories_verified_count}/#{current_node_status.repositories_count} "
+      print "#{current_node_status.repositories_verified_count}/#{current_node_status.projects_count} "
       puts using_percentage(current_node_status.repositories_verified_in_percentage)
     end
 
@@ -284,7 +284,7 @@ namespace :geo do
     if Gitlab::CurrentSettings.repository_checks_enabled
       print 'Repositories Checked: '.rjust(COLUMN_WIDTH)
       show_failed_value(current_node_status.repositories_checked_failed_count)
-      print "#{current_node_status.repositories_checked_count}/#{current_node_status.repositories_count} "
+      print "#{current_node_status.repositories_checked_count}/#{current_node_status.projects_count} "
       puts using_percentage(current_node_status.repositories_checked_in_percentage)
     end
 

ee/spec/factories/geo_node_statuses.rb

@@ -18,6 +18,7 @@ FactoryBot.define do
       job_artifacts_synced_count 577
       job_artifacts_synced_missing_on_primary_count 91
       repositories_count 10
+      projects_count 10
       repositories_synced_count 5
       repositories_failed_count 0
       wikis_count 9

ee/spec/finders/geo/project_registry_finder_spec.rb

@@ -97,13 +97,6 @@ describe Geo::ProjectRegistryFinder, :geo do
         expect(subject.count_synced_wikis).to eq 1
       end
 
-      it 'does not count disabled wikis' do
-        create(:geo_project_registry, :synced, project: project_synced)
-        create(:geo_project_registry, :synced, project: create(:project, :wiki_disabled))
-
-        expect(subject.count_synced_wikis).to eq 1
-      end
-
       context 'with selective sync' do
         before do
           secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
@@ -248,13 +241,6 @@ describe Geo::ProjectRegistryFinder, :geo do
 
         expect(subject.count_verified_wikis).to eq 2
       end
-
-      it 'does not count disabled wikis' do
-        create(:geo_project_registry, :wiki_verified, project: project_wiki_verified)
-        create(:geo_project_registry, :wiki_verified, project: create(:project, :wiki_disabled))
-
-        expect(subject.count_verified_wikis).to eq 1
-      end
     end
 
     describe '#count_verification_failed_repositories' do

ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -20,6 +20,7 @@
     "job_artifacts_synced_missing_on_primary_count",
     "db_replication_lag_seconds",
     "repositories_count",
+    "projects_count",
     "repositories_failed_count",
     "repositories_synced_count",
     "wikis_count",
@@ -83,6 +84,7 @@
     "job_artifacts_synced_missing_on_primary_count": { "type": ["integer", "null"] },
     "job_artifacts_synced_in_percentage": { "type": "string" },
     "repositories_count": { "type": "integer" },
+    "projects_count": { "type": "integer" },
     "repositories_failed_count": { "type": ["integer", "null"] },
     "repository_verification_enabled": { "type": "boolean" },
     "repositories_synced_count": { "type": ["integer", "null"] },

ee/spec/lib/ee/api/entities/geo_node_status_spec.rb

@@ -85,7 +85,7 @@ describe EE::API::Entities::GeoNodeStatus, :postgresql do
 
   describe '#repositories_synced_in_percentage' do
     it 'formats as percentage' do
-      geo_node_status.assign_attributes(repositories_count: 10,
+      geo_node_status.assign_attributes(projects_count: 10,
                                         repositories_synced_count: 5,
                                         repositories_failed_count: 0)
 

ee/spec/models/geo/project_registry_spec.rb

@@ -207,17 +207,7 @@ describe Geo::ProjectRegistry do
         )
       end
 
-      context 'wiki enabled' do
-        it { expect(registry.wiki_sync_due?(Time.now)).to eq(expected) }
-      end
-
-      context 'wiki disabled' do
-        before do
-          project.update!(wiki_enabled: false)
-        end
-
-        it { expect(registry.wiki_sync_due?(Time.now)).to be_falsy }
-      end
+      it { expect(registry.wiki_sync_due?(Time.now)).to eq(expected) }
     end
   end
 

ee/spec/services/geo/metrics_update_service_spec.rb

@@ -16,6 +16,7 @@ describe Geo::MetricsUpdateService, :geo, :prometheus do
       status_message: nil,
       db_replication_lag_seconds: 0,
       repositories_count: 10,
+      projects_count: 10,
       repositories_synced_count: 1,
       repositories_failed_count: 2,
       wikis_count: 10,
@@ -56,6 +57,7 @@ describe Geo::MetricsUpdateService, :geo, :prometheus do
       status_message: nil,
       repositories_count: 10,
       wikis_count: 10,
+      projects_count: 10,
       lfs_objects_count: 100,
       job_artifacts_count: 100,
       attachments_count: 30,

ee/spec/workers/geo/project_sync_worker_spec.rb

@@ -121,68 +121,6 @@ RSpec.describe Geo::ProjectSyncWorker do
       end
     end
 
-    context 'wiki is not enabled for project' do
-      let!(:registry) { create(:geo_project_registry, resync_repository: true, resync_wiki: true, project: project) }
-
-      before do
-        project.update!(wiki_enabled: false)
-      end
-
-      it 'syncs the project repository' do
-        subject.perform(project.id, Time.now)
-
-        expect(repository_sync_service).to have_received(:execute)
-      end
-
-      it 'does not sync the project wiki' do
-        subject.perform(project.id, Time.now)
-
-        expect(wiki_sync_service).not_to have_received(:execute)
-      end
-
-      context 'when the wiki has failed to sync before' do
-        let!(:registry) { create(:geo_project_registry, :wiki_sync_failed, project: project) }
-
-        it 'marks the wiki as synced, to remove it from failed Geo wiki queries' do
-          subject.perform(project.id, Time.now)
-
-          expect(registry.reload.resync_wiki).to be_falsey
-          expect(registry.reload.last_wiki_sync_failure).to be_nil
-          expect(registry.reload.last_wiki_synced_at).to be_present
-          expect(registry.reload.last_wiki_successful_sync_at).to be_present
-          expect(registry.reload.wiki_retry_count).to be_nil
-          expect(registry.reload.wiki_retry_at).to be_nil
-          expect(registry.reload.force_to_redownload_wiki).to be_falsey
-        end
-
-        it 'logs that the wiki was marked as not needing a sync' do
-          expect(subject).to receive(:log_info).with("Successfully marked disabled wiki as synced", registry_id: registry.id, project_id: registry.project_id)
-
-          subject.perform(project.id, Time.now)
-        end
-      end
-
-      context 'when the wiki has never been synced before' do
-        it 'marks the wiki as synced, to remove it from out-of-sync Geo wiki queries' do
-          subject.perform(project.id, Time.now)
-
-          expect(registry.reload.resync_wiki).to be_falsey
-          expect(registry.reload.last_wiki_sync_failure).to be_nil
-          expect(registry.reload.last_wiki_synced_at).to be_present
-          expect(registry.reload.last_wiki_successful_sync_at).to be_present
-          expect(registry.reload.wiki_retry_count).to be_nil
-          expect(registry.reload.wiki_retry_at).to be_nil
-          expect(registry.reload.force_to_redownload_wiki).to be_falsey
-        end
-
-        it 'logs that the wiki was marked as not needing a sync' do
-          expect(subject).to receive(:log_info).with("Successfully marked disabled wiki as synced", registry_id: registry.id, project_id: registry.project_id)
-
-          subject.perform(project.id, Time.now)
-        end
-      end
-    end
-
     context 'when project repository was synced after the time the job was scheduled in' do
       it 'does not perform Geo::RepositorySyncService for the given project' do
         create(:geo_project_registry, :synced, :repository_dirty, project: project, last_repository_synced_at: Time.now)

qa/qa/specs/features/ee/api/geo_nodes_spec.rb

@@ -37,6 +37,7 @@ module QA
                                db_replication_lag_seconds: :integer_or_null,
                                lfs_objects_count: :integer,
                                job_artifacts_count: :integer,
+                               projects_count: :integer,
                                repositories_count: :integer,
                                wikis_count: :integer,
                                replication_slots_count: :integer_or_null,