Merge branch 'da-fix-geo-node-status-calculates-numbers-inefficiently' into 'master'

Resolve "GeoNodeStatus calculates numbers inefficiently"

Closes #3699

See merge request gitlab-org/gitlab-ee!3595

app/models/geo_node.rb

@@ -120,104 +120,22 @@ class GeoNode < ActiveRecord::Base
     end
   end
 
-  def projects_include?(project_id)
-    return true if restricted_project_ids.nil?
-
-    restricted_project_ids.include?(project_id)
-  end
-
-  def restricted_project_ids
-    return unless namespaces.presence
-
-    relations = namespaces.map { |namespace| namespace.all_projects.select(:id) }
-
-    Project.unscoped
-       .from("(#{Gitlab::SQL::Union.new(relations).to_sql}) #{Project.table_name}")
-       .pluck(:id)
-  end
-
-  def lfs_objects
-    relation =
-      if restricted_project_ids
-        LfsObject.joins(:projects).where(projects: { id: restricted_project_ids })
-      else
-        LfsObject.all
-      end
-
-    relation.with_files_stored_locally
-  end
-
   def projects
-    if restricted_project_ids
-      Project.where(id: restricted_project_ids)
+    if selective_sync?
+      Project.where(namespace_id: Gitlab::GroupHierarchy.new(namespaces).base_and_descendants.select(:id))
     else
       Project.all
     end
   end
 
-  def project_registries
-    if restricted_project_ids
-      Geo::ProjectRegistry.where(project_id: restricted_project_ids)
-    else
-      Geo::ProjectRegistry.all
-    end
-  end
-
-  def filtered_project_registries(type = nil)
-    case type
-    when 'repository'
-      project_registries.failed_repos
-    when 'wiki'
-      project_registries.failed_wikis
-    else
-      project_registries.failed
-    end
-  end
-
-  def uploads
-    if restricted_project_ids
-      uploads_table   = Upload.arel_table
-      group_uploads   = uploads_table[:model_type].eq('Namespace').and(uploads_table[:model_id].in(Gitlab::Geo.current_node.namespace_ids))
-      project_uploads = uploads_table[:model_type].eq('Project').and(uploads_table[:model_id].in(restricted_project_ids))
-      other_uploads   = uploads_table[:model_type].not_in(%w[Namespace Project])
-
-      Upload.where(group_uploads.or(project_uploads).or(other_uploads))
-    else
-      Upload.all
-    end
-  end
-
-  def lfs_objects_synced_count
-    return unless secondary?
-
-    relation = Geo::FileRegistry.lfs_objects.synced
-
-    if restricted_project_ids
-      relation = relation.where(file_id: lfs_objects.pluck(:id))
-    end
-
-    relation.count
-  end
-
-  def lfs_objects_failed_count
-    return unless secondary?
-
-    Geo::FileRegistry.lfs_objects.failed.count
-  end
-
-  def attachments_synced_count
-    return unless secondary?
-
-    upload_ids = uploads.pluck(:id)
-    synced_ids = Geo::FileRegistry.attachments.synced.pluck(:file_id)
+  def projects_include?(project_id)
+    return true unless selective_sync?
 
-    (synced_ids & upload_ids).length
+    projects.where(id: project_id).exists?
   end
 
-  def attachments_failed_count
-    return unless secondary?
-
-    Geo::FileRegistry.attachments.failed.count
+  def selective_sync?
+    namespaces.exists?
   end
 
   def find_or_build_status

app/models/geo_node_status.rb

@@ -27,7 +27,6 @@ class GeoNodeStatus < ActiveRecord::Base
 
   def self.current_node_status
     current_node = Gitlab::Geo.current_node
-
     return unless current_node
 
     status = current_node.find_or_build_status
@@ -64,21 +63,21 @@ class GeoNodeStatus < ActiveRecord::Base
     latest_event = Geo::EventLog.latest_event
     self.last_event_id = latest_event&.id
     self.last_event_date = latest_event&.created_at
-    self.repositories_count = geo_node.projects.count
-    self.lfs_objects_count = geo_node.lfs_objects.count
-    self.attachments_count = geo_node.uploads.count
+    self.repositories_count = projects_finder.count_projects
+    self.lfs_objects_count = lfs_objects_finder.count_lfs_objects
+    self.attachments_count = attachments_finder.count_attachments
     self.last_successful_status_check_at = Time.now
 
     if Gitlab::Geo.secondary?
       self.db_replication_lag_seconds = Gitlab::Geo::HealthCheck.db_replication_lag_seconds
       self.cursor_last_event_id = Geo::EventLogState.last_processed&.event_id
       self.cursor_last_event_date = Geo::EventLog.find_by(id: self.cursor_last_event_id)&.created_at
-      self.repositories_synced_count = geo_node.project_registries.synced.count
-      self.repositories_failed_count = geo_node.project_registries.failed.count
-      self.lfs_objects_synced_count = geo_node.lfs_objects_synced_count
-      self.lfs_objects_failed_count = geo_node.lfs_objects_failed_count
-      self.attachments_synced_count = geo_node.attachments_synced_count
-      self.attachments_failed_count = geo_node.attachments_failed_count
+      self.repositories_synced_count = projects_finder.count_synced_project_registries
+      self.repositories_failed_count = projects_finder.count_failed_project_registries
+      self.lfs_objects_synced_count = lfs_objects_finder.count_synced_lfs_objects
+      self.lfs_objects_failed_count = lfs_objects_finder.count_failed_lfs_objects
+      self.attachments_synced_count = attachments_finder.count_synced_attachments
+      self.attachments_failed_count = attachments_finder.count_failed_attachments
     end
 
     self
@@ -136,6 +135,18 @@ class GeoNodeStatus < ActiveRecord::Base
 
   private
 
+  def attachments_finder
+    @attachments_finder ||= Geo::AttachmentRegistryFinder.new(current_node: geo_node)
+  end
+
+  def lfs_objects_finder
+    @lfs_objects_finder ||= Geo::LfsObjectRegistryFinder.new(current_node: geo_node)
+  end
+
+  def projects_finder
+    @projects_finder ||= Geo::ProjectRegistryFinder.new(current_node: geo_node)
+  end
+
   def sync_percentage(total, synced)
     return 0 if !total.present? || total.zero?
 

app/workers/geo/repositories_clean_up_worker.rb

@@ -12,11 +12,9 @@ module Geo
       # Prevent multiple Sidekiq workers from performing repositories clean up
       try_obtain_lease do
         geo_node = GeoNode.find(geo_node_id)
+        return unless geo_node.selective_sync?
 
-        restricted_project_ids = geo_node.restricted_project_ids
-        return unless restricted_project_ids
-
-        Project.where.not(id: restricted_project_ids).find_in_batches(batch_size: BATCH_SIZE) do |batch|
+        Project.where.not(id: geo_node.projects).find_in_batches(batch_size: BATCH_SIZE) do |batch|
           batch.each do |project|
             clean_up_repositories(project)
           end

changelogs/unreleased-ee/da-fix-geo-node-status-calculates-numbers-inefficiently.yml

+---
+title: Geo - Improve performance when calculating the node status
+merge_request: 3595
+author:
+type: performance

ee/app/finders/geo/attachment_registry_finder.rb

+module Geo
+  class AttachmentRegistryFinder < RegistryFinder
+    def count_attachments
+      uploads.count
+    end
+
+    def count_synced_attachments
+      find_synced_attachments.count
+    end
+
+    def count_failed_attachments
+      find_failed_attachments.count
+    end
+
+    def find_synced_attachments
+      relation =
+        if use_legacy_queries?
+          legacy_find_synced_attachments
+        else
+          fdw_find_synced_attachments
+        end
+
+      relation
+    end
+
+    def find_failed_attachments
+      relation =
+        if use_legacy_queries?
+          legacy_find_failed_attachments
+        else
+          fdw_find_failed_attachments
+        end
+
+      relation
+    end
+
+    def uploads
+      if selective_sync?
+        Upload.where(group_uploads.or(project_uploads).or(other_uploads))
+      else
+        Upload.all
+      end
+    end
+
+    private
+
+    def group_uploads
+      namespace_ids = Gitlab::GroupHierarchy.new(current_node.namespaces).base_and_descendants.select(:id)
+      arel_namespace_ids = Arel::Nodes::SqlLiteral.new(namespace_ids.to_sql)
+
+      upload_table[:model_type].eq('Namespace').and(upload_table[:model_id].in(arel_namespace_ids))
+    end
+
+    def project_uploads
+      project_ids = current_node.projects.select(:id)
+      arel_project_ids = Arel::Nodes::SqlLiteral.new(project_ids.to_sql)
+
+      upload_table[:model_type].eq('Project').and(upload_table[:model_id].in(arel_project_ids))
+    end
+
+    def other_uploads
+      upload_table[:model_type].not_in(%w[Namespace Project])
+    end
+
+    def upload_table
+      Upload.arel_table
+    end
+
+    #
+    # FDW accessors
+    #
+
+    def fdw_find_synced_attachments
+      fdw_find_attachments.merge(Geo::FileRegistry.synced)
+    end
+
+    def fdw_find_failed_attachments
+      fdw_find_attachments.merge(Geo::FileRegistry.failed)
+    end
+
+    def fdw_find_attachments
+      fdw_table = Geo::Fdw::Upload.table_name
+
+      Geo::Fdw::Upload.joins("INNER JOIN file_registry ON file_registry.file_id = #{fdw_table}.id")
+        .merge(Geo::FileRegistry.attachments)
+    end
+
+    #
+    # Legacy accessors (non FDW)
+    #
+
+    def legacy_find_synced_attachments
+      legacy_find_attachments(Geo::FileRegistry.attachments.synced.pluck(:file_id))
+    end
+
+    def legacy_find_failed_attachments
+      legacy_find_attachments(Geo::FileRegistry.attachments.failed.pluck(:file_id))
+    end
+
+    def legacy_find_attachments(registry_file_ids)
+      return Upload.none if registry_file_ids.empty?
+
+      joined_relation = uploads.joins(<<~SQL)
+        INNER JOIN
+        (VALUES #{registry_file_ids.map { |id| "(#{id})" }.join(',')})
+        file_registry(file_id)
+        ON #{Upload.table_name}.id = file_registry.file_id
+      SQL
+
+      joined_relation
+    end
+  end
+end

ee/app/finders/geo/file_registry_finder.rb

@@ -22,10 +22,10 @@ module Geo
       # Selective project replication adds a wrinkle to FDW queries, so
       # we fallback to the legacy version for now.
       relation =
-        if fdw?
-          fdw_find_nonreplicated_lfs_objects
-        else
+        if use_legacy_queries?
           legacy_find_nonreplicated_lfs_objects(except_registry_ids: except_registry_ids)
+        else
+          fdw_find_nonreplicated_lfs_objects
         end
 
       relation
@@ -48,10 +48,10 @@ module Geo
       # Selective project replication adds a wrinkle to FDW queries, so
       # we fallback to the legacy version for now.
       relation =
-        if fdw?
-          fdw_find_nonreplicated_uploads
-        else
+        if use_legacy_queries?
           legacy_find_nonreplicated_uploads(except_registry_ids: except_registry_ids)
+        else
+          fdw_find_nonreplicated_uploads
         end
 
       relation
@@ -95,7 +95,7 @@ module Geo
       registry_ids = legacy_pluck_registry_ids(file_types: :lfs, except_registry_ids: except_registry_ids)
 
       legacy_filter_registry_ids(
-        current_node.lfs_objects,
+        lfs_objects_finder.lfs_objects,
         registry_ids,
         LfsObject.table_name
       )
@@ -105,7 +105,7 @@ module Geo
       registry_ids = legacy_pluck_registry_ids(file_types: Geo::FileService::DEFAULT_OBJECT_TYPES, except_registry_ids: except_registry_ids)
 
       legacy_filter_registry_ids(
-        current_node.uploads,
+        attachments_finder.uploads,
         registry_ids,
         Upload.table_name
       )
@@ -132,5 +132,13 @@ module Geo
       ids = Geo::FileRegistry.where(file_type: file_types).pluck(:file_id)
       (ids + except_registry_ids).uniq
     end
+
+    def attachments_finder
+      @attachments_finder ||= AttachmentRegistryFinder.new(current_node: current_node)
+    end
+
+    def lfs_objects_finder
+      @lfs_objects_finder ||= LfsObjectRegistryFinder.new(current_node: current_node)
+    end
   end
 end

ee/app/finders/geo/lfs_object_registry_finder.rb

+module Geo
+  class LfsObjectRegistryFinder < RegistryFinder
+    def count_lfs_objects
+      lfs_objects.count
+    end
+
+    def count_synced_lfs_objects
+      relation =
+        if selective_sync?
+          legacy_find_synced_lfs_objects
+        else
+          find_synced_lfs_objects_registries
+        end
+
+      relation.count
+    end
+
+    def count_failed_lfs_objects
+      relation =
+        if selective_sync?
+          legacy_find_failed_lfs_objects
+        else
+          find_failed_lfs_objects_registries
+        end
+
+      relation.count
+    end
+
+    def lfs_objects
+      relation =
+        if selective_sync?
+          LfsObject.joins(:projects).where(projects: { id: current_node.projects })
+        else
+          LfsObject.all
+        end
+
+      relation.with_files_stored_locally
+    end
+
+    private
+
+    def find_synced_lfs_objects_registries
+      Geo::FileRegistry.lfs_objects.synced
+    end
+
+    def find_failed_lfs_objects_registries
+      Geo::FileRegistry.lfs_objects.failed
+    end
+
+    def legacy_find_synced_lfs_objects
+      legacy_find_lfs_objects(find_synced_lfs_objects_registries.pluck(:file_id))
+    end
+
+    def legacy_find_failed_lfs_objects
+      legacy_find_lfs_objects(find_failed_lfs_objects_registries.pluck(:file_id))
+    end
+
+    def legacy_find_lfs_objects(registry_file_ids)
+      return LfsObject.none if registry_file_ids.empty?
+
+      lfs_objects = LfsObject.joins(:projects)
+        .where(projects: { id: current_node.projects })
+        .with_files_stored_locally
+
+      joined_relation = lfs_objects.joins(<<~SQL)
+        INNER JOIN
+        (VALUES #{registry_file_ids.map { |id| "(#{id})" }.join(',')})
+        file_registry(file_id)
+        ON #{LfsObject.table_name}.id = file_registry.file_id
+      SQL
+
+      joined_relation
+    end
+  end
+end

ee/app/finders/geo/project_registry_finder.rb

 module Geo
   class ProjectRegistryFinder < RegistryFinder
-    def find_unsynced_projects(batch_size:)
+    def count_projects
+      current_node.projects.count
+    end
+
+    def count_synced_project_registries
       relation =
-        if fdw?
-          fdw_find_unsynced_projects
+        if selective_sync?
+          legacy_find_synced_project_registries
+        else
+          find_synced_project_registries
+        end
+
+      relation.count
+    end
+
+    def count_failed_project_registries
+      find_failed_project_registries.count
+    end
+
+    def find_failed_project_registries(type = nil)
+      relation =
+        if selective_sync?
+          legacy_find_filtered_failed_project_registries(type)
         else
+          find_filtered_failed_project_registries(type)
+        end
+
+      relation
+    end
+
+    def find_unsynced_projects(batch_size:)
+      relation =
+        if use_legacy_queries?
           legacy_find_unsynced_projects
+        else
+          fdw_find_unsynced_projects
         end
 
       relation.limit(batch_size)
@@ -13,10 +43,10 @@ module Geo
 
     def find_projects_updated_recently(batch_size:)
       relation =
-        if fdw?
-          fdw_find_projects_updated_recently
-        else
+        if use_legacy_queries?
           legacy_find_projects_updated_recently
+        else
+          fdw_find_projects_updated_recently
         end
 
       relation.limit(batch_size)
@@ -24,14 +54,29 @@ module Geo
 
     protected
 
-    def fdw_table
-      Geo::Fdw::Project.table_name
+    def find_synced_project_registries
+      Geo::ProjectRegistry.synced
+    end
+
+    def find_filtered_failed_project_registries(type = nil)
+      case type
+      when 'repository'
+        Geo::ProjectRegistry.failed_repos
+      when 'wiki'
+        Geo::ProjectRegistry.failed_wikis
+      else
+        Geo::ProjectRegistry.failed
+      end
     end
 
     #
     # FDW accessors
     #
 
+    def fdw_table
+      Geo::Fdw::Project.table_name
+    end
+
     # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
     def fdw_find_unsynced_projects
       Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
@@ -51,7 +96,7 @@ module Geo
 
     # @return [ActiveRecord::Relation<Project>] list of unsynced projects
     def legacy_find_unsynced_projects
-      registry_project_ids = current_node.project_registries.pluck(:project_id)
+      registry_project_ids = Geo::ProjectRegistry.pluck(:project_id)
       return current_node.projects if registry_project_ids.empty?
 
       joined_relation = current_node.projects.joins(<<~SQL)
@@ -66,14 +111,43 @@ module Geo
 
     # @return [ActiveRecord::Relation<Project>] list of projects updated recently
     def legacy_find_projects_updated_recently
-      registry_project_ids = current_node.project_registries.dirty.retry_due.pluck(:project_id)
+      legacy_find_projects(Geo::ProjectRegistry.dirty.retry_due.pluck(:project_id))
+    end
+
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of synced projects
+    def legacy_find_synced_project_registries
+      legacy_find_project_registries(Geo::ProjectRegistry.synced)
+    end
+
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of projects that sync has failed
+    def legacy_find_filtered_failed_project_registries(type = nil)
+      project_registries = find_filtered_failed_project_registries(type)
+      legacy_find_project_registries(project_registries)
+    end
+
+    # @return [ActiveRecord::Relation<Project>]
+    def legacy_find_projects(registry_project_ids)
       return Project.none if registry_project_ids.empty?
 
       joined_relation = current_node.projects.joins(<<~SQL)
         INNER JOIN
         (VALUES #{registry_project_ids.map { |id| "(#{id})" }.join(',')})
         project_registry(project_id)
-        ON projects.id = project_registry.project_id
+        ON #{Project.table_name}.id = project_registry.project_id
+      SQL
+
+      joined_relation
+    end
+
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>]
+    def legacy_find_project_registries(project_registries)
+      return Geo::ProjectRegistry.none if project_registries.empty?
+
+      joined_relation = project_registries.joins(<<~SQL)
+        INNER JOIN
+        (VALUES #{current_node.projects.pluck(:id).map { |id| "(#{id})" }.join(',')})
+        projects(id)
+        ON #{Geo::ProjectRegistry.table_name}.project_id = projects.id
       SQL
 
       joined_relation

ee/app/finders/geo/registry_finder.rb

@@ -2,20 +2,18 @@ module Geo
   class RegistryFinder
     attr_reader :current_node
 
+    delegate :selective_sync?, to: :current_node, allow_nil: true
+
     def initialize(current_node: nil)
       @current_node = current_node
     end
 
     protected
 
-    def fdw?
+    def use_legacy_queries?
       # Selective project replication adds a wrinkle to FDW
       # queries, so we fallback to the legacy version for now.
-      Gitlab::Geo.fdw? && !selective_sync
-    end
-
-    def selective_sync
-      current_node.restricted_project_ids
+      !Gitlab::Geo.fdw? || selective_sync?
     end
   end
 end

lib/api/geo_nodes.rb

@@ -49,7 +49,8 @@ module API
 
         not_found('Geo node not found') unless geo_node
 
-        project_registries = paginate(geo_node.filtered_project_registries(params[:type]))
+        finder = ::Geo::ProjectRegistryFinder.new(current_node: geo_node)
+        project_registries = paginate(finder.find_failed_project_registries(params[:type]))
 
         present project_registries, with: ::GeoProjectRegistryEntity
       end

spec/ee/spec/finders/geo/attachment_registry_finder_spec.rb

+require 'spec_helper'
+
+# Disable transactions via :truncate method because a foreign table
+# can't see changes inside a transaction of a different connection.
+describe Geo::AttachmentRegistryFinder, :geo, :truncate do
+  include ::EE::GeoHelpers
+
+  let(:secondary) { create(:geo_node) }
+
+  let(:synced_group) { create(:group) }
+  let(:synced_subgroup) { create(:group, parent: synced_group) }
+  let(:unsynced_group) { create(:group) }
+  let(:synced_project) { create(:project, group: synced_group) }
+  let(:unsynced_project) { create(:project, group: unsynced_group) }
+
+  let(:upload_1) { create(:upload, model: synced_group) }
+  let(:upload_2) { create(:upload, model: unsynced_group) }
+  let(:upload_3) { create(:upload, :issuable_upload, model: synced_project) }
+  let(:upload_4) { create(:upload, model: unsynced_project) }
+  let(:upload_5) { create(:upload, model: synced_project) }
+  let(:upload_6) { create(:upload, :personal_snippet) }
+  let(:upload_7) { create(:upload, model: synced_subgroup) }
+  let(:lfs_object) { create(:lfs_object) }
+
+  subject { described_class.new(current_node: secondary) }
+
+  before do
+    stub_current_geo_node(secondary)
+  end
+
+  context 'FDW' do
+    before do
+      skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo.fdw?
+    end
+
+    describe '#find_synced_attachments' do
+      it 'delegates to #fdw_find_synced_attachments' do
+        expect(subject).to receive(:fdw_find_synced_attachments).and_call_original
+
+        subject.find_synced_attachments
+      end
+
+      it 'returns synced avatars, attachment, personal snippets and files' do
+        create(:geo_file_registry, :avatar, file_id: upload_1.id)
+        create(:geo_file_registry, :avatar, file_id: upload_2.id)
+        create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
+        create(:geo_file_registry, :avatar, file_id: upload_6.id)
+        create(:geo_file_registry, :avatar, file_id: upload_7.id)
+        create(:geo_file_registry, :lfs, file_id: lfs_object.id)
+
+        synced_attachments = subject.find_synced_attachments
+
+        expect(synced_attachments.pluck(:id)).to match_array([upload_1.id, upload_2.id, upload_6.id, upload_7.id])
+      end
+
+      context 'with selective sync' do
+        it 'returns synced avatars, attachment, personal snippets and files' do
+          create(:geo_file_registry, :avatar, file_id: upload_1.id)
+          create(:geo_file_registry, :avatar, file_id: upload_2.id)
+          create(:geo_file_registry, :avatar, file_id: upload_3.id)
+          create(:geo_file_registry, :avatar, file_id: upload_4.id)
+          create(:geo_file_registry, :avatar, file_id: upload_5.id, success: false)
+          create(:geo_file_registry, :avatar, file_id: upload_6.id)
+          create(:geo_file_registry, :avatar, file_id: upload_7.id)
+          create(:geo_file_registry, :lfs, file_id: lfs_object.id)
+
+          secondary.update_attribute(:namespaces, [synced_group])
+
+          synced_attachments = subject.find_synced_attachments
+
+          expect(synced_attachments.pluck(:id)).to match_array([upload_1.id, upload_3.id, upload_6.id, upload_7.id])
+        end
+      end
+    end
+
+    describe '#find_failed_attachments' do
+      it 'delegates to #fdw_find_failed_attachments' do
+        expect(subject).to receive(:fdw_find_failed_attachments).and_call_original
+
+        subject.find_failed_attachments
+      end
+
+      it 'returns failed avatars, attachment, personal snippets and files' do
+        create(:geo_file_registry, :avatar, file_id: upload_1.id)
+        create(:geo_file_registry, :avatar, file_id: upload_2.id)
+        create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
+        create(:geo_file_registry, :avatar, file_id: upload_6.id, success: false)
+        create(:geo_file_registry, :avatar, file_id: upload_7.id, success: false)
+        create(:geo_file_registry, :lfs, file_id: lfs_object.id, success: false)
+
+        failed_attachments = subject.find_failed_attachments
+
+        expect(failed_attachments.pluck(:id)).to match_array([upload_3.id, upload_6.id, upload_7.id])
+      end
+
+      context 'with selective sync' do
+        it 'returns failed avatars, attachment, personal snippets and files' do
+          create(:geo_file_registry, :avatar, file_id: upload_1.id, success: false)
+          create(:geo_file_registry, :avatar, file_id: upload_2.id)
+          create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
+          create(:geo_file_registry, :avatar, file_id: upload_4.id)
+          create(:geo_file_registry, :avatar, file_id: upload_5.id)
+          create(:geo_file_registry, :avatar, file_id: upload_6.id, success: false)
+          create(:geo_file_registry, :avatar, file_id: upload_7.id, success: false)
+          create(:geo_file_registry, :lfs, file_id: lfs_object.id, success: false)
+
+          secondary.update_attribute(:namespaces, [synced_group])
+
+          failed_attachments = subject.find_failed_attachments
+
+          expect(failed_attachments.pluck(:id)).to match_array([upload_1.id, upload_3.id, upload_6.id, upload_7.id])
+        end
+      end
+    end
+  end
+
+  context 'Legacy' do
+    before do
+      allow(Gitlab::Geo).to receive(:fdw?).and_return(false)
+    end
+
+    describe '#find_synced_attachments' do
+      it 'delegates to #legacy_find_synced_attachments' do
+        expect(subject).to receive(:legacy_find_synced_attachments).and_call_original
+
+        subject.find_synced_attachments
+      end
+
+      it 'returns synced avatars, attachment, personal snippets and files' do
+        create(:geo_file_registry, :avatar, file_id: upload_1.id)
+        create(:geo_file_registry, :avatar, file_id: upload_2.id)
+        create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
+        create(:geo_file_registry, :avatar, file_id: upload_6.id)
+        create(:geo_file_registry, :avatar, file_id: upload_7.id)
+        create(:geo_file_registry, :lfs, file_id: lfs_object.id)
+
+        synced_attachments = subject.find_synced_attachments
+
+        expect(synced_attachments).to match_array([upload_1, upload_2, upload_6, upload_7])
+      end
+
+      context 'with selective sync' do
+        it 'returns synced avatars, attachment, personal snippets and files' do
+          create(:geo_file_registry, :avatar, file_id: upload_1.id)
+          create(:geo_file_registry, :avatar, file_id: upload_2.id)
+          create(:geo_file_registry, :avatar, file_id: upload_3.id)
+          create(:geo_file_registry, :avatar, file_id: upload_4.id)
+          create(:geo_file_registry, :avatar, file_id: upload_5.id, success: false)
+          create(:geo_file_registry, :avatar, file_id: upload_6.id)
+          create(:geo_file_registry, :avatar, file_id: upload_7.id)
+          create(:geo_file_registry, :lfs, file_id: lfs_object.id)
+
+          secondary.update_attribute(:namespaces, [synced_group])
+
+          synced_attachments = subject.find_synced_attachments
+
+          expect(synced_attachments).to match_array([upload_1, upload_3, upload_6, upload_7])
+        end
+      end
+    end
+
+    describe '#find_failed_attachments' do
+      it 'delegates to #legacy_find_failed_attachments' do
+        expect(subject).to receive(:legacy_find_failed_attachments).and_call_original
+
+        subject.find_failed_attachments
+      end
+
+      it 'returns failed avatars, attachment, personal snippets and files' do
+        create(:geo_file_registry, :avatar, file_id: upload_1.id)
+        create(:geo_file_registry, :avatar, file_id: upload_2.id)
+        create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
+        create(:geo_file_registry, :avatar, file_id: upload_6.id, success: false)
+        create(:geo_file_registry, :avatar, file_id: upload_7.id, success: false)
+        create(:geo_file_registry, :lfs, file_id: lfs_object.id, success: false)
+
+        failed_attachments = subject.find_failed_attachments
+
+        expect(failed_attachments).to match_array([upload_3, upload_6, upload_7])
+      end
+
+      context 'with selective sync' do
+        it 'returns failed avatars, attachment, personal snippets and files' do
+          create(:geo_file_registry, :avatar, file_id: upload_1.id, success: false)
+          create(:geo_file_registry, :avatar, file_id: upload_2.id)
+          create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
+          create(:geo_file_registry, :avatar, file_id: upload_4.id)
+          create(:geo_file_registry, :avatar, file_id: upload_5.id)
+          create(:geo_file_registry, :avatar, file_id: upload_6.id, success: false)
+          create(:geo_file_registry, :avatar, file_id: upload_7.id, success: false)
+          create(:geo_file_registry, :lfs, file_id: lfs_object.id, success: false)
+
+          secondary.update_attribute(:namespaces, [synced_group])
+
+          failed_attachments = subject.find_failed_attachments
+
+          expect(failed_attachments).to match_array([upload_1, upload_3, upload_6, upload_7])
+        end
+      end
+    end
+  end
+end

spec/ee/spec/finders/geo/lfs_object_registry_finder_spec.rb

+require 'spec_helper'
+
+describe Geo::LfsObjectRegistryFinder, :geo do
+  include ::EE::GeoHelpers
+
+  let(:secondary) { create(:geo_node) }
+  let(:synced_group) { create(:group) }
+  let(:synced_project) { create(:project, group: synced_group) }
+  let(:unsynced_project) { create(:project) }
+  let(:lfs_object_1) { create(:lfs_object) }
+  let(:lfs_object_2) { create(:lfs_object) }
+  let(:lfs_object_3) { create(:lfs_object) }
+
+  subject { described_class.new(current_node: secondary) }
+
+  before do
+    stub_current_geo_node(secondary)
+  end
+
+  describe '#count_synced_lfs_objects' do
+    it 'delegates to #find_synced_lfs_objects_registries' do
+      expect(subject).to receive(:find_synced_lfs_objects_registries).and_call_original
+
+      subject.count_synced_lfs_objects
+    end
+
+    it 'counts LFS objects that has been synced' do
+      create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
+      create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
+      create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
+
+      expect(subject.count_synced_lfs_objects).to eq 2
+    end
+
+    context 'with selective sync' do
+      before do
+        secondary.update_attribute(:namespaces, [synced_group])
+      end
+
+      it 'delegates to #legacy_find_synced_lfs_objects' do
+        expect(subject).to receive(:legacy_find_synced_lfs_objects).and_call_original
+
+        subject.count_synced_lfs_objects
+      end
+
+      it 'counts LFS objects that has been synced' do
+        allow_any_instance_of(LfsObjectsProject).to receive(:update_project_statistics).and_return(nil)
+
+        create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
+        create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
+        create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
+
+        create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
+        create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
+        create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
+
+        expect(subject.count_synced_lfs_objects).to eq 1
+      end
+    end
+  end
+
+  describe '#count_failed_lfs_objects' do
+    it 'delegates to #find_failed_lfs_objects_registries' do
+      expect(subject).to receive(:find_failed_lfs_objects_registries).and_call_original
+
+      subject.count_failed_lfs_objects
+    end
+
+    it 'counts LFS objects that sync has failed' do
+      create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
+      create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
+      create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
+
+      expect(subject.count_failed_lfs_objects).to eq 2
+    end
+
+    context 'with selective sync' do
+      before do
+        secondary.update_attribute(:namespaces, [synced_group])
+      end
+
+      it 'delegates to #legacy_find_failed_lfs_objects' do
+        expect(subject).to receive(:legacy_find_failed_lfs_objects).and_call_original
+
+        subject.count_failed_lfs_objects
+      end
+
+      it 'counts LFS objects that sync has failed' do
+        allow_any_instance_of(LfsObjectsProject).to receive(:update_project_statistics).and_return(nil)
+
+        create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
+        create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
+        create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
+
+        create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
+        create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
+        create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
+
+        expect(subject.count_failed_lfs_objects).to eq 1
+      end
+    end
+  end
+end

spec/ee/spec/finders/geo/project_registry_finder_spec.rb

@@ -8,6 +8,7 @@ describe Geo::ProjectRegistryFinder, :geo, :truncate do
   let(:secondary) { create(:geo_node) }
   let(:synced_group) { create(:group) }
   let!(:project_not_synced) { create(:project) }
+  let(:project_synced) { create(:project) }
   let(:project_repository_dirty) { create(:project) }
   let(:project_wiki_dirty) { create(:project) }
 
@@ -17,6 +18,142 @@ describe Geo::ProjectRegistryFinder, :geo, :truncate do
     stub_current_geo_node(secondary)
   end
 
+  describe '#count_synced_project_registries' do
+    it 'delegates to #find_synced_project_registries' do
+      expect(subject).to receive(:find_synced_project_registries).and_call_original
+
+      subject.count_synced_project_registries
+    end
+
+    it 'counts projects that has been synced' do
+      create(:geo_project_registry, :sync_failed)
+      create(:geo_project_registry, :synced, project: project_synced)
+      create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
+      create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
+
+      expect(subject.count_synced_project_registries).to eq 1
+    end
+
+    context 'with selective sync' do
+      before do
+        secondary.update_attribute(:namespaces, [synced_group])
+      end
+
+      it 'delegates to #legacy_find_synced_project_registries' do
+        expect(subject).to receive(:legacy_find_synced_project_registries).and_call_original
+
+        subject.count_synced_project_registries
+      end
+
+      it 'counts projects that has been synced' do
+        project_1_in_synced_group = create(:project, group: synced_group)
+        project_2_in_synced_group = create(:project, group: synced_group)
+
+        create(:geo_project_registry, :synced, project: project_synced)
+        create(:geo_project_registry, :synced, project: project_1_in_synced_group)
+        create(:geo_project_registry, :sync_failed, project: project_2_in_synced_group)
+
+        expect(subject.count_synced_project_registries).to eq 1
+      end
+    end
+  end
+
+  describe '#count_failed_project_registries' do
+    it 'delegates to #find_failed_project_registries' do
+      expect(subject).to receive(:find_failed_project_registries).and_call_original
+
+      subject.count_failed_project_registries
+    end
+
+    it 'counts projects that sync has failed' do
+      create(:geo_project_registry, :synced)
+      create(:geo_project_registry, :sync_failed, project: project_synced)
+      create(:geo_project_registry, :repository_sync_failed, project: project_repository_dirty)
+      create(:geo_project_registry, :wiki_sync_failed, project: project_wiki_dirty)
+
+      expect(subject.count_failed_project_registries).to eq 3
+    end
+
+    context 'with selective sync' do
+      before do
+        secondary.update_attribute(:namespaces, [synced_group])
+      end
+
+      it 'delegates to #find_failed_project_registries' do
+        expect(subject).to receive(:find_failed_project_registries).and_call_original
+
+        subject.count_failed_project_registries
+      end
+
+      it 'counts projects that sync has failed' do
+        project_1_in_synced_group = create(:project, group: synced_group)
+        project_2_in_synced_group = create(:project, group: synced_group)
+
+        create(:geo_project_registry, :sync_failed, project: project_synced)
+        create(:geo_project_registry, :repository_sync_failed, project: project_1_in_synced_group)
+        create(:geo_project_registry, :synced, project: project_2_in_synced_group)
+
+        expect(subject.count_failed_project_registries).to eq 1
+      end
+    end
+  end
+
+  describe '#find_failed_project_registries' do
+    let(:project_1_in_synced_group) { create(:project, group: synced_group) }
+    let(:project_2_in_synced_group) { create(:project, group: synced_group) }
+
+    let!(:synced) { create(:geo_project_registry, :synced) }
+    let!(:sync_failed) { create(:geo_project_registry, :sync_failed, project: project_synced) }
+    let!(:repository_sync_failed) { create(:geo_project_registry, :repository_sync_failed, project: project_1_in_synced_group) }
+    let!(:wiki_sync_failed) { create(:geo_project_registry, :wiki_sync_failed, project: project_2_in_synced_group) }
+
+    it 'delegates to #find_filtered_failed_project_registries' do
+      expect(subject).to receive(:find_filtered_failed_project_registries).and_call_original
+
+      subject.find_failed_project_registries
+    end
+
+    it 'returns project registries that sync has failed' do
+      expect(subject.find_failed_project_registries).to match_array([sync_failed, repository_sync_failed, wiki_sync_failed])
+    end
+
+    it 'returns only project registries that repository sync has failed' do
+      expect(subject.find_failed_project_registries('repository')).to match_array([sync_failed, repository_sync_failed])
+    end
+
+    it 'returns only project registries that wiki sync has failed' do
+      expect(subject.find_failed_project_registries('wiki')).to match_array([sync_failed, wiki_sync_failed])
+    end
+
+    context 'with selective sync' do
+      before do
+        secondary.update_attribute(:namespaces, [synced_group])
+      end
+
+      it 'delegates to #legacy_find_filtered_failed_project_registries' do
+        expect(subject).to receive(:legacy_find_filtered_failed_project_registries).and_call_original
+
+        subject.find_failed_project_registries
+      end
+
+      it 'returns project registries that sync has failed' do
+        expect(subject.find_failed_project_registries).to match_array([repository_sync_failed, wiki_sync_failed])
+      end
+
+      it 'returns only project registries that repository sync has failed' do
+        create(:geo_project_registry, :repository_sync_failed)
+
+        expect(subject.find_failed_project_registries('repository')).to match_array([repository_sync_failed])
+      end
+
+      it 'returns only project registries that wiki sync has failed' do
+        create(:geo_project_registry, :wiki_sync_failed)
+
+        expect(subject.find_failed_project_registries('wiki')).to match_array([wiki_sync_failed])
+      end
+    end
+  end
+
   context 'FDW' do
     before do
       skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo.fdw?

spec/factories/geo/file_registry.rb

@@ -11,5 +11,18 @@ FactoryGirl.define do
     trait :lfs do
       file_type :lfs
     end
+
+    trait :with_file do
+      after(:build, :stub) do |registry, _|
+        file =
+          if registry.file_type.to_sym == :lfs
+            create(:lfs_object)
+          else
+            create(:upload)
+          end
+
+        registry.file_id = file.id
+      end
+    end
   end
 end

spec/factories/geo/project_registry.rb

@@ -40,6 +40,7 @@ FactoryGirl.define do
       resync_repository true
       resync_wiki true
       repository_retry_count 1
+      wiki_retry_count 1
     end
 
     trait :repository_sync_failed do

spec/models/geo_node_spec.rb

@@ -268,98 +268,34 @@ describe GeoNode, type: :model do
     end
   end
 
-  describe '#restricted_project_ids' do
-    context 'without namespace restriction' do
-      it 'returns nil' do
-        expect(node.restricted_project_ids).to be_nil
-      end
-    end
-
-    context 'with namespace restrictions' do
-      it 'returns an array with unique project ids that belong to the namespaces' do
-        group_1 = create(:group)
-        group_2 = create(:group)
-        nested_group_1 = create(:group, parent: group_1)
-        project_1 = create(:project, group: group_1)
-        project_2 = create(:project, group: nested_group_1)
-        project_3 = create(:project, group: group_2)
-
-        node.update_attribute(:namespaces, [group_1, group_2, nested_group_1])
+  describe '#projects' do
+    let(:group_1) { create(:group) }
+    let(:group_2) { create(:group) }
+    let(:nested_group_1) { create(:group, parent: group_1) }
+    let!(:project_1) { create(:project, group: group_1) }
+    let!(:project_2) { create(:project, group: nested_group_1) }
+    let!(:project_3) { create(:project, group: group_2) }
 
-        expect(node.restricted_project_ids).to match_array([project_1.id, project_2.id, project_3.id])
-      end
+    it 'returns all projects without selective sync' do
+      expect(node.projects).to match_array([project_1, project_2, project_3])
     end
-  end
 
-  describe '#lfs_objects_synced_count' do
-    context 'primary node' do
-      subject { primary_node }
+    it 'returns projects that belong to the namespaces with selective sync' do
+      node.update_attribute(:namespaces, [group_1, nested_group_1])
 
-      it 'returns nil' do
-        expect(subject.lfs_objects_synced_count).to be_nil
-      end
-    end
-
-    context 'secondary node' do
-      subject { node }
-
-      it 'returns a value' do
-        expect(subject.lfs_objects_synced_count).to eq(0)
-      end
+      expect(node.projects).to match_array([project_1, project_2])
     end
   end
 
-  describe '#lfs_objects_failed_count' do
-    context 'primary node' do
-      subject { primary_node }
+  describe '#selective_sync?' do
+    it 'returns true when Geo node has namespace restrictions' do
+      node.update_attribute(:namespaces, [create(:group)])
 
-      it 'returns nil' do
-        expect(subject.lfs_objects_failed_count).to be_nil
-      end
+      expect(node.selective_sync?).to be true
     end
 
-    context 'secondary node' do
-      subject { node }
-
-      it 'returns a value' do
-        expect(subject.lfs_objects_failed_count).to eq(0)
-      end
-    end
-  end
-
-  describe '#attachments_synced_count' do
-    context 'primary node' do
-      subject { primary_node }
-
-      it 'returns nil' do
-        expect(subject.attachments_synced_count).to be_nil
-      end
-    end
-
-    context 'secondary node' do
-      subject { node }
-
-      it 'returns a value' do
-        expect(subject.attachments_synced_count).to eq(0)
-      end
-    end
-  end
-
-  describe '#attachments_failed_count' do
-    context 'primary node' do
-      subject { primary_node }
-
-      it 'returns nil' do
-        expect(subject.attachments_failed_count).to be_nil
-      end
-    end
-
-    context 'secondary node' do
-      subject { node }
-
-      it 'returns a value' do
-        expect(subject.attachments_failed_count).to eq(0)
-      end
+    it 'returns false when Geo node does not have namespace restrictions' do
+      expect(node.selective_sync?).to be false
     end
   end
 end

spec/models/geo_node_status_spec.rb

 require 'spec_helper'
 
-describe GeoNodeStatus, :geo do
+# Disable transactions via :truncate method because a foreign table
+# can't see changes inside a transaction of a different connection.
+describe GeoNodeStatus, :geo, :truncate do
   include ::EE::GeoHelpers
 
-  set(:primary)  { create(:geo_node, :primary) }
-  set(:secondary) { create(:geo_node) }
+  let!(:primary)  { create(:geo_node, :primary) }
+  let!(:secondary) { create(:geo_node) }
 
-  set(:group)     { create(:group) }
-  set(:project_1) { create(:project, group: group) }
-  set(:project_2) { create(:project, group: group) }
-  set(:project_3) { create(:project) }
-  set(:project_4) { create(:project) }
+  let!(:group)     { create(:group) }
+  let!(:project_1) { create(:project, group: group) }
+  let!(:project_2) { create(:project, group: group) }
+  let!(:project_3) { create(:project) }
+  let!(:project_4) { create(:project) }
 
   subject { described_class.current_node_status }
 
@@ -96,13 +98,13 @@ describe GeoNodeStatus, :geo do
   describe '#attachments_failed_count' do
     it 'counts failed avatars, attachment, personal snippets and files' do
       # These two should be ignored
-      create(:geo_file_registry, :lfs, success: false)
-      create(:geo_file_registry)
+      create(:geo_file_registry, :lfs, :with_file, success: false)
+      create(:geo_file_registry, :with_file)
 
-      create(:geo_file_registry, file_type: :personal_file, success: false)
-      create(:geo_file_registry, file_type: :attachment, success: false)
-      create(:geo_file_registry, :avatar, success: false)
-      create(:geo_file_registry, success: false)
+      create(:geo_file_registry, :with_file, file_type: :personal_file, success: false)
+      create(:geo_file_registry, :with_file, file_type: :attachment, success: false)
+      create(:geo_file_registry, :avatar, :with_file, success: false)
+      create(:geo_file_registry, :with_file, success: false)
 
       expect(subject.attachments_failed_count).to eq(4)
     end
@@ -160,9 +162,9 @@ describe GeoNodeStatus, :geo do
       create(:geo_file_registry, success: false)
       create(:geo_file_registry, :avatar, success: false)
       create(:geo_file_registry, file_type: :attachment, success: false)
-      create(:geo_file_registry, :lfs)
+      create(:geo_file_registry, :lfs, :with_file)
 
-      create(:geo_file_registry, :lfs, success: false)
+      create(:geo_file_registry, :lfs, :with_file, success: false)
 
       expect(subject.lfs_objects_failed_count).to eq(1)
     end