Add methods to count synced/failed projects to Geo::ProjecRegistryFinder

parent d6a5fa5e
...@@ -72,14 +72,10 @@ class GeoNodeStatus < ActiveRecord::Base ...@@ -72,14 +72,10 @@ class GeoNodeStatus < ActiveRecord::Base
self.db_replication_lag_seconds = Gitlab::Geo::HealthCheck.db_replication_lag_seconds self.db_replication_lag_seconds = Gitlab::Geo::HealthCheck.db_replication_lag_seconds
self.cursor_last_event_id = Geo::EventLogState.last_processed&.event_id self.cursor_last_event_id = Geo::EventLogState.last_processed&.event_id
self.cursor_last_event_date = Geo::EventLog.find_by(id: self.cursor_last_event_id)&.created_at self.cursor_last_event_date = Geo::EventLog.find_by(id: self.cursor_last_event_id)&.created_at
self.repositories_synced_count = geo_node.project_registries.synced.count self.repositories_synced_count = projects_finder.count_synced_projects
self.repositories_failed_count = geo_node.project_registries.failed.count self.repositories_failed_count = projects_finder.count_failed_projects
lfs_objects_finder = Geo::LfsObjectRegistryFinder.new(current_node: geo_node)
self.lfs_objects_synced_count = lfs_objects_finder.count_synced_lfs_objects self.lfs_objects_synced_count = lfs_objects_finder.count_synced_lfs_objects
self.lfs_objects_failed_count = lfs_objects_finder.count_failed_lfs_objects self.lfs_objects_failed_count = lfs_objects_finder.count_failed_lfs_objects
attachments_finder = Geo::AttachmentRegistryFinder.new(current_node: geo_node)
self.attachments_synced_count = attachments_finder.find_synced_attachments.count self.attachments_synced_count = attachments_finder.find_synced_attachments.count
self.attachments_failed_count = attachments_finder.find_failed_attachments.count self.attachments_failed_count = attachments_finder.find_failed_attachments.count
end end
...@@ -139,6 +135,18 @@ class GeoNodeStatus < ActiveRecord::Base ...@@ -139,6 +135,18 @@ class GeoNodeStatus < ActiveRecord::Base
private private
def attachments_finder
@attachments_finder ||= Geo::AttachmentRegistryFinder.new(current_node: geo_node)
end
def lfs_objects_finder
@lfs_objects_finder ||= Geo::LfsObjectRegistryFinder.new(current_node: geo_node)
end
def projects_finder
@projects_finder ||= Geo::ProjectRegistryFinder.new(current_node: geo_node)
end
def sync_percentage(total, synced) def sync_percentage(total, synced)
return 0 if !total.present? || total.zero? return 0 if !total.present? || total.zero?
......
module Geo module Geo
class ProjectRegistryFinder < RegistryFinder class ProjectRegistryFinder < RegistryFinder
def count_synced_projects
relation =
if selective_sync?
legacy_find_synced_projects
else
find_synced_projects_registries
end
relation.count
end
def count_failed_projects
relation =
if selective_sync?
legacy_find_failed_projects
else
find_failed_projects_registries
end
relation.count
end
def find_unsynced_projects(batch_size:) def find_unsynced_projects(batch_size:)
relation = relation =
if fdw? if fdw?
...@@ -24,14 +46,22 @@ module Geo ...@@ -24,14 +46,22 @@ module Geo
protected protected
def fdw_table def find_synced_projects_registries
Geo::Fdw::Project.table_name Geo::ProjectRegistry.synced
end
def find_failed_projects_registries
Geo::ProjectRegistry.failed
end end
# #
# FDW accessors # FDW accessors
# #
def fdw_table
Geo::Fdw::Project.table_name
end
# @return [ActiveRecord::Relation<Geo::Fdw::Project>] # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
def fdw_find_unsynced_projects def fdw_find_unsynced_projects
Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id") Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
...@@ -66,14 +96,27 @@ module Geo ...@@ -66,14 +96,27 @@ module Geo
# @return [ActiveRecord::Relation<Project>] list of projects updated recently # @return [ActiveRecord::Relation<Project>] list of projects updated recently
def legacy_find_projects_updated_recently def legacy_find_projects_updated_recently
registry_project_ids = current_node.project_registries.dirty.retry_due.pluck(:project_id) legacy_find_projects(current_node.project_registries.dirty.retry_due.pluck(:project_id))
end
# @return [ActiveRecord::Relation<Project>] list of synced projects
def legacy_find_synced_projects
legacy_find_projects(Geo::ProjectRegistry.synced.pluck(:project_id))
end
# @return [ActiveRecord::Relation<Project>] list of projects that sync has failed
def legacy_find_failed_projects
legacy_find_projects(Geo::ProjectRegistry.failed.pluck(:project_id))
end
def legacy_find_projects(registry_project_ids)
return Project.none if registry_project_ids.empty? return Project.none if registry_project_ids.empty?
joined_relation = current_node.projects.joins(<<~SQL) joined_relation = current_node.projects.joins(<<~SQL)
INNER JOIN INNER JOIN
(VALUES #{registry_project_ids.map { |id| "(#{id})" }.join(',')}) (VALUES #{registry_project_ids.map { |id| "(#{id})" }.join(',')})
project_registry(project_id) project_registry(project_id)
ON projects.id = project_registry.project_id ON #{Project.table_name}.id = project_registry.project_id
SQL SQL
joined_relation joined_relation
......
...@@ -8,6 +8,7 @@ describe Geo::ProjectRegistryFinder, :geo, :truncate do ...@@ -8,6 +8,7 @@ describe Geo::ProjectRegistryFinder, :geo, :truncate do
let(:secondary) { create(:geo_node) } let(:secondary) { create(:geo_node) }
let(:synced_group) { create(:group) } let(:synced_group) { create(:group) }
let!(:project_not_synced) { create(:project) } let!(:project_not_synced) { create(:project) }
let(:project_synced) { create(:project) }
let(:project_repository_dirty) { create(:project) } let(:project_repository_dirty) { create(:project) }
let(:project_wiki_dirty) { create(:project) } let(:project_wiki_dirty) { create(:project) }
...@@ -17,6 +18,86 @@ describe Geo::ProjectRegistryFinder, :geo, :truncate do ...@@ -17,6 +18,86 @@ describe Geo::ProjectRegistryFinder, :geo, :truncate do
stub_current_geo_node(secondary) stub_current_geo_node(secondary)
end end
describe '#count_synced_projects' do
it 'delegates to #find_synced_projects_registries' do
expect(subject).to receive(:find_synced_projects_registries).and_call_original
subject.count_synced_projects
end
it 'counts projects that has been synced' do
create(:geo_project_registry, :sync_failed)
create(:geo_project_registry, :synced, project: project_synced)
create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
expect(subject.count_synced_projects).to eq 1
end
context 'with selective sync' do
before do
secondary.update_attribute(:namespaces, [synced_group])
end
it 'delegates to #legacy_find_synced_projects' do
expect(subject).to receive(:legacy_find_synced_projects).and_call_original
subject.count_synced_projects
end
it 'counts projects that has been synced' do
project_1_in_synced_group = create(:project, group: synced_group)
project_2_in_synced_group = create(:project, group: synced_group)
create(:geo_project_registry, :synced, project: project_synced)
create(:geo_project_registry, :synced, project: project_1_in_synced_group)
create(:geo_project_registry, :sync_failed, project: project_2_in_synced_group)
expect(subject.count_synced_projects).to eq 1
end
end
end
describe '#count_failed_projects' do
it 'delegates to #find_failed_projects_registries' do
expect(subject).to receive(:find_failed_projects_registries).and_call_original
subject.count_failed_projects
end
it 'counts projects that sync has failed' do
create(:geo_project_registry, :synced)
create(:geo_project_registry, :sync_failed, project: project_synced)
create(:geo_project_registry, :repository_sync_failed, project: project_repository_dirty)
create(:geo_project_registry, :wiki_sync_failed, project: project_wiki_dirty)
expect(subject.count_failed_projects).to eq 3
end
context 'with selective sync' do
before do
secondary.update_attribute(:namespaces, [synced_group])
end
it 'delegates to #legacy_find_failed_projects' do
expect(subject).to receive(:legacy_find_failed_projects).and_call_original
subject.count_failed_projects
end
it 'counts projects that sync has failed' do
project_1_in_synced_group = create(:project, group: synced_group)
project_2_in_synced_group = create(:project, group: synced_group)
create(:geo_project_registry, :sync_failed, project: project_synced)
create(:geo_project_registry, :repository_sync_failed, project: project_1_in_synced_group)
create(:geo_project_registry, :synced, project: project_2_in_synced_group)
expect(subject.count_failed_projects).to eq 1
end
end
end
context 'FDW' do context 'FDW' do
before do before do
skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo.fdw? skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo.fdw?
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment