Skip to content

G
gitlab-ce
Commit ec247475 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre
Browse files
Options

Merge branch... 

Merge branch '227681-remove-feature-flag-to-make-registry-table-ssot-for-container-registry' into 'master'

Geo - Remove feature flag to make registry table SSOT for Container Registry

Closes #227681

See merge request gitlab-org/gitlab!38356
parents 6b0d7e8a 99af0f44
Hide whitespace changes
Inline Side-by-side
Showing with 178 additions and 520 deletions
ee/app/finders/geo/container_repository_registry_finder.rb
View file @ ec247475
...@@ -7,11 +7,11 @@ module Geo ...@@ -7,11 +7,11 @@ module Geo
end end
def count_synced def count_synced
registries.merge(Geo::ContainerRepositoryRegistry.synced).count Geo::ContainerRepositoryRegistry.synced.count
end end
def count_failed def count_failed
registries.merge(Geo::ContainerRepositoryRegistry.failed).count Geo::ContainerRepositoryRegistry.failed.count
end end
def count_registry def count_registry
...@@ -63,43 +63,5 @@ module Geo ...@@ -63,43 +63,5 @@ module Geo
.limit(batch_size) .limit(batch_size)
end end
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
# Find limited amount of non replicated container repositories.
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_ids: [])
current_node_fdw
.container_repositories
.missing_container_repository_registry
.id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
def find_retryable_failed_ids(batch_size:, except_ids: [])
Geo::ContainerRepositoryRegistry
.failed
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
.pluck_container_repository_key
end
# rubocop:enable CodeReuse/ActiveRecord
private
def registries
if Geo::ContainerRepositoryRegistry.registry_consistency_worker_enabled?
Geo::ContainerRepositoryRegistry.all
else
current_node_fdw.container_repositories.inner_join_container_repository_registry
end
end
end end
end end
ee/app/models/geo/container_repository_registry.rb
View file @ ec247475
...@@ -55,10 +55,6 @@ class Geo::ContainerRepositoryRegistry < Geo::BaseRegistry ...@@ -55,10 +55,6 @@ class Geo::ContainerRepositoryRegistry < Geo::BaseRegistry
where(nil).pluck(:container_repository_id) where(nil).pluck(:container_repository_id)
end end
def self.registry_consistency_worker_enabled?
Feature.enabled?(:geo_container_registry_ssot_sync, default_enabled: true)
end
def self.replication_enabled? def self.replication_enabled?
Gitlab.config.geo.registry_replication.enabled Gitlab.config.geo.registry_replication.enabled
end end
......
ee/app/models/geo/fdw/container_repository.rb deleted 100644 → 0
View file @ 6b0d7e8a
# frozen_string_literal: true
module Geo
module Fdw
class ContainerRepository < ::Geo::BaseFdw
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('container_repositories')
self.primary_key = :id
belongs_to :project, class_name: 'Geo::Fdw::Project', inverse_of: :container_repositories
scope :project_id_in, ->(ids) { joins(:project).merge(Geo::Fdw::Project.id_in(ids)) }
class << self
def inner_join_container_repository_registry
join_statement =
arel_table
.join(container_repository_registry_table, Arel::Nodes::InnerJoin)
.on(arel_table[:id].eq(container_repository_registry_table[:container_repository_id]))
joins(join_statement.join_sources)
end
def missing_container_repository_registry
left_outer_join_container_repository_registry
.where(container_repository_registry_table[:id].eq(nil))
end
private
def container_repository_registry_table
Geo::ContainerRepositoryRegistry.arel_table
end
def left_outer_join_container_repository_registry
join_statement =
arel_table
.join(container_repository_registry_table, Arel::Nodes::OuterJoin)
.on(arel_table[:id].eq(container_repository_registry_table[:container_repository_id]))
joins(join_statement.join_sources)
end
end
end
end
end
ee/app/models/geo/fdw/geo_node.rb
View file @ ec247475
...@@ -26,12 +26,6 @@ module Geo ...@@ -26,12 +26,6 @@ module Geo
end end
end end
def container_repositories
return Geo::Fdw::ContainerRepository.all unless selective_sync?
Geo::Fdw::ContainerRepository.project_id_in(projects)
end
private private
def projects_for_selected_namespaces def projects_for_selected_namespaces
......
ee/app/models/geo/fdw/project.rb
View file @ ec247475
...@@ -8,8 +8,6 @@ module Geo ...@@ -8,8 +8,6 @@ module Geo
self.primary_key = :id self.primary_key = :id
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('projects') self.table_name = Gitlab::Geo::Fdw.foreign_table_name('projects')
has_many :container_repositories, class_name: 'Geo::Fdw::ContainerRepository'
belongs_to :namespace, class_name: 'Geo::Fdw::Namespace' belongs_to :namespace, class_name: 'Geo::Fdw::Namespace'
alias_method :parent, :namespace alias_method :parent, :namespace
......
ee/app/workers/geo/container_repository_sync_dispatch_worker.rb
View file @ ec247475
...@@ -47,26 +47,15 @@ module Geo ...@@ -47,26 +47,15 @@ module Geo
end end
def find_container_repository_ids_not_synced(batch_size:) def find_container_repository_ids_not_synced(batch_size:)
if Geo::ContainerRepositoryRegistry.registry_consistency_worker_enabled? registry_finder
registry_finder .find_never_synced_registries(batch_size: batch_size, except_ids: scheduled_repository_ids)
.find_never_synced_registries(batch_size: batch_size, except_ids: scheduled_repository_ids) .pluck_model_foreign_key
.pluck_model_foreign_key
else
registry_finder
.find_unsynced(batch_size: batch_size, except_ids: scheduled_repository_ids)
.pluck_primary_key
end
end end
def find_retryable_container_registry_ids(batch_size:) def find_retryable_container_registry_ids(batch_size:)
if Geo::ContainerRepositoryRegistry.registry_consistency_worker_enabled? registry_finder
registry_finder .find_retryable_dirty_registries(batch_size: batch_size, except_ids: scheduled_repository_ids)
.find_retryable_dirty_registries(batch_size: batch_size, except_ids: scheduled_repository_ids) .pluck_model_foreign_key
.pluck_model_foreign_key
else
registry_finder
.find_retryable_failed_ids(batch_size: batch_size, except_ids: scheduled_repository_ids)
end
end end
def registry_finder def registry_finder
......
ee/spec/finders/geo/container_repository_registry_finder_spec.rb
View file @ ec247475
...@@ -4,337 +4,232 @@ require 'spec_helper' ...@@ -4,337 +4,232 @@ require 'spec_helper'
RSpec.describe Geo::ContainerRepositoryRegistryFinder, :geo do RSpec.describe Geo::ContainerRepositoryRegistryFinder, :geo do
include ::EE::GeoHelpers include ::EE::GeoHelpers
let_it_be(:secondary) { create(:geo_node) }
let_it_be(:synced_group) { create(:group) }
let_it_be(:nested_group) { create(:group, parent: synced_group) }
let_it_be(:project_synced_group) { create(:project, group: synced_group) }
let_it_be(:project_nested_group) { create(:project, group: nested_group) }
let_it_be(:project_broken_storage) { create(:project, :broken_storage) }
let_it_be(:container_repository_1) { create(:container_repository, project: project_synced_group) }
let_it_be(:container_repository_2) { create(:container_repository, project: project_nested_group) }
let_it_be(:container_repository_3) { create(:container_repository) }
let_it_be(:container_repository_4) { create(:container_repository) }
let_it_be(:container_repository_5) { create(:container_repository, project: project_broken_storage) }
let_it_be(:container_repository_6) { create(:container_repository, project: project_broken_storage) }
subject { described_class.new(current_node_id: secondary.id) }
before do before do
stub_current_geo_node(secondary)
stub_registry_replication_config(enabled: true) stub_registry_replication_config(enabled: true)
end end
context 'when geo_container_registry_ssot_sync is disabled', :geo_fdw do describe '#count_syncable' do
let!(:secondary) { create(:geo_node) } it 'returns number of container repositories' do
let!(:container_repository) { create(:container_repository) } expect(subject.count_syncable).to eq(6)
let!(:failed_registry) { create(:container_repository_registry, :sync_failed) }
let!(:synced_registry) { create(:container_repository_registry, :synced) }
let(:synced_group) { create(:group) }
let(:unsynced_group) { create(:group) }
let(:synced_project) { create(:project, group: synced_group) }
let(:unsynced_project) { create(:project, :broken_storage, group: unsynced_group) }
subject { described_class.new(current_node_id: secondary.id) }
before do
stub_current_geo_node(secondary)
stub_feature_flags(geo_container_registry_ssot_sync: false)
end end
end
describe '#count_syncable' do describe '#count_synced' do
it 'returns number of container repositories' do it 'returns only synced registry' do
expect(subject.count_syncable).to eq(3) create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id)
end create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_3.id)
expect(subject.count_synced).to eq(1)
end end
end
describe '#count_synced' do describe '#count_failed' do
it 'returns only synced registry' do it 'returns only failed registry' do
expect(subject.count_synced).to eq(1) create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id)
end create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_3.id)
expect(subject.count_failed).to eq(1)
end end
end
describe '#count_failed' do describe '#count_registry' do
it 'returns only failed registry' do it 'returns number of all registries' do
expect(subject.count_failed).to eq(1) create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id)
end create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_3.id)
expect(subject.count_registry).to eq(2)
end end
end
describe '#count_registry' do describe '#find_registry_differences' do
it 'returns number of all registries' do context 'untracked IDs' do
expect(subject.count_registry).to eq(2) before do
create(:container_repository_registry, container_repository_id: container_repository_1.id)
create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_3.id)
create(:container_repository_registry, container_repository_id: container_repository_5.id)
end end
end
describe '#find_unsynced' do it 'includes container registries IDs without an entry on the tracking database' do
it 'returns repositories without an entry in the tracking database' do range = ContainerRepository.minimum(:id)..ContainerRepository.maximum(:id)
repositories = subject.find_unsynced(batch_size: 10)
untracked_ids, _ = subject.find_registry_differences(range)
expect(repositories).to match_ids(container_repository) expect(untracked_ids).to match_array([container_repository_2.id, container_repository_4.id, container_repository_6.id])
end end
it 'returns repositories without an entry in the tracking database, excluding exception list' do it 'excludes container registries outside the ID range' do
except_repository = create(:container_repository) untracked_ids, _ = subject.find_registry_differences(container_repository_4.id..container_repository_6.id)
repositories = subject.find_unsynced(batch_size: 10, except_ids: [except_repository.id])
expect(repositories).to match_ids(container_repository) expect(untracked_ids).to match_array([container_repository_4.id, container_repository_6.id])
end end
context 'with selective sync by namespace' do context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'returns repositories without an entry in the tracking database, excluding exception list' do it 'excludes container_registry IDs that projects are not in the selected namespaces' do
except_repository = create(:container_repository, project: synced_project) range = ContainerRepository.minimum(:id)..ContainerRepository.maximum(:id)
repository = create(:container_repository, project: synced_project, name: 'second')
repositories = subject.find_unsynced(batch_size: 10, except_ids: [except_repository.id]) untracked_ids, _ = subject.find_registry_differences(range)
expect(repositories).to match_ids(repository) expect(untracked_ids).to match_array([container_repository_2.id])
end end
end end
context 'with selective sync by shard' do context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'returns repositories without an entry in the tracking database' do it 'excludes container_registry IDs that projects are not in the selected shards' do
unsynced_repository = create(:container_repository, project: unsynced_project) range = ContainerRepository.minimum(:id)..ContainerRepository.maximum(:id)
repositories = subject.find_unsynced(batch_size: 10) untracked_ids, _ = subject.find_registry_differences(range)
expect(repositories).to match_ids(unsynced_repository) expect(untracked_ids).to match_array([container_repository_6.id])
end end
end end
end end
describe '#find_retryable_failed_ids' do context 'unused tracked IDs' do
it 'returns only registry that have to be retried' do context 'with an orphaned registry' do
result = subject.find_retryable_failed_ids(batch_size: 10) let!(:orphaned) { create(:container_repository_registry, container_repository_id: container_repository_1.id) }
expect(result).to eq([failed_registry.container_repository_id])
end
it 'returns only registry that have to be retried, excluding exception list' do
except_repository = create(:container_repository)
create(:container_repository_registry, :sync_failed, container_repository: except_repository)
result = subject.find_retryable_failed_ids(batch_size: 10, except_ids: [except_repository.id])
expect(result).to eq([failed_registry.container_repository_id])
end
end
end
context 'when geo_container_registry_ssot_sync is enabled' do
let_it_be(:secondary) { create(:geo_node) }
let_it_be(:synced_group) { create(:group) }
let_it_be(:nested_group) { create(:group, parent: synced_group) }
let_it_be(:project_synced_group) { create(:project, group: synced_group) }
let_it_be(:project_nested_group) { create(:project, group: nested_group) }
let_it_be(:project_broken_storage) { create(:project, :broken_storage) }
let_it_be(:container_repository_1) { create(:container_repository, project: project_synced_group) }
let_it_be(:container_repository_2) { create(:container_repository, project: project_nested_group) }
let_it_be(:container_repository_3) { create(:container_repository) }
let_it_be(:container_repository_4) { create(:container_repository) }
let_it_be(:container_repository_5) { create(:container_repository, project: project_broken_storage) }
let_it_be(:container_repository_6) { create(:container_repository, project: project_broken_storage) }
subject { described_class.new(current_node_id: secondary.id) }
before do
stub_current_geo_node(secondary)
stub_feature_flags(geo_container_registry_ssot_sync: true)
end
describe '#count_syncable' do
it 'returns number of container repositories' do
expect(subject.count_syncable).to eq(6)
end
end
describe '#count_synced' do
it 'returns only synced registry' do
create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id)
create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_3.id)
expect(subject.count_synced).to eq(1)
end
end
describe '#count_failed' do
it 'returns only failed registry' do
create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id)
create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_3.id)
expect(subject.count_failed).to eq(1)
end
end
describe '#count_registry' do
it 'returns number of all registries' do
create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id)
create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_3.id)
expect(subject.count_registry).to eq(2)
end
end
describe '#find_registry_differences' do
context 'untracked IDs' do
before do before do
create(:container_repository_registry, container_repository_id: container_repository_1.id) container_repository_1.delete
create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_3.id)
create(:container_repository_registry, container_repository_id: container_repository_5.id)
end
it 'includes container registries IDs without an entry on the tracking database' do
range = ContainerRepository.minimum(:id)..ContainerRepository.maximum(:id)
untracked_ids, _ = subject.find_registry_differences(range)
expect(untracked_ids).to match_array([container_repository_2.id, container_repository_4.id, container_repository_6.id])
end
it 'excludes container registries outside the ID range' do
untracked_ids, _ = subject.find_registry_differences(container_repository_4.id..container_repository_6.id)
expect(untracked_ids).to match_array([container_repository_4.id, container_repository_6.id])
end end
context 'with selective sync by namespace' do it 'includes tracked IDs that do not exist in the model table' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } range = container_repository_1.id..container_repository_1.id
it 'excludes container_registry IDs that projects are not in the selected namespaces' do _, unused_tracked_ids = subject.find_registry_differences(range)
range = ContainerRepository.minimum(:id)..ContainerRepository.maximum(:id)
untracked_ids, _ = subject.find_registry_differences(range) expect(unused_tracked_ids).to match_array([container_repository_1.id])
expect(untracked_ids).to match_array([container_repository_2.id])
end
end end
context 'with selective sync by shard' do it 'excludes IDs outside the ID range' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } range = (container_repository_1.id + 1)..ContainerRepository.maximum(:id)
it 'excludes container_registry IDs that projects are not in the selected shards' do
range = ContainerRepository.minimum(:id)..ContainerRepository.maximum(:id)
untracked_ids, _ = subject.find_registry_differences(range) _, unused_tracked_ids = subject.find_registry_differences(range)
expect(untracked_ids).to match_array([container_repository_6.id]) expect(unused_tracked_ids).to be_empty
end
end end
end end
context 'unused tracked IDs' do context 'with selective sync by namespace' do
context 'with an orphaned registry' do let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
let!(:orphaned) { create(:container_repository_registry, container_repository_id: container_repository_1.id) }
before do
container_repository_1.delete
end
it 'includes tracked IDs that do not exist in the model table' do
range = container_repository_1.id..container_repository_1.id
_, unused_tracked_ids = subject.find_registry_differences(range)
expect(unused_tracked_ids).to match_array([container_repository_1.id])
end
it 'excludes IDs outside the ID range' do
range = (container_repository_1.id + 1)..ContainerRepository.maximum(:id)
_, unused_tracked_ids = subject.find_registry_differences(range)
expect(unused_tracked_ids).to be_empty
end
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
context 'with a tracked container_registry' do context 'with a tracked container_registry' do
context 'excluded from selective sync' do context 'excluded from selective sync' do
let!(:registry_entry) { create(:container_repository_registry, container_repository_id: container_repository_3.id) } let!(:registry_entry) { create(:container_repository_registry, container_repository_id: container_repository_3.id) }
it 'includes tracked container_registry IDs that exist but are not in a selectively synced project' do it 'includes tracked container_registry IDs that exist but are not in a selectively synced project' do
range = container_repository_3.id..container_repository_3.id range = container_repository_3.id..container_repository_3.id
_, unused_tracked_ids = subject.find_registry_differences(range) _, unused_tracked_ids = subject.find_registry_differences(range)
expect(unused_tracked_ids).to match_array([container_repository_3.id]) expect(unused_tracked_ids).to match_array([container_repository_3.id])
end
end end
end
context 'included in selective sync' do context 'included in selective sync' do
let!(:registry_entry) { create(:container_repository_registry, container_repository_id: container_repository_1.id) } let!(:registry_entry) { create(:container_repository_registry, container_repository_id: container_repository_1.id) }
it 'excludes tracked container_registry IDs that are in selectively synced projects' do it 'excludes tracked container_registry IDs that are in selectively synced projects' do
range = container_repository_1.id..container_repository_1.id range = container_repository_1.id..container_repository_1.id
_, unused_tracked_ids = subject.find_registry_differences(range) _, unused_tracked_ids = subject.find_registry_differences(range)
expect(unused_tracked_ids).to be_empty expect(unused_tracked_ids).to be_empty
end
end end
end end
end end
end
context 'with selective sync by shard' do context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
context 'with a tracked container_registry' do context 'with a tracked container_registry' do
let!(:registry_entry) { create(:container_repository_registry, container_repository_id: container_repository_1.id) } let!(:registry_entry) { create(:container_repository_registry, container_repository_id: container_repository_1.id) }
context 'excluded from selective sync' do context 'excluded from selective sync' do
it 'includes tracked container_registry IDs that exist but are not in a selectively synced project' do it 'includes tracked container_registry IDs that exist but are not in a selectively synced project' do
range = container_repository_1.id..container_repository_1.id range = container_repository_1.id..container_repository_1.id
_, unused_tracked_ids = subject.find_registry_differences(range) _, unused_tracked_ids = subject.find_registry_differences(range)
expect(unused_tracked_ids).to match_array([container_repository_1.id]) expect(unused_tracked_ids).to match_array([container_repository_1.id])
end
end end
end
context 'included in selective sync' do context 'included in selective sync' do
let!(:registry_entry) { create(:container_repository_registry, container_repository_id: container_repository_5.id) } let!(:registry_entry) { create(:container_repository_registry, container_repository_id: container_repository_5.id) }
it 'excludes tracked container_registry IDs that are in selectively synced projects' do it 'excludes tracked container_registry IDs that are in selectively synced projects' do
range = container_repository_5.id..container_repository_5.id range = container_repository_5.id..container_repository_5.id
_, unused_tracked_ids = subject.find_registry_differences(range) _, unused_tracked_ids = subject.find_registry_differences(range)
expect(unused_tracked_ids).to be_empty expect(unused_tracked_ids).to be_empty
end
end end
end end
end end
end end
end end
end
describe '#find_never_synced_registries' do describe '#find_never_synced_registries' do
let!(:registry_container_registry_1) { create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id) } let_it_be(:registry_container_registry_1) { create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id) }
let!(:registry_container_registry_2) { create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_2.id) } let_it_be(:registry_container_registry_2) { create(:container_repository_registry, :sync_failed, container_repository_id: container_repository_2.id) }
let!(:registry_container_registry_3) { create(:container_repository_registry, container_repository_id: container_repository_3.id, last_synced_at: nil) } let_it_be(:registry_container_registry_3) { create(:container_repository_registry, container_repository_id: container_repository_3.id, last_synced_at: nil) }
let!(:registry_container_registry_4) { create(:container_repository_registry, container_repository_id: container_repository_4.id, last_synced_at: 3.days.ago, retry_at: 2.days.ago) } let_it_be(:registry_container_registry_4) { create(:container_repository_registry, container_repository_id: container_repository_4.id, last_synced_at: 3.days.ago, retry_at: 2.days.ago) }
let!(:registry_container_registry_5) { create(:container_repository_registry, container_repository_id: container_repository_5.id, last_synced_at: 6.days.ago) } let_it_be(:registry_container_registry_5) { create(:container_repository_registry, container_repository_id: container_repository_5.id, last_synced_at: 6.days.ago) }
let!(:registry_container_registry_6) { create(:container_repository_registry, container_repository_id: container_repository_6.id, last_synced_at: nil) } let_it_be(:registry_container_registry_6) { create(:container_repository_registry, container_repository_id: container_repository_6.id, last_synced_at: nil) }
it 'returns registries for projects that have never been synced' do it 'returns registries for projects that have never been synced' do
registries = subject.find_never_synced_registries(batch_size: 10) registries = subject.find_never_synced_registries(batch_size: 10)
expect(registries).to match_ids(registry_container_registry_3, registry_container_registry_6) expect(registries).to match_ids(registry_container_registry_3, registry_container_registry_6)
end end
it 'excludes except_ids' do it 'excludes except_ids' do
registries = subject.find_never_synced_registries(batch_size: 10, except_ids: [container_repository_3.id]) registries = subject.find_never_synced_registries(batch_size: 10, except_ids: [container_repository_3.id])
expect(registries).to match_ids(registry_container_registry_6) expect(registries).to match_ids(registry_container_registry_6)
end
end end
end
describe '#find_retryable_dirty_registries' do describe '#find_retryable_dirty_registries' do
let!(:registry_container_registry_1) { create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id) } let_it_be(:registry_container_registry_1) { create(:container_repository_registry, :synced, container_repository_id: container_repository_1.id) }
let!(:registry_container_registry_2) { create(:container_repository_registry, :sync_started, container_repository_id: container_repository_2.id) } let_it_be(:registry_container_registry_2) { create(:container_repository_registry, :sync_started, container_repository_id: container_repository_2.id) }
let!(:registry_container_registry_3) { create(:container_repository_registry, state: :failed, container_repository_id: container_repository_3.id, last_synced_at: nil) } let_it_be(:registry_container_registry_3) { create(:container_repository_registry, state: :failed, container_repository_id: container_repository_3.id, last_synced_at: nil) }
let!(:registry_container_registry_4) { create(:container_repository_registry, state: :failed, container_repository_id: container_repository_4.id, last_synced_at: 3.days.ago, retry_at: 2.days.ago) } let_it_be(:registry_container_registry_4) { create(:container_repository_registry, state: :failed, container_repository_id: container_repository_4.id, last_synced_at: 3.days.ago, retry_at: 2.days.ago) }
let!(:registry_container_registry_5) { create(:container_repository_registry, state: :failed, container_repository_id: container_repository_5.id, last_synced_at: 6.days.ago) } let_it_be(:registry_container_registry_5) { create(:container_repository_registry, state: :failed, container_repository_id: container_repository_5.id, last_synced_at: 6.days.ago) }
let!(:registry_container_registry_6) { create(:container_repository_registry, state: :failed, container_repository_id: container_repository_6.id, last_synced_at: nil) } let_it_be(:registry_container_registry_6) { create(:container_repository_registry, state: :failed, container_repository_id: container_repository_6.id, last_synced_at: nil) }
it 'returns registries for projects that have been recently updated' do it 'returns registries for projects that have been recently updated' do
registries = subject.find_retryable_dirty_registries(batch_size: 10) registries = subject.find_retryable_dirty_registries(batch_size: 10)
expect(registries).to match_ids(registry_container_registry_3, registry_container_registry_4, registry_container_registry_5, registry_container_registry_6) expect(registries).to match_ids(registry_container_registry_3, registry_container_registry_4, registry_container_registry_5, registry_container_registry_6)
end end
it 'excludes except_ids' do it 'excludes except_ids' do
registries = subject.find_retryable_dirty_registries(batch_size: 10, except_ids: [container_repository_4.id, container_repository_5.id, container_repository_6.id]) registries = subject.find_retryable_dirty_registries(batch_size: 10, except_ids: [container_repository_4.id, container_repository_5.id, container_repository_6.id])
expect(registries).to match_ids(registry_container_registry_3) expect(registries).to match_ids(registry_container_registry_3)
end
end end
end end
end end
ee/spec/models/geo/fdw/container_repository_spec.rb deleted 100644 → 0
View file @ 6b0d7e8a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::Fdw::ContainerRepository, :geo, type: :model do
context 'relationships' do
it { is_expected.to belong_to(:project).class_name('Geo::Fdw::Project') }
end
end
ee/spec/models/geo/fdw/project_spec.rb deleted 100644 → 0
View file @ 6b0d7e8a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::Fdw::Project, :geo_fdw, type: :model do
context 'relationships' do
it { is_expected.to have_many(:container_repositories).class_name('Geo::Fdw::ContainerRepository') }
end
end
ee/spec/workers/geo/container_repository_sync_dispatch_worker_spec.rb
View file @ ec247475
...@@ -6,8 +6,8 @@ RSpec.describe Geo::ContainerRepositorySyncDispatchWorker, :geo, :use_sql_query_ ...@@ -6,8 +6,8 @@ RSpec.describe Geo::ContainerRepositorySyncDispatchWorker, :geo, :use_sql_query_
include ::EE::GeoHelpers include ::EE::GeoHelpers
include ExclusiveLeaseHelpers include ExclusiveLeaseHelpers
let(:primary) { create(:geo_node, :primary) } let_it_be(:primary) { create(:geo_node, :primary) }
let(:secondary) { create(:geo_node) } let_it_be(:secondary) { create(:geo_node) }
before do before do
stub_current_geo_node(secondary) stub_current_geo_node(secondary)
...@@ -49,149 +49,60 @@ RSpec.describe Geo::ContainerRepositorySyncDispatchWorker, :geo, :use_sql_query_ ...@@ -49,149 +49,60 @@ RSpec.describe Geo::ContainerRepositorySyncDispatchWorker, :geo, :use_sql_query_
expect(Geo::ContainerRepositorySyncWorker).not_to receive(:perform_async) expect(Geo::ContainerRepositorySyncWorker).not_to receive(:perform_async)
end end
context 'when geo_container_registry_ssot_sync is disabled', :geo_fdw do it 'performs Geo::ContainerRepositorySyncWorker' do
before do registry = create(:container_repository_registry)
stub_feature_flags(geo_container_registry_ssot_sync: false)
end
it 'performs Geo::ContainerRepositorySyncWorker' do
container_repository = create(:container_repository)
expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async).with(container_repository.id)
subject.perform
end
it 'performs Geo::ContainerRepositorySyncWorker for failed syncs' do
registry = create(:container_repository_registry, :sync_failed)
expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async)
.with(registry.container_repository_id).once.and_return(spy)
subject.perform
end
it 'does not perform Geo::ContainerRepositorySyncWorker for synced repositories' do
create(:container_repository_registry, :synced)
expect(Geo::ContainerRepositorySyncWorker).not_to receive(:perform_async)
subject.perform
end
context 'with a failed sync' do
it 'does not stall backfill' do
failed_registry = create(:container_repository_registry, :sync_failed)
unsynced_container_repository = create(:container_repository)
stub_const('Geo::Scheduler::SchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 1)
expect(Geo::ContainerRepositorySyncWorker).not_to receive(:perform_async).with(failed_registry.container_repository_id)
expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async).with(unsynced_container_repository.id)
subject.perform
end
it 'does not retry failed files when retry_at is tomorrow' do
failed_registry = create(:container_repository_registry, :sync_failed, retry_at: Date.tomorrow)
expect(Geo::ContainerRepositorySyncWorker)
.not_to receive(:perform_async).with( failed_registry.container_repository_id)
subject.perform expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async).with(registry.container_repository_id)
end
it 'retries failed files when retry_at is in the past' do subject.perform
failed_registry = create(:container_repository_registry, :sync_failed, retry_at: Date.yesterday) end
expect(Geo::ContainerRepositorySyncWorker)
.to receive(:perform_async).with(failed_registry.container_repository_id)
subject.perform it 'performs Geo::ContainerRepositorySyncWorker for failed syncs' do
end registry = create(:container_repository_registry, :sync_failed)
end
context 'when node has namespace restrictions', :request_store do expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async)
let(:synced_group) { create(:group) } .with(registry.container_repository_id).once.and_return(spy)
let(:project_in_synced_group) { create(:project, group: synced_group) }
let(:unsynced_project) { create(:project) }
before do subject.perform
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) end
end
it 'does not perform Geo::ContainerRepositorySyncWorker for repositories that does not belong to selected namespaces' do it 'does not perform Geo::ContainerRepositorySyncWorker for synced repositories' do
container_repository = create(:container_repository, project: project_in_synced_group) create(:container_repository_registry, :synced)
create(:container_repository, project: unsynced_project)
expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async) expect(Geo::ContainerRepositorySyncWorker).not_to receive(:perform_async)
.with(container_repository.id).once.and_return(spy)
subject.perform subject.perform
end
end
end end
context 'when geo_container_registry_ssot_sync is enabled' do context 'with a failed sync' do
before do it 'does not stall backfill' do
stub_feature_flags(geo_container_registry_ssot_sync: true) failed_registry = create(:container_repository_registry, :sync_failed)
end unsynced_registry = create(:container_repository_registry)
it 'performs Geo::ContainerRepositorySyncWorker' do stub_const('Geo::Scheduler::SchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 1)
registry = create(:container_repository_registry)
expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async).with(registry.container_repository_id) expect(Geo::ContainerRepositorySyncWorker).not_to receive(:perform_async).with(failed_registry.container_repository_id)
expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async).with(unsynced_registry.container_repository_id)
subject.perform subject.perform
end end
it 'performs Geo::ContainerRepositorySyncWorker for failed syncs' do it 'does not retry failed files when retry_at is tomorrow' do
registry = create(:container_repository_registry, :sync_failed) failed_registry = create(:container_repository_registry, :sync_failed, retry_at: Date.tomorrow)
expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async) expect(Geo::ContainerRepositorySyncWorker)
.with(registry.container_repository_id).once.and_return(spy) .not_to receive(:perform_async).with( failed_registry.container_repository_id)
subject.perform subject.perform
end end
it 'does not perform Geo::ContainerRepositorySyncWorker for synced repositories' do it 'retries failed files when retry_at is in the past' do
create(:container_repository_registry, :synced) failed_registry = create(:container_repository_registry, :sync_failed, retry_at: Date.yesterday)
expect(Geo::ContainerRepositorySyncWorker).not_to receive(:perform_async) expect(Geo::ContainerRepositorySyncWorker)
.to receive(:perform_async).with(failed_registry.container_repository_id)
subject.perform subject.perform
end end
context 'with a failed sync' do
it 'does not stall backfill' do
failed_registry = create(:container_repository_registry, :sync_failed)
unsynced_registry = create(:container_repository_registry)
stub_const('Geo::Scheduler::SchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 1)
expect(Geo::ContainerRepositorySyncWorker).not_to receive(:perform_async).with(failed_registry.container_repository_id)
expect(Geo::ContainerRepositorySyncWorker).to receive(:perform_async).with(unsynced_registry.container_repository_id)
subject.perform
end
it 'does not retry failed files when retry_at is tomorrow' do
failed_registry = create(:container_repository_registry, :sync_failed, retry_at: Date.tomorrow)
expect(Geo::ContainerRepositorySyncWorker)
.not_to receive(:perform_async).with( failed_registry.container_repository_id)
subject.perform
end
it 'retries failed files when retry_at is in the past' do
failed_registry = create(:container_repository_registry, :sync_failed, retry_at: Date.yesterday)
expect(Geo::ContainerRepositorySyncWorker)
.to receive(:perform_async).with(failed_registry.container_repository_id)
subject.perform
end
end
end end
end end
ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb
View file @ ec247475
...@@ -108,30 +108,6 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do ...@@ -108,30 +108,6 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(1) expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(1)
end end
context 'when geo_container_registry_ssot_sync is disabled' do
before do
stub_feature_flags(geo_container_registry_ssot_sync: false)
end
it 'returns false' do
expect(subject.perform).to be_falsey
end
it 'does not execute RegistryConsistencyService for container repositories' do
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::JobArtifactRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::LfsObjectRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::PackageFileRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::UploadRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::ProjectRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::DesignRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::VulnerabilityExportRegistry, batch_size: batch_size).and_call_original
expect(Geo::RegistryConsistencyService).not_to receive(:new).with(Geo::ContainerRepositoryRegistry, batch_size: batch_size)
subject.perform
end
end
context 'when the current Geo node is disabled or primary' do context 'when the current Geo node is disabled or primary' do
before do before do
stub_primary_node stub_primary_node
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7