Skip to content

G
gitlab-ce
Commit e7021c96 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre Committed by Mike Kozono
Browse files
Options

Default FDW to false on Geo::LfsObjectRegistryFinder

parent 59d52d9a
Show whitespace changes
Inline Side-by-side
Showing with 189 additions and 505 deletions
ee/app/finders/geo/lfs_object_registry_finder.rb
View file @ e7021c96
...@@ -2,10 +2,8 @@ ...@@ -2,10 +2,8 @@
module Geo module Geo
class LfsObjectRegistryFinder < FileRegistryFinder class LfsObjectRegistryFinder < FileRegistryFinder
# Counts all existing registries independent
# of any change on filters / selective sync
def count_registry def count_registry
Geo::LfsObjectRegistry.count syncable.count
end end
def count_syncable def count_syncable
...@@ -13,22 +11,19 @@ module Geo ...@@ -13,22 +11,19 @@ module Geo
end end
def count_synced def count_synced
lfs_objects.synced.count syncable.synced.count
end end
def count_failed def count_failed
lfs_objects.failed.count syncable.failed.count
end end
def count_synced_missing_on_primary def count_synced_missing_on_primary
lfs_objects.synced.missing_on_primary.count syncable.synced.missing_on_primary.count
end end
def syncable def syncable
return lfs_objects if selective_sync? Geo::LfsObjectRegistry
return LfsObject.with_files_stored_locally if local_storage_only?
LfsObject
end end
# Returns untracked IDs as well as tracked IDs that are unused. # Returns untracked IDs as well as tracked IDs that are unused.
...@@ -49,14 +44,8 @@ module Geo ...@@ -49,14 +44,8 @@ module Geo
# #
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused # @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def find_registry_differences(range) def find_registry_differences(range)
source_ids = source_ids = lfs_objects..id_in(range).pluck_primary_key
lfs_objects(fdw: false) tracked_ids = syncable.pluck_model_ids_in_range(range)
.id_in(range)
.pluck_primary_key
tracked_ids =
Geo::LfsObjectRegistry
.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids untracked_ids = source_ids - tracked_ids
unused_tracked_ids = tracked_ids - source_ids unused_tracked_ids = tracked_ids - source_ids
...@@ -82,30 +71,12 @@ module Geo ...@@ -82,30 +71,12 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query # @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: []) def find_never_synced_registries(batch_size:, except_ids: [])
Geo::LfsObjectRegistry syncable
.never .never
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
.limit(batch_size) .limit(batch_size)
end end
# rubocop:enable CodeReuse/ActiveRecord alias_method :find_unsynced, :find_never_synced_registries
# Deprecated in favor of the process using
# #find_registry_differences and #find_never_synced_registries
#
# Find limited amount of non replicated lfs objects.
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_ids: [])
lfs_objects
.missing_registry
.id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
...@@ -120,9 +91,9 @@ module Geo ...@@ -120,9 +91,9 @@ module Geo
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: []) def find_retryable_failed_registries(batch_size:, except_ids: [])
registries_for_lfs_objects syncable
.merge(Geo::LfsObjectRegistry.failed) .failed
.merge(Geo::LfsObjectRegistry.retry_due) .retry_due
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
.limit(batch_size) .limit(batch_size)
end end
...@@ -130,7 +101,7 @@ module Geo ...@@ -130,7 +101,7 @@ module Geo
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: []) def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
registries_for_lfs_objects syncable
.synced .synced
.missing_on_primary .missing_on_primary
.retry_due .retry_due
...@@ -141,16 +112,12 @@ module Geo ...@@ -141,16 +112,12 @@ module Geo
private private
def lfs_objects(fdw: true) def lfs_objects
local_storage_only?(fdw: fdw) ? all_lfs_objects(fdw: fdw).with_files_stored_locally : all_lfs_objects(fdw: fdw) local_storage_only?(fdw: false) ? all_lfs_objects.with_files_stored_locally : all_lfs_objects
end
def all_lfs_objects(fdw: true)
current_node(fdw: fdw).lfs_objects
end end
def registries_for_lfs_objects def all_lfs_objects
current_node.lfs_object_registries current_node(fdw: false).lfs_objects
end end
end end
end end
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
View file @ e7021c96
...@@ -35,241 +35,77 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do ...@@ -35,241 +35,77 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do
context 'counts all the things' do context 'counts all the things' do
describe '#count_syncable' do describe '#count_syncable' do
before do it 'counts registries for LFS objects' do
allow_next_instance_of(LfsObjectsProject) do |instance| create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
allow(instance).to receive(:update_project_statistics).and_return(nil) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
end create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 8 expect(subject.count_syncable).to eq 8
end end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 2
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 2
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects ignoring remote objects' do
expect(subject.count_syncable).to eq 5
end
end
end end
describe '#count_registry' do describe '#count_registry' do
before do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id)
create(:geo_upload_registry, :avatar)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_1)
end
it 'counts registries for LFS objects' do it 'counts registries for LFS objects' do
expect(subject.count_registry).to eq 4 create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
end create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
context 'with selective sync by namespace' do create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
it 'does not apply the selective sync restriction' do create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
expect(subject.count_registry).to eq 4 create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'does not apply the selective sync restriction' do expect(subject.count_registry).to eq 8
expect(subject.count_registry).to eq 4
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts registries for LFS objects' do
expect(subject.count_registry).to eq 4
end
end end
end end
describe '#count_synced' do describe '#count_synced' do
before do it 'counts registries that has been synced' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
allow_next_instance_of(LfsObjectsProject) do |instance| create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 4
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
end
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 1
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that has been synced ignoring remote objects' do
expect(subject.count_synced).to eq 3
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that has been synced, ignoring remotes' do
expect(subject.count_synced).to eq 3 expect(subject.count_synced).to eq 3
end end
end end
end
describe '#count_failed' do describe '#count_failed' do
before do it 'counts registries that sync has failed' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_5.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 4
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 1
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that sync has failed, ignoring remotes' do
expect(subject.count_failed).to eq 3 expect(subject.count_failed).to eq 3
end end
end end
end
describe '#count_synced_missing_on_primary' do describe '#count_synced_missing_on_primary' do
before do it 'counts registries that have been synced and are missing on the primary, excluding not synced ones' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id, missing_on_primary: true) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id, missing_on_primary: true) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
allow_next_instance_of(LfsObjectsProject) do |instance| create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
allow(instance).to receive(:update_project_statistics).and_return(nil) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
end create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
end
it 'counts LFS objects that have been synced and are missing on the primary, excluding not synced ones' do
expect(subject.count_synced_missing_on_primary).to eq 2
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'counts LFS objects that has been synced' do
expect(subject.count_synced_missing_on_primary).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that have been synced and are missing on the primary, excluding not synced ones' do expect(subject.count_synced_missing_on_primary).to eq 3
expect(subject.count_synced_missing_on_primary).to eq 1
end
end end
end end
end end
...@@ -444,283 +280,177 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do ...@@ -444,283 +280,177 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do
end end
describe '#find_never_synced_registries' do describe '#find_never_synced_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_1.id) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_2.id) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id) }
let!(:registry_lfs_object_remote_1) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_1.id) }
it 'returns registries for LFS objects that have never been synced' do it 'returns registries for LFS objects that have never been synced' do
registries = subject.find_never_synced_registries(batch_size: 10) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_2, registry_lfs_object_remote_1) registry_lfs_object_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
end create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
describe '#find_unsynced' do create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
before do registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'returns LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_2, lfs_object_5,
lfs_object_remote_1, lfs_object_remote_2, lfs_object_remote_3)
end
it 'excludes LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_2.id])
expect(lfs_objects).to match_ids(lfs_object_5, lfs_object_remote_1,
lfs_object_remote_2, lfs_object_remote_3)
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'returns LFS objects without an entry on the tracking database' do registries = subject.find_never_synced_registries(batch_size: 10)
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_2) expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_remote_3)
end
end end
context 'with selective sync by shard' do it 'excludes except_ids' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'counts LFS objects that sync has failed' do registries = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_3.id])
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_5) expect(registries).to match_ids(registry_lfs_object_remote_3)
end end
end end
context 'with object storage sync disabled' do describe '#find_unsynced' do
let(:secondary) { create(:geo_node, :local_storage_only) } it 'returns registries for LFS objects that have never been synced' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
registry_lfs_object_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'returns LFS objects without an entry on the tracking database' do registries = subject.find_unsynced(batch_size: 10)
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_2, lfs_object_5) expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_remote_3)
end
end
end end
describe '#find_migrated_local' do it 'excludes except_ids' do
it 'returns LFS objects remotely and successfully synced locally' do create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
lfs_objects = subject.find_migrated_local(batch_size: 10) registries = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_3.id])
expect(lfs_objects).to match_ids(lfs_object_remote_1) expect(registries).to match_ids(registry_lfs_object_remote_3)
end end
it 'excludes LFS objects stored remotely, but not synced yet' do
create(:lfs_object, :object_storage)
lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to be_empty
end end
it 'excludes synced LFS objects that are stored locally' do # describe '#find_migrated_local' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id) # it 'returns LFS objects remotely and successfully synced locally' do
# create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
lfs_objects = subject.find_migrated_local(batch_size: 10) # lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to be_empty # expect(lfs_objects).to match_ids(lfs_object_remote_1)
end # end
it 'excludes except_ids' do # it 'excludes LFS objects stored remotely, but not synced yet' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id) # create(:lfs_object, :object_storage)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id)
lfs_objects = subject.find_migrated_local(batch_size: 10, except_ids: [lfs_object_remote_1.id]) # lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_remote_2) # expect(lfs_objects).to be_empty
end # end
context 'with selective sync by namespace' do # it 'excludes synced LFS objects that are stored locally' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } # create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
before do # lfs_objects = subject.find_migrated_local(batch_size: 10)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_1) # expect(lfs_objects).to be_empty
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_remote_2) # end
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_remote_3)
end
it 'returns LFS objects remotely and successfully synced locally' do # it 'excludes except_ids' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id) # create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_3.id) # create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id)
lfs_objects = subject.find_migrated_local(batch_size: 10) # lfs_objects = subject.find_migrated_local(batch_size: 10, except_ids: [lfs_object_remote_1.id])
expect(lfs_objects).to match_ids(lfs_object_remote_2) # expect(lfs_objects).to match_ids(lfs_object_remote_2)
end # end
end
context 'with selective sync by shard' do # context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } # let(:secondary) { create(:geo_node, :local_storage_only) }
before do # it 'returns LFS objects remotely and successfully synced locally' do
allow_next_instance_of(LfsObjectsProject) do |instance| # create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_1) # lfs_objects = subject.find_migrated_local(batch_size: 10)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_2)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_3)
end
it 'returns LFS objects remotely and successfully synced locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id)
lfs_objects = subject.find_migrated_local(batch_size: 10) # expect(lfs_objects).to match_ids(lfs_object_remote_1)
# end
expect(lfs_objects).to match_ids(lfs_object_remote_2) # end
end # end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'returns LFS objects remotely and successfully synced locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_remote_1)
end
end
end
describe '#find_retryable_failed_registries' do describe '#find_retryable_failed_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_2.id) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_remote_1) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id) }
it 'returns registries for LFS objects that have failed to sync' do it 'returns registries for LFS objects that have failed to sync' do
registries = subject.find_retryable_failed_registries(batch_size: 10) registry_lfs_object_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_2, registry_lfs_object_remote_1) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end registry_lfs_object_4 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
context 'with selective sync by namespace' do registry_lfs_object_remote_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_4)
end
it 'returns registries for LFS objects that have failed to sync' do
registries = subject.find_retryable_failed_registries(batch_size: 10) registries = subject.find_retryable_failed_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_2) expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_4, registry_lfs_object_remote_1)
end
end end
context 'with selective sync by shard' do it 'excludes except_ids' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } registry_lfs_object_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
before do create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
allow_next_instance_of(LfsObjectsProject) do |instance| create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
allow(instance).to receive(:update_project_statistics).and_return(nil) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
end registry_lfs_object_remote_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_1) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
end
it 'returns registries for LFS objects that have failed to sync' do registries = subject.find_retryable_failed_registries(batch_size: 10, except_ids: [lfs_object_4.id])
registries = subject.find_retryable_failed_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_1) expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_remote_1)
end
end end
end end
describe '#find_retryable_synced_missing_on_primary_registries' do describe '#find_retryable_synced_missing_on_primary_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id, missing_on_primary: true, retry_at: nil) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id, missing_on_primary: true, retry_at: 1.day.ago) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id, missing_on_primary: true, retry_at: nil) }
let!(:registry_lfs_object_5) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: false) }
it 'returns registries for LFS objects that have been synced and are missing on the primary' do it 'returns registries for LFS objects that have been synced and are missing on the primary' do
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
registry_lfs_object_2 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_3, registry_lfs_object_4) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
registry_lfs_object_5 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
context 'with selective sync by namespace' do create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
end
it 'returns registries for LFS objects that have been synced and are missing on the primary' do
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10) registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_1) expect(registries).to match_ids(registry_lfs_object_2, registry_lfs_object_5)
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1) it 'excludes except_ids' do
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_3) registry_lfs_object_2 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'returns registries for LFS objects that have been synced and are missing on the primary' do registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10, except_ids: [lfs_object_5.id])
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_4) expect(registries).to match_ids(registry_lfs_object_2)
end
end end
end end
end end
......
ee/spec/models/geo_node_status_spec.rb
View file @ e7021c96
...@@ -244,9 +244,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do ...@@ -244,9 +244,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed) create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar) create(:geo_upload_registry, :avatar)
create(:geo_upload_registry, file_type: :attachment) create(:geo_upload_registry, file_type: :attachment)
create(:geo_lfs_object_registry, :with_lfs_object, :failed) create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :with_lfs_object) create(:geo_lfs_object_registry)
expect(subject.lfs_objects_synced_count).to eq(1) expect(subject.lfs_objects_synced_count).to eq(1)
end end
...@@ -258,9 +258,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do ...@@ -258,9 +258,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed) create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar, missing_on_primary: true) create(:geo_upload_registry, :avatar, missing_on_primary: true)
create(:geo_upload_registry, file_type: :attachment, missing_on_primary: true) create(:geo_upload_registry, file_type: :attachment, missing_on_primary: true)
create(:geo_lfs_object_registry, :with_lfs_object, :failed) create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :with_lfs_object, missing_on_primary: true) create(:geo_lfs_object_registry, missing_on_primary: true)
expect(subject.lfs_objects_synced_missing_on_primary_count).to eq(1) expect(subject.lfs_objects_synced_missing_on_primary_count).to eq(1)
end end
...@@ -272,40 +272,27 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do ...@@ -272,40 +272,27 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed) create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar, :failed) create(:geo_upload_registry, :avatar, :failed)
create(:geo_upload_registry, :failed, file_type: :attachment) create(:geo_upload_registry, :failed, file_type: :attachment)
create(:geo_lfs_object_registry, :with_lfs_object) create(:geo_lfs_object_registry)
create(:geo_lfs_object_registry, :with_lfs_object, :failed) create(:geo_lfs_object_registry, :failed)
expect(subject.lfs_objects_failed_count).to eq(1) expect(subject.lfs_objects_failed_count).to eq(1)
end end
end end
describe '#lfs_objects_synced_in_percentage' do describe '#lfs_objects_synced_in_percentage' do
let(:lfs_object_project) { create(:lfs_objects_project, project: project_1) } it 'returns 0 when there are no registries' do
before do
allow(ProjectCacheWorker).to receive(:perform_async).and_return(true)
create(:lfs_objects_project, project: project_1)
create_list(:lfs_objects_project, 2, project: project_3)
end
it 'returns 0 when no objects are available' do
expect(subject.lfs_objects_synced_in_percentage).to eq(0) expect(subject.lfs_objects_synced_in_percentage).to eq(0)
end end
it 'returns the right percentage with no group restrictions' do it 'returns the right percentage' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_project.lfs_object_id) create(:geo_lfs_object_registry)
create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :never_synced)
create(:geo_lfs_object_registry, :never_synced)
expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(25) expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(25)
end end
it 'returns the right percentage with group restrictions' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_project.lfs_object_id)
expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(50)
end
end end
describe '#job_artifacts_synced_count' do describe '#job_artifacts_synced_count' do
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7