Skip to content

G
gitlab-ce
Commit e7021c96 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre Committed by Mike Kozono
Browse files
Options

Default FDW to false on Geo::LfsObjectRegistryFinder

parent 59d52d9a
Hide whitespace changes
Inline Side-by-side
Showing with 189 additions and 505 deletions
ee/app/finders/geo/lfs_object_registry_finder.rb
View file @ e7021c96
......@@ -2,10 +2,8 @@
module Geo
class LfsObjectRegistryFinder < FileRegistryFinder
# Counts all existing registries independent
# of any change on filters / selective sync
def count_registry
Geo::LfsObjectRegistry.count
syncable.count
end
def count_syncable
......@@ -13,22 +11,19 @@ module Geo
end
def count_synced
lfs_objects.synced.count
syncable.synced.count
end
def count_failed
lfs_objects.failed.count
syncable.failed.count
end
def count_synced_missing_on_primary
lfs_objects.synced.missing_on_primary.count
syncable.synced.missing_on_primary.count
end
def syncable
return lfs_objects if selective_sync?
return LfsObject.with_files_stored_locally if local_storage_only?
LfsObject
Geo::LfsObjectRegistry
end
# Returns untracked IDs as well as tracked IDs that are unused.
......@@ -49,14 +44,8 @@ module Geo
#
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def find_registry_differences(range)
source_ids =
lfs_objects(fdw: false)
.id_in(range)
.pluck_primary_key
tracked_ids =
Geo::LfsObjectRegistry
.pluck_model_ids_in_range(range)
source_ids = lfs_objects..id_in(range).pluck_primary_key
tracked_ids = syncable.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids
unused_tracked_ids = tracked_ids - source_ids
......@@ -82,30 +71,12 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: [])
Geo::LfsObjectRegistry
syncable
.never
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
# Deprecated in favor of the process using
# #find_registry_differences and #find_never_synced_registries
#
# Find limited amount of non replicated lfs objects.
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_ids: [])
lfs_objects
.missing_registry
.id_not_in(except_ids)
.limit(batch_size)
end
alias_method :find_unsynced, :find_never_synced_registries
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
......@@ -120,9 +91,9 @@ module Geo
# rubocop:disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: [])
registries_for_lfs_objects
.merge(Geo::LfsObjectRegistry.failed)
.merge(Geo::LfsObjectRegistry.retry_due)
syncable
.failed
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end
......@@ -130,7 +101,7 @@ module Geo
# rubocop:disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
registries_for_lfs_objects
syncable
.synced
.missing_on_primary
.retry_due
......@@ -141,16 +112,12 @@ module Geo
private
def lfs_objects(fdw: true)
local_storage_only?(fdw: fdw) ? all_lfs_objects(fdw: fdw).with_files_stored_locally : all_lfs_objects(fdw: fdw)
end
def all_lfs_objects(fdw: true)
current_node(fdw: fdw).lfs_objects
def lfs_objects
local_storage_only?(fdw: false) ? all_lfs_objects.with_files_stored_locally : all_lfs_objects
end
def registries_for_lfs_objects
current_node.lfs_object_registries
def all_lfs_objects
current_node(fdw: false).lfs_objects
end
end
end
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
View file @ e7021c96
......@@ -35,241 +35,77 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do
context 'counts all the things' do
describe '#count_syncable' do
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'counts registries for LFS objects' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 8
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 2
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 2
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects ignoring remote objects' do
expect(subject.count_syncable).to eq 5
end
end
end
describe '#count_registry' do
before do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id)
create(:geo_upload_registry, :avatar)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_1)
end
it 'counts registries for LFS objects' do
expect(subject.count_registry).to eq 4
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'does not apply the selective sync restriction' do
expect(subject.count_registry).to eq 4
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'does not apply the selective sync restriction' do
expect(subject.count_registry).to eq 4
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'counts registries for LFS objects' do
expect(subject.count_registry).to eq 4
end
expect(subject.count_registry).to eq 8
end
end
describe '#count_synced' do
before do
it 'counts registries that has been synced' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 4
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
end
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 1
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that has been synced ignoring remote objects' do
expect(subject.count_synced).to eq 3
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'counts LFS objects that has been synced, ignoring remotes' do
expect(subject.count_synced).to eq 3
end
expect(subject.count_synced).to eq 3
end
end
describe '#count_failed' do
before do
it 'counts registries that sync has failed' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_5.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 4
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 1
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that sync has failed, ignoring remotes' do
expect(subject.count_failed).to eq 3
end
expect(subject.count_failed).to eq 3
end
end
describe '#count_synced_missing_on_primary' do
before do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
it 'counts registries that have been synced and are missing on the primary, excluding not synced ones' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id, missing_on_primary: true)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id, missing_on_primary: true)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
end
it 'counts LFS objects that have been synced and are missing on the primary, excluding not synced ones' do
expect(subject.count_synced_missing_on_primary).to eq 2
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'counts LFS objects that has been synced' do
expect(subject.count_synced_missing_on_primary).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that have been synced and are missing on the primary, excluding not synced ones' do
expect(subject.count_synced_missing_on_primary).to eq 1
end
expect(subject.count_synced_missing_on_primary).to eq 3
end
end
end
......@@ -444,283 +280,177 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do
end
describe '#find_never_synced_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_1.id) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_2.id) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id) }
let!(:registry_lfs_object_remote_1) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_1.id) }
it 'returns registries for LFS objects that have never been synced' do
registries = subject.find_never_synced_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_2, registry_lfs_object_remote_1)
end
end
describe '#find_unsynced' do
before do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'returns LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_2, lfs_object_5,
lfs_object_remote_1, lfs_object_remote_2, lfs_object_remote_3)
end
it 'excludes LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_2.id])
expect(lfs_objects).to match_ids(lfs_object_5, lfs_object_remote_1,
lfs_object_remote_2, lfs_object_remote_3)
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'returns LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_2)
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
registry_lfs_object_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'counts LFS objects that sync has failed' do
lfs_objects = subject.find_unsynced(batch_size: 10)
registries = subject.find_never_synced_registries(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_5)
end
expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_remote_3)
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'excludes except_ids' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'returns LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10)
registries = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_3.id])
expect(lfs_objects).to match_ids(lfs_object_2, lfs_object_5)
end
expect(registries).to match_ids(registry_lfs_object_remote_3)
end
end
describe '#find_migrated_local' do
it 'returns LFS objects remotely and successfully synced locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_remote_1)
end
it 'excludes LFS objects stored remotely, but not synced yet' do
create(:lfs_object, :object_storage)
lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to be_empty
end
it 'excludes synced LFS objects that are stored locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
describe '#find_unsynced' do
it 'returns registries for LFS objects that have never been synced' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
registry_lfs_object_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
lfs_objects = subject.find_migrated_local(batch_size: 10)
registries = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to be_empty
expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_remote_3)
end
it 'excludes except_ids' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
lfs_objects = subject.find_migrated_local(batch_size: 10, except_ids: [lfs_object_remote_1.id])
registries = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_3.id])
expect(lfs_objects).to match_ids(lfs_object_remote_2)
expect(registries).to match_ids(registry_lfs_object_remote_3)
end
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
# describe '#find_migrated_local' do
# it 'returns LFS objects remotely and successfully synced locally' do
# create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
# lfs_objects = subject.find_migrated_local(batch_size: 10)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_remote_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_remote_3)
end
# expect(lfs_objects).to match_ids(lfs_object_remote_1)
# end
it 'returns LFS objects remotely and successfully synced locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_3.id)
# it 'excludes LFS objects stored remotely, but not synced yet' do
# create(:lfs_object, :object_storage)
lfs_objects = subject.find_migrated_local(batch_size: 10)
# lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_remote_2)
end
end
# expect(lfs_objects).to be_empty
# end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
# it 'excludes synced LFS objects that are stored locally' do
# create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
# lfs_objects = subject.find_migrated_local(batch_size: 10)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_1)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_2)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_3)
end
# expect(lfs_objects).to be_empty
# end
it 'returns LFS objects remotely and successfully synced locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id)
# it 'excludes except_ids' do
# create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
# create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id)
lfs_objects = subject.find_migrated_local(batch_size: 10)
# lfs_objects = subject.find_migrated_local(batch_size: 10, except_ids: [lfs_object_remote_1.id])
expect(lfs_objects).to match_ids(lfs_object_remote_2)
end
end
# expect(lfs_objects).to match_ids(lfs_object_remote_2)
# end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
# context 'with object storage sync disabled' do
# let(:secondary) { create(:geo_node, :local_storage_only) }
it 'returns LFS objects remotely and successfully synced locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
# it 'returns LFS objects remotely and successfully synced locally' do
# create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
lfs_objects = subject.find_migrated_local(batch_size: 10)
# lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_remote_1)
end
end
end
# expect(lfs_objects).to match_ids(lfs_object_remote_1)
# end
# end
# end
describe '#find_retryable_failed_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_2.id) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_remote_1) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id) }
it 'returns registries for LFS objects that have failed to sync' do
registries = subject.find_retryable_failed_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_2, registry_lfs_object_remote_1)
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_4)
end
registry_lfs_object_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
registry_lfs_object_4 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
registry_lfs_object_remote_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'returns registries for LFS objects that have failed to sync' do
registries = subject.find_retryable_failed_registries(batch_size: 10)
registries = subject.find_retryable_failed_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_2)
end
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_4, registry_lfs_object_remote_1)
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
end
it 'excludes except_ids' do
registry_lfs_object_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
registry_lfs_object_remote_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'returns registries for LFS objects that have failed to sync' do
registries = subject.find_retryable_failed_registries(batch_size: 10)
registries = subject.find_retryable_failed_registries(batch_size: 10, except_ids: [lfs_object_4.id])
expect(registries).to match_ids(registry_lfs_object_1)
end
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_remote_1)
end
end
describe '#find_retryable_synced_missing_on_primary_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id, missing_on_primary: true, retry_at: nil) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id, missing_on_primary: true, retry_at: 1.day.ago) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id, missing_on_primary: true, retry_at: nil) }
let!(:registry_lfs_object_5) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: false) }
it 'returns registries for LFS objects that have been synced and are missing on the primary' do
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_3, registry_lfs_object_4)
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
end
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
registry_lfs_object_2 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
registry_lfs_object_5 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'returns registries for LFS objects that have been synced and are missing on the primary' do
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_1)
end
expect(registries).to match_ids(registry_lfs_object_2, registry_lfs_object_5)
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
end
it 'excludes except_ids' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
registry_lfs_object_2 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'returns registries for LFS objects that have been synced and are missing on the primary' do
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10, except_ids: [lfs_object_5.id])
expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_4)
end
expect(registries).to match_ids(registry_lfs_object_2)
end
end
end
......
ee/spec/models/geo_node_status_spec.rb
View file @ e7021c96
......@@ -244,9 +244,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar)
create(:geo_upload_registry, file_type: :attachment)
create(:geo_lfs_object_registry, :with_lfs_object, :failed)
create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :with_lfs_object)
create(:geo_lfs_object_registry)
expect(subject.lfs_objects_synced_count).to eq(1)
end
......@@ -258,9 +258,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar, missing_on_primary: true)
create(:geo_upload_registry, file_type: :attachment, missing_on_primary: true)
create(:geo_lfs_object_registry, :with_lfs_object, :failed)
create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :with_lfs_object, missing_on_primary: true)
create(:geo_lfs_object_registry, missing_on_primary: true)
expect(subject.lfs_objects_synced_missing_on_primary_count).to eq(1)
end
......@@ -272,40 +272,27 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar, :failed)
create(:geo_upload_registry, :failed, file_type: :attachment)
create(:geo_lfs_object_registry, :with_lfs_object)
create(:geo_lfs_object_registry)
create(:geo_lfs_object_registry, :with_lfs_object, :failed)
create(:geo_lfs_object_registry, :failed)
expect(subject.lfs_objects_failed_count).to eq(1)
end
end
describe '#lfs_objects_synced_in_percentage' do
let(:lfs_object_project) { create(:lfs_objects_project, project: project_1) }
before do
allow(ProjectCacheWorker).to receive(:perform_async).and_return(true)
create(:lfs_objects_project, project: project_1)
create_list(:lfs_objects_project, 2, project: project_3)
end
it 'returns 0 when no objects are available' do
it 'returns 0 when there are no registries' do
expect(subject.lfs_objects_synced_in_percentage).to eq(0)
end
it 'returns the right percentage with no group restrictions' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_project.lfs_object_id)
it 'returns the right percentage' do
create(:geo_lfs_object_registry)
create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :never_synced)
create(:geo_lfs_object_registry, :never_synced)
expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(25)
end
it 'returns the right percentage with group restrictions' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_project.lfs_object_id)
expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(50)
end
end
describe '#job_artifacts_synced_count' do
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7