Rename finder methods to reveal its intent

These changes are an attempt to reconcile
these methods that have different names
but sahre the same intent under the hood.
parent 20f6caf7
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
module Geo module Geo
class ProjectRegistryFinder class ProjectRegistryFinder
# Returns ProjectRegistry records that have never been synced. # Returns ProjectRegistry records that have never have an attempt tp sync.
# #
# Does not care about selective sync, because it considers the Registry # Does not care about selective sync, because it considers the Registry
# table to be the single source of truth. The contract is that other # table to be the single source of truth. The contract is that other
...@@ -11,7 +11,7 @@ module Geo ...@@ -11,7 +11,7 @@ module Geo
# #
# Any registries that have ever been synced that currently need to be # Any registries that have ever been synced that currently need to be
# resynced will be handled by other find methods (like # resynced will be handled by other find methods (like
# #find_failed_registries) # #find_registries_needs_sync_again)
# #
# You can pass a list with `except_ids:` so you can exclude items you # You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet # already scheduled but haven't finished and aren't persisted to the database yet
...@@ -19,16 +19,16 @@ module Geo ...@@ -19,16 +19,16 @@ module Geo
# @param [Integer] batch_size used to limit the results returned # @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query # @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_unsynced_registries(batch_size:, except_ids: []) def find_registries_never_attempted_sync(batch_size:, except_ids: [])
registry_class registry_class
.find_unsynced_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_never_attempted_sync(batch_size: batch_size, except_ids: except_ids)
end end
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_failed_registries(batch_size:, except_ids: []) def find_registries_needs_sync_again(batch_size:, except_ids: [])
registry_class registry_class
.find_failed_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_needs_sync_again(batch_size: batch_size, except_ids: except_ids)
end end
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
......
...@@ -2,9 +2,9 @@ ...@@ -2,9 +2,9 @@
module Geo module Geo
class RegistryFinder class RegistryFinder
# @!method find_unsynced_registries # @!method find_registries_never_attempted_sync
# Return an ActiveRecord::Relation of the registry records for the # Return an ActiveRecord::Relation of the registry records for the
# tracked ype that have never been synced. # tracked ype that have never have an attempt to sync.
# #
# Does not care about selective sync, because it considers the Registry # Does not care about selective sync, because it considers the Registry
# table to be the single source of truth. The contract is that other # table to be the single source of truth. The contract is that other
...@@ -13,7 +13,7 @@ module Geo ...@@ -13,7 +13,7 @@ module Geo
# #
# Any registries that have ever been synced that currently need to be # Any registries that have ever been synced that currently need to be
# resynced will be handled by other find methods (like # resynced will be handled by other find methods (like
# #find_failed_registries) # #find_registries_needs_sync_again)
# #
# You can pass a list with `except_ids:` so you can exclude items you # You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet # already scheduled but haven't finished and aren't persisted to the database yet
...@@ -22,13 +22,13 @@ module Geo ...@@ -22,13 +22,13 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query # @param [Array<Integer>] except_ids ids that will be ignored from the query
# #
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_unsynced_registries(batch_size:, except_ids: []) def find_registries_never_attempted_sync(batch_size:, except_ids: [])
registry_class registry_class
.find_unsynced_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_never_attempted_sync(batch_size: batch_size, except_ids: except_ids)
end end
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
# @!method find_failed_registries # @!method find_registries_needs_sync_again
# Return an ActiveRecord::Relation of registry records marked as failed, # Return an ActiveRecord::Relation of registry records marked as failed,
# which are ready to be retried, excluding specified IDs, limited to # which are ready to be retried, excluding specified IDs, limited to
# batch_size # batch_size
...@@ -37,9 +37,9 @@ module Geo ...@@ -37,9 +37,9 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query # @param [Array<Integer>] except_ids ids that will be ignored from the query
# #
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_failed_registries(batch_size:, except_ids: []) def find_registries_needs_sync_again(batch_size:, except_ids: [])
registry_class registry_class
.find_failed_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_needs_sync_again(batch_size: batch_size, except_ids: except_ids)
end end
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
......
...@@ -35,13 +35,13 @@ module Geo::ReplicableRegistry ...@@ -35,13 +35,13 @@ module Geo::ReplicableRegistry
included do included do
include ::Delay include ::Delay
scope :never, -> { where(last_synced_at: nil) }
scope :failed, -> { with_state(:failed) } scope :failed, -> { with_state(:failed) }
scope :synced, -> { with_state(:synced) } scope :needs_sync_again, -> { failed.retry_due }
scope :never_attempted_sync, -> { with_state(:pending).where(last_synced_at: nil) }
scope :ordered, -> { order(:id) }
scope :pending, -> { with_state(:pending) } scope :pending, -> { with_state(:pending) }
scope :retry_due, -> { where(arel_table[:retry_at].eq(nil).or(arel_table[:retry_at].lt(Time.current))) } scope :retry_due, -> { where(arel_table[:retry_at].eq(nil).or(arel_table[:retry_at].lt(Time.current))) }
scope :retryable, -> { failed.retry_due } scope :synced, -> { with_state(:synced) }
scope :ordered, -> { order(:id) }
state_machine :state, initial: :pending do state_machine :state, initial: :pending do
state :pending, value: STATE_VALUES[:pending] state :pending, value: STATE_VALUES[:pending]
......
...@@ -6,9 +6,9 @@ module Geo::Syncable ...@@ -6,9 +6,9 @@ module Geo::Syncable
included do included do
scope :failed, -> { where(success: false).where.not(retry_count: nil) } scope :failed, -> { where(success: false).where.not(retry_count: nil) }
scope :missing_on_primary, -> { where(missing_on_primary: true) } scope :missing_on_primary, -> { where(missing_on_primary: true) }
scope :pending, -> { where(success: false, retry_count: nil) } scope :needs_sync_again, -> { failed.retry_due }
scope :never_attempted_sync, -> { where(success: false, retry_count: nil) }
scope :retry_due, -> { where('retry_at is NULL OR retry_at < ?', Time.current) } scope :retry_due, -> { where('retry_at is NULL OR retry_at < ?', Time.current) }
scope :retryable, -> { failed.retry_due }
scope :synced, -> { where(success: true) } scope :synced, -> { where(success: true) }
end end
end end
...@@ -63,14 +63,14 @@ class Geo::BaseRegistry < Geo::TrackingBase ...@@ -63,14 +63,14 @@ class Geo::BaseRegistry < Geo::TrackingBase
[untracked_ids, unused_tracked_ids] [untracked_ids, unused_tracked_ids]
end end
def self.find_unsynced_registries(batch_size:, except_ids: []) def self.find_registries_never_attempted_sync(batch_size:, except_ids: [])
pending never_attempted_sync
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
.limit(batch_size) .limit(batch_size)
end end
def self.find_failed_registries(batch_size:, except_ids: []) def self.find_registries_needs_sync_again(batch_size:, except_ids: [])
retryable needs_sync_again
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
.limit(batch_size) .limit(batch_size)
end end
......
...@@ -9,9 +9,9 @@ class Geo::ContainerRepositoryRegistry < Geo::BaseRegistry ...@@ -9,9 +9,9 @@ class Geo::ContainerRepositoryRegistry < Geo::BaseRegistry
belongs_to :container_repository belongs_to :container_repository
scope :failed, -> { with_state(:failed) } scope :failed, -> { with_state(:failed) }
scope :pending, -> { with_state(:pending).where(last_synced_at: nil) } scope :needs_sync_again, -> { failed.retry_due }
scope :never_attempted_sync, -> { with_state(:pending).where(last_synced_at: nil) }
scope :retry_due, -> { where(arel_table[:retry_at].eq(nil).or(arel_table[:retry_at].lt(Time.current))) } scope :retry_due, -> { where(arel_table[:retry_at].eq(nil).or(arel_table[:retry_at].lt(Time.current))) }
scope :retryable, -> { failed.retry_due }
scope :synced, -> { with_state(:synced) } scope :synced, -> { with_state(:synced) }
state_machine :state, initial: :pending do state_machine :state, initial: :pending do
...@@ -38,9 +38,8 @@ class Geo::ContainerRepositoryRegistry < Geo::BaseRegistry ...@@ -38,9 +38,8 @@ class Geo::ContainerRepositoryRegistry < Geo::BaseRegistry
end end
end end
def self.find_failed_registries(batch_size:, except_ids: []) def self.find_registries_needs_sync_again(batch_size:, except_ids: [])
super super.order(Gitlab::Database.nulls_first_order(:last_synced_at))
.order(Gitlab::Database.nulls_first_order(:last_synced_at))
end end
def self.delete_for_model_ids(container_repository_ids) def self.delete_for_model_ids(container_repository_ids)
......
...@@ -10,12 +10,12 @@ class Geo::DesignRegistry < Geo::BaseRegistry ...@@ -10,12 +10,12 @@ class Geo::DesignRegistry < Geo::BaseRegistry
belongs_to :project belongs_to :project
scope :never, -> { with_state(:pending).where(last_synced_at: nil) }
scope :pending, -> { with_state(:pending) }
scope :failed, -> { with_state(:failed) } scope :failed, -> { with_state(:failed) }
scope :synced, -> { with_state(:synced) } scope :needs_sync_again, -> { failed.retry_due }
scope :never_attempted_sync, -> { with_state(:pending).where(last_synced_at: nil) }
scope :pending, -> { with_state(:pending) }
scope :retry_due, -> { where(arel_table[:retry_at].eq(nil).or(arel_table[:retry_at].lt(Time.current))) } scope :retry_due, -> { where(arel_table[:retry_at].eq(nil).or(arel_table[:retry_at].lt(Time.current))) }
scope :retryable, -> { failed.retry_due } scope :synced, -> { with_state(:synced) }
state_machine :state, initial: :pending do state_machine :state, initial: :pending do
state :started state :started
...@@ -61,15 +61,8 @@ class Geo::DesignRegistry < Geo::BaseRegistry ...@@ -61,15 +61,8 @@ class Geo::DesignRegistry < Geo::BaseRegistry
[untracked_ids, unused_tracked_ids] [untracked_ids, unused_tracked_ids]
end end
def self.find_unsynced_registries(batch_size:, except_ids: []) def self.find_registries_needs_sync_again(batch_size:, except_ids: [])
never super.order(Gitlab::Database.nulls_first_order(:last_synced_at))
.model_id_not_in(except_ids)
.limit(batch_size)
end
def self.find_failed_registries(batch_size:, except_ids: [])
super
.order(Gitlab::Database.nulls_first_order(:last_synced_at))
end end
# Search for a list of projects associated with registries, # Search for a list of projects associated with registries,
......
...@@ -22,8 +22,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry ...@@ -22,8 +22,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
validates :project, presence: true, uniqueness: true validates :project, presence: true, uniqueness: true
scope :dirty, -> { where(arel_table[:resync_repository].eq(true).or(arel_table[:resync_wiki].eq(true))) } scope :dirty, -> { where(arel_table[:resync_repository].eq(true).or(arel_table[:resync_wiki].eq(true))) }
scope :retryable, -> { dirty.retry_due } scope :needs_sync_again, -> { dirty.retry_due }
scope :pending, -> { where(last_repository_synced_at: nil) } scope :never_attempted_sync, -> { where(last_repository_synced_at: nil) }
scope :synced_repos, -> { where(resync_repository: false) } scope :synced_repos, -> { where(resync_repository: false) }
scope :synced_wikis, -> { where(resync_wiki: false) } scope :synced_wikis, -> { where(resync_wiki: false) }
scope :failed_repos, -> { where(arel_table[:repository_retry_count].gt(0)) } scope :failed_repos, -> { where(arel_table[:repository_retry_count].gt(0)) }
...@@ -44,9 +44,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry ...@@ -44,9 +44,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
where(nil).pluck(:project_id) where(nil).pluck(:project_id)
end end
def self.find_failed_registries(batch_size:, except_ids: []) def self.find_registries_needs_sync_again(batch_size:, except_ids: [])
super super.order(Gitlab::Database.nulls_first_order(:last_repository_synced_at))
.order(Gitlab::Database.nulls_first_order(:last_repository_synced_at))
end end
def self.delete_worker_class def self.delete_worker_class
......
...@@ -77,8 +77,10 @@ class Geo::UploadRegistry < Geo::BaseRegistry ...@@ -77,8 +77,10 @@ class Geo::UploadRegistry < Geo::BaseRegistry
def self.with_status(status) def self.with_status(status)
case status case status
when 'synced', 'failed', 'pending' when 'synced', 'failed'
self.public_send(status) # rubocop: disable GitlabSecurity/PublicSend self.public_send(status) # rubocop: disable GitlabSecurity/PublicSend
when 'pending'
never_attempted_sync
else else
all all
end end
......
...@@ -36,25 +36,25 @@ module Geo ...@@ -36,25 +36,25 @@ module Geo
# #
# @return [Array] resources to be transferred # @return [Array] resources to be transferred
def load_pending_resources def load_pending_resources
resources = find_container_repository_ids_not_synced(batch_size: db_retrieve_batch_size) resources = find_jobs_never_attempted_sync(batch_size: db_retrieve_batch_size)
remaining_capacity = db_retrieve_batch_size - resources.size remaining_capacity = db_retrieve_batch_size - resources.size
if remaining_capacity == 0 if remaining_capacity == 0
resources resources
else else
resources + find_retryable_container_registry_ids(batch_size: remaining_capacity) resources + fin_jobs_needs_sync_again(batch_size: remaining_capacity)
end end
end end
def find_container_repository_ids_not_synced(batch_size:) def find_jobs_never_attempted_sync(batch_size:)
registry_finder registry_finder
.find_unsynced_registries(batch_size: batch_size, except_ids: scheduled_repository_ids) .find_registries_never_attempted_sync(batch_size: batch_size, except_ids: scheduled_repository_ids)
.pluck_model_foreign_key .pluck_model_foreign_key
end end
def find_retryable_container_registry_ids(batch_size:) def fin_jobs_needs_sync_again(batch_size:)
registry_finder registry_finder
.find_failed_registries(batch_size: batch_size, except_ids: scheduled_repository_ids) .find_registries_needs_sync_again(batch_size: batch_size, except_ids: scheduled_repository_ids)
.pluck_model_foreign_key .pluck_model_foreign_key
end end
......
...@@ -11,10 +11,10 @@ module Geo ...@@ -11,10 +11,10 @@ module Geo
end end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_not_synced(except_ids:, batch_size:) def find_jobs_never_attempted_sync(except_ids:, batch_size:)
project_ids = project_ids =
registry_finder registry_finder
.find_unsynced_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_never_attempted_sync(batch_size: batch_size, except_ids: except_ids)
.pluck_model_foreign_key .pluck_model_foreign_key
find_project_ids_within_shard(project_ids, direction: :desc) find_project_ids_within_shard(project_ids, direction: :desc)
...@@ -22,10 +22,10 @@ module Geo ...@@ -22,10 +22,10 @@ module Geo
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_updated_recently(except_ids:, batch_size:) def find_jobs_needs_sync_again(except_ids:, batch_size:)
project_ids = project_ids =
registry_finder registry_finder
.find_failed_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_needs_sync_again(batch_size: batch_size, except_ids: except_ids)
.pluck_model_foreign_key .pluck_model_foreign_key
find_project_ids_within_shard(project_ids, direction: :asc) find_project_ids_within_shard(project_ids, direction: :asc)
......
...@@ -33,7 +33,7 @@ module Geo ...@@ -33,7 +33,7 @@ module Geo
# #
# @return [Array] resources to be transferred # @return [Array] resources to be transferred
def load_pending_resources def load_pending_resources
resources = find_unsynced_jobs(batch_size: db_retrieve_batch_size) resources = find_jobs_never_attempted_sync(batch_size: db_retrieve_batch_size)
remaining_capacity = db_retrieve_batch_size - resources.count remaining_capacity = db_retrieve_batch_size - resources.count
if remaining_capacity == 0 if remaining_capacity == 0
...@@ -43,12 +43,13 @@ module Geo ...@@ -43,12 +43,13 @@ module Geo
end end
end end
# Get a batch of unsynced resources, taking equal parts from each resource. # Get a batch of resources that never have an attempt to sync, taking
# equal parts from each resource.
# #
# @return [Array] job arguments of unsynced resources # @return [Array] job arguments of resources that never have an attempt to sync
def find_unsynced_jobs(batch_size:) def find_jobs_never_attempted_sync(batch_size:)
jobs = job_finders.reduce([]) do |jobs, job_finder| jobs = job_finders.reduce([]) do |jobs, job_finder|
jobs << job_finder.find_unsynced_jobs(batch_size: batch_size) jobs << job_finder.find_jobs_never_attempted_sync(batch_size: batch_size)
end end
take_batch(*jobs, batch_size: batch_size) take_batch(*jobs, batch_size: batch_size)
...@@ -60,8 +61,8 @@ module Geo ...@@ -60,8 +61,8 @@ module Geo
# @return [Array] job arguments of low priority resources # @return [Array] job arguments of low priority resources
def find_low_priority_jobs(batch_size:) def find_low_priority_jobs(batch_size:)
jobs = job_finders.reduce([]) do |jobs, job_finder| jobs = job_finders.reduce([]) do |jobs, job_finder|
jobs << job_finder.find_failed_jobs(batch_size: batch_size) jobs << job_finder.find_jobs_needs_sync_again(batch_size: batch_size)
jobs << job_finder.find_synced_missing_on_primary_jobs(batch_size: batch_size) jobs << job_finder.find_jobs_synced_missing_on_primary(batch_size: batch_size)
end end
take_batch(*jobs, batch_size: batch_size) take_batch(*jobs, batch_size: batch_size)
......
...@@ -21,19 +21,19 @@ module Geo ...@@ -21,19 +21,19 @@ module Geo
@scheduled_file_ids = scheduled_file_ids @scheduled_file_ids = scheduled_file_ids
end end
def find_unsynced_jobs(batch_size:) def find_jobs_never_attempted_sync(batch_size:)
convert_registry_relation_to_job_args( convert_registry_relation_to_job_args(
registry_finder.find_unsynced_registries(find_batch_params(batch_size)) registry_finder.find_registries_never_attempted_sync(find_batch_params(batch_size))
) )
end end
def find_failed_jobs(batch_size:) def find_jobs_needs_sync_again(batch_size:)
convert_registry_relation_to_job_args( convert_registry_relation_to_job_args(
registry_finder.find_failed_registries(find_batch_params(batch_size)) registry_finder.find_registries_needs_sync_again(find_batch_params(batch_size))
) )
end end
def find_synced_missing_on_primary_jobs(batch_size:) def find_jobs_synced_missing_on_primary(batch_size:)
convert_registry_relation_to_job_args( convert_registry_relation_to_job_args(
registry_finder.find_retryable_synced_missing_on_primary_registries(find_batch_params(batch_size)) registry_finder.find_retryable_synced_missing_on_primary_registries(find_batch_params(batch_size))
) )
......
...@@ -32,25 +32,26 @@ module Geo ...@@ -32,25 +32,26 @@ module Geo
# #
# @return [Array] resources to be transferred # @return [Array] resources to be transferred
def load_pending_resources def load_pending_resources
resources = find_unsynced_jobs(batch_size: db_retrieve_batch_size) resources = find_jobs_never_attempted_sync(batch_size: db_retrieve_batch_size)
remaining_capacity = db_retrieve_batch_size - resources.count remaining_capacity = db_retrieve_batch_size - resources.count
if remaining_capacity == 0 if remaining_capacity == 0
resources resources
else else
resources + find_low_priority_jobs(batch_size: remaining_capacity) resources + find_jobs_needs_sync_again(batch_size: remaining_capacity)
end end
end end
# Get a batch of unsynced resources, taking equal parts from each resource. # Get a batch of resources that never have an attempt to sync, taking
# equal parts from each resource.
# #
# @return [Array] job arguments of unsynced resources # @return [Array] job arguments of resources that never have an attempt to sync
def find_unsynced_jobs(batch_size:) def find_jobs_never_attempted_sync(batch_size:)
jobs = replicator_classes.reduce([]) do |jobs, replicator_class| jobs = replicator_classes.reduce([]) do |jobs, replicator_class|
except_ids = scheduled_replicable_ids(replicator_class.replicable_name) except_ids = scheduled_replicable_ids(replicator_class.replicable_name)
jobs << replicator_class jobs << replicator_class
.find_unsynced_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_never_attempted_sync(batch_size: batch_size, except_ids: except_ids)
.map { |registry| [replicator_class.replicable_name, registry.model_record_id] } .map { |registry| [replicator_class.replicable_name, registry.model_record_id] }
end end
...@@ -61,12 +62,12 @@ module Geo ...@@ -61,12 +62,12 @@ module Geo
# equal parts from each resource. # equal parts from each resource.
# #
# @return [Array] job arguments of low priority resources # @return [Array] job arguments of low priority resources
def find_low_priority_jobs(batch_size:) def find_jobs_needs_sync_again(batch_size:)
jobs = replicator_classes.reduce([]) do |jobs, replicator_class| jobs = replicator_classes.reduce([]) do |jobs, replicator_class|
except_ids = scheduled_replicable_ids(replicator_class.replicable_name) except_ids = scheduled_replicable_ids(replicator_class.replicable_name)
jobs << replicator_class jobs << replicator_class
.find_failed_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_needs_sync_again(batch_size: batch_size, except_ids: except_ids)
.map { |registry| [replicator_class.replicable_name, registry.model_record_id] } .map { |registry| [replicator_class.replicable_name, registry.model_record_id] }
end end
......
...@@ -64,21 +64,21 @@ module Geo ...@@ -64,21 +64,21 @@ module Geo
def load_pending_resources def load_pending_resources
return [] unless valid_shard? return [] unless valid_shard?
resources = find_project_ids_not_synced(except_ids: scheduled_project_ids, batch_size: db_retrieve_batch_size) resources = find_jobs_never_attempted_sync(except_ids: scheduled_project_ids, batch_size: db_retrieve_batch_size)
remaining_capacity = db_retrieve_batch_size - resources.size remaining_capacity = db_retrieve_batch_size - resources.size
if remaining_capacity == 0 if remaining_capacity == 0
resources resources
else else
resources + find_project_ids_updated_recently(except_ids: scheduled_project_ids + resources, batch_size: remaining_capacity) resources + find_jobs_needs_sync_again(except_ids: scheduled_project_ids + resources, batch_size: remaining_capacity)
end end
end end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_not_synced(except_ids:, batch_size:) def find_jobs_never_attempted_sync(except_ids:, batch_size:)
project_ids = project_ids =
registry_finder registry_finder
.find_unsynced_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_never_attempted_sync(batch_size: batch_size, except_ids: except_ids)
.pluck_model_foreign_key .pluck_model_foreign_key
find_project_ids_within_shard(project_ids, direction: :desc) find_project_ids_within_shard(project_ids, direction: :desc)
...@@ -86,10 +86,10 @@ module Geo ...@@ -86,10 +86,10 @@ module Geo
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_updated_recently(except_ids:, batch_size:) def find_jobs_needs_sync_again(except_ids:, batch_size:)
project_ids = project_ids =
registry_finder registry_finder
.find_failed_registries(batch_size: batch_size, except_ids: except_ids) .find_registries_needs_sync_again(batch_size: batch_size, except_ids: except_ids)
.pluck_model_foreign_key .pluck_model_foreign_key
find_project_ids_within_shard(project_ids, direction: :asc) find_project_ids_within_shard(project_ids, direction: :asc)
......
...@@ -22,7 +22,7 @@ module Gitlab ...@@ -22,7 +22,7 @@ module Gitlab
delegate :in_replicables_for_geo_node?, to: :model_record delegate :in_replicables_for_geo_node?, to: :model_record
class << self class << self
delegate :find_unsynced_registries, :find_failed_registries, to: :registry_class delegate :find_registries_never_attempted_sync, :find_registries_needs_sync_again, to: :registry_class
end end
# Declare supported event # Declare supported event
......
...@@ -17,29 +17,29 @@ RSpec.describe Geo::ProjectRegistryFinder, :geo do ...@@ -17,29 +17,29 @@ RSpec.describe Geo::ProjectRegistryFinder, :geo do
let_it_be(:registry_project_5) { create(:geo_project_registry, :wiki_dirty, project_id: project_5.id, last_repository_synced_at: 5.days.ago) } let_it_be(:registry_project_5) { create(:geo_project_registry, :wiki_dirty, project_id: project_5.id, last_repository_synced_at: 5.days.ago) }
let_it_be(:registry_project_6) { create(:geo_project_registry, project_id: project_6.id) } let_it_be(:registry_project_6) { create(:geo_project_registry, project_id: project_6.id) }
describe '#find_unsynced_registries' do describe '#find_registries_never_attempted_sync' do
it 'returns registries for projects that have never been synced' do it 'returns registries for projects that have never have an attempt to sync' do
registries = subject.find_unsynced_registries(batch_size: 10) registries = subject.find_registries_never_attempted_sync(batch_size: 10)
expect(registries).to match_ids(registry_project_3, registry_project_6) expect(registries).to match_ids(registry_project_3, registry_project_6)
end end
it 'excludes except_ids' do it 'excludes except_ids' do
registries = subject.find_unsynced_registries(batch_size: 10, except_ids: [project_3.id]) registries = subject.find_registries_never_attempted_sync(batch_size: 10, except_ids: [project_3.id])
expect(registries).to match_ids(registry_project_6) expect(registries).to match_ids(registry_project_6)
end end
end end
describe '#find_failed_registries' do describe '#find_registries_needs_sync_again' do
it 'returns registries for projects that have been recently updated or that have never been synced' do it 'returns registries for dirty projects or that have failed to sync' do
registries = subject.find_failed_registries(batch_size: 10) registries = subject.find_registries_needs_sync_again(batch_size: 10)
expect(registries).to match_ids(registry_project_2, registry_project_3, registry_project_4, registry_project_5, registry_project_6) expect(registries).to match_ids(registry_project_2, registry_project_3, registry_project_4, registry_project_5, registry_project_6)
end end
it 'excludes except_ids' do it 'excludes except_ids' do
registries = subject.find_failed_registries(batch_size: 10, except_ids: [project_4.id, project_5.id, project_6.id]) registries = subject.find_registries_needs_sync_again(batch_size: 10, except_ids: [project_4.id, project_5.id, project_6.id])
expect(registries).to match_ids(registry_project_2, registry_project_3) expect(registries).to match_ids(registry_project_2, registry_project_3)
end end
......
...@@ -92,13 +92,13 @@ RSpec.describe Geo::UploadRegistry, :geo do ...@@ -92,13 +92,13 @@ RSpec.describe Geo::UploadRegistry, :geo do
end end
end end
describe '.pending' do describe '.never_attempted_sync' do
it 'returns registries that are never synced' do it 'returns registries that are never synced' do
create(:geo_upload_registry, :failed) create(:geo_upload_registry, :failed)
create(:geo_upload_registry) create(:geo_upload_registry)
pending = create(:geo_upload_registry, retry_count: nil, success: false) pending = create(:geo_upload_registry, retry_count: nil, success: false)
expect(described_class.pending).to match_ids([pending]) expect(described_class.never_attempted_sync).to match_ids([pending])
end end
end end
...@@ -109,8 +109,8 @@ RSpec.describe Geo::UploadRegistry, :geo do ...@@ -109,8 +109,8 @@ RSpec.describe Geo::UploadRegistry, :geo do
described_class.with_status('synced') described_class.with_status('synced')
end end
it 'finds the registries with status "pending" when filter is set to "pending"' do it 'finds the registries with status "never_attempted_sync" when filter is set to "pending"' do
expect(described_class).to receive(:pending) expect(described_class).to receive(:never_attempted_sync)
described_class.with_status('pending') described_class.with_status('pending')
end end
......
...@@ -4,8 +4,8 @@ RSpec.shared_examples 'a registry finder' do ...@@ -4,8 +4,8 @@ RSpec.shared_examples 'a registry finder' do
it 'responds to registry finder methods' do it 'responds to registry finder methods' do
registry_finder_methods = %i{ registry_finder_methods = %i{
failed_count failed_count
find_failed_registries find_registries_never_attempted_sync
find_unsynced_registries find_registries_needs_sync_again
registry_class registry_class
registry_count registry_count
synced_count synced_count
...@@ -34,29 +34,29 @@ RSpec.shared_examples 'a registry finder' do ...@@ -34,29 +34,29 @@ RSpec.shared_examples 'a registry finder' do
end end
end end
describe '#find_unsynced_registries' do describe '#find_registries_never_attempted_sync' do
it 'returns registries that have never been synced' do it 'returns registries that have never been synced' do
registries = subject.find_unsynced_registries(batch_size: 10) registries = subject.find_registries_never_attempted_sync(batch_size: 10)
expect(registries).to match_ids(registry_3, registry_8) expect(registries).to match_ids(registry_3, registry_8)
end end
it 'excludes except_ids' do it 'excludes except_ids' do
registries = subject.find_unsynced_registries(batch_size: 10, except_ids: [replicable_3.id]) registries = subject.find_registries_never_attempted_sync(batch_size: 10, except_ids: [replicable_3.id])
expect(registries).to match_ids(registry_8) expect(registries).to match_ids(registry_8)
end end
end end
describe '#find_failed_registries' do describe '#find_registries_needs_sync_again' do
it 'returns registries for that have failed to sync' do it 'returns registries for that have failed to sync' do
registries = subject.find_failed_registries(batch_size: 10) registries = subject.find_registries_needs_sync_again(batch_size: 10)
expect(registries).to match_ids(registry_1, registry_4, registry_6, registry_7) expect(registries).to match_ids(registry_1, registry_4, registry_6, registry_7)
end end
it 'excludes except_ids' do it 'excludes except_ids' do
registries = subject.find_failed_registries(batch_size: 10, except_ids: [replicable_4.id, replicable_7.id]) registries = subject.find_registries_needs_sync_again(batch_size: 10, except_ids: [replicable_4.id, replicable_7.id])
expect(registries).to match_ids(registry_1, registry_6) expect(registries).to match_ids(registry_1, registry_6)
end end
......
...@@ -9,26 +9,26 @@ RSpec.shared_examples 'a Geo framework registry' do ...@@ -9,26 +9,26 @@ RSpec.shared_examples 'a Geo framework registry' do
let!(:unsynced_item1) { create(registry_class_factory) } let!(:unsynced_item1) { create(registry_class_factory) }
let!(:unsynced_item2) { create(registry_class_factory) } let!(:unsynced_item2) { create(registry_class_factory) }
describe '.find_unsynced_registries' do describe '.find_registries_never_attempted_sync' do
it 'returns unsynced items' do it 'returns unsynced items' do
result = described_class.find_unsynced_registries(batch_size: 10) result = described_class.find_registries_never_attempted_sync(batch_size: 10)
expect(result).to include(unsynced_item1, unsynced_item2) expect(result).to include(unsynced_item1, unsynced_item2)
end end
it 'returns unsynced items except some specific item ID' do it 'returns items that never have an attempt to sync except some specific item ID' do
except_id = unsynced_item1.model_record_id except_id = unsynced_item1.model_record_id
result = described_class.find_unsynced_registries(batch_size: 10, except_ids: [except_id]) result = described_class.find_registries_never_attempted_sync(batch_size: 10, except_ids: [except_id])
expect(result).to include(unsynced_item2) expect(result).to include(unsynced_item2)
expect(result).not_to include(unsynced_item1) expect(result).not_to include(unsynced_item1)
end end
end end
describe '.find_failed_registries' do describe '.find_registries_needs_sync_again' do
it 'returns failed items' do it 'returns failed items' do
result = described_class.find_failed_registries(batch_size: 10) result = described_class.find_registries_needs_sync_again(batch_size: 10)
expect(result).to include(failed_item1, failed_item2) expect(result).to include(failed_item1, failed_item2)
end end
...@@ -36,7 +36,7 @@ RSpec.shared_examples 'a Geo framework registry' do ...@@ -36,7 +36,7 @@ RSpec.shared_examples 'a Geo framework registry' do
it 'returns failed items except some specific item ID' do it 'returns failed items except some specific item ID' do
except_id = failed_item1.model_record_id except_id = failed_item1.model_record_id
result = described_class.find_failed_registries(batch_size: 10, except_ids: [except_id]) result = described_class.find_registries_needs_sync_again(batch_size: 10, except_ids: [except_id])
expect(result).to include(failed_item2) expect(result).to include(failed_item2)
expect(result).not_to include(failed_item1) expect(result).not_to include(failed_item1)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment