Default FDW to false on Geo::JobArtifactRegistryFinder

The feature flag have that disable the FDW queries
has been enabled by default and will be removed.
parent 2675f3e0
...@@ -2,10 +2,8 @@ ...@@ -2,10 +2,8 @@
module Geo module Geo
class JobArtifactRegistryFinder < FileRegistryFinder class JobArtifactRegistryFinder < FileRegistryFinder
# Counts all existing registries independent
# of any change on filters / selective sync
def count_registry def count_registry
Geo::JobArtifactRegistry.count syncable.count
end end
def count_syncable def count_syncable
...@@ -13,22 +11,19 @@ module Geo ...@@ -13,22 +11,19 @@ module Geo
end end
def count_synced def count_synced
registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.synced).count syncable.synced.count
end end
def count_failed def count_failed
registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.failed).count syncable.failed.count
end end
def count_synced_missing_on_primary def count_synced_missing_on_primary
registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.synced.missing_on_primary).count syncable.synced.missing_on_primary.count
end end
def syncable def syncable
return job_artifacts.not_expired if selective_sync? Geo::JobArtifactRegistry
return Ci::JobArtifact.not_expired.with_files_stored_locally if local_storage_only?
Ci::JobArtifact.not_expired
end end
# Returns untracked IDs as well as tracked IDs that are unused. # Returns untracked IDs as well as tracked IDs that are unused.
...@@ -49,16 +44,8 @@ module Geo ...@@ -49,16 +44,8 @@ module Geo
# #
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused # @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def find_registry_differences(range) def find_registry_differences(range)
# rubocop:disable CodeReuse/ActiveRecord source_ids = job_artifacts.id_in(range).pluck(::Ci::JobArtifact.arel_table[:id]) # rubocop:disable CodeReuse/ActiveRecord
source_ids = tracked_ids = syncable.pluck_model_ids_in_range(range)
job_artifacts(fdw: false)
.id_in(range)
.pluck(::Ci::JobArtifact.arel_table[:id])
# rubocop:enable CodeReuse/ActiveRecord
tracked_ids =
Geo::JobArtifactRegistry
.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids untracked_ids = source_ids - tracked_ids
unused_tracked_ids = tracked_ids - source_ids unused_tracked_ids = tracked_ids - source_ids
...@@ -84,49 +71,27 @@ module Geo ...@@ -84,49 +71,27 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query # @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: []) def find_never_synced_registries(batch_size:, except_ids: [])
Geo::JobArtifactRegistry syncable
.never .never
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
.limit(batch_size) .limit(batch_size)
end end
alias_method :find_unsynced, :find_never_synced_registries
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
# Deprecated in favor of the process using
# #find_registry_differences and #find_never_synced_registries
#
# Find limited amount of non replicated job artifacts.
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# TODO: Alternative here is to use some sort of window function with a cursor instead
# of simply limiting the query and passing a list of items we don't want
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop: disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_ids: [])
job_artifacts
.not_expired
.missing_job_artifact_registry
.id_not_in(except_ids)
.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local(batch_size:, except_ids: []) def find_migrated_local(batch_size:, except_ids: [])
all_job_artifacts # all_job_artifacts
.inner_join_job_artifact_registry # .inner_join_job_artifact_registry
.with_files_stored_remotely # .with_files_stored_remotely
.id_not_in(except_ids) # .id_not_in(except_ids)
.limit(batch_size) # .limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: []) def find_retryable_failed_registries(batch_size:, except_ids: [])
Geo::JobArtifactRegistry syncable
.failed .failed
.retry_due .retry_due
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
...@@ -136,7 +101,7 @@ module Geo ...@@ -136,7 +101,7 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: []) def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
Geo::JobArtifactRegistry syncable
.synced .synced
.missing_on_primary .missing_on_primary
.retry_due .retry_due
...@@ -147,18 +112,12 @@ module Geo ...@@ -147,18 +112,12 @@ module Geo
private private
def job_artifacts(fdw: true) def job_artifacts
local_storage_only?(fdw: fdw) ? all_job_artifacts(fdw: fdw).with_files_stored_locally : all_job_artifacts(fdw: fdw) local_storage_only?(fdw: false) ? all_job_artifacts.with_files_stored_locally : all_job_artifacts
end
def all_job_artifacts(fdw: true)
current_node(fdw: fdw).job_artifacts
end end
def registries_for_job_artifacts def all_job_artifacts
job_artifacts current_node(fdw: false).job_artifacts
.inner_join_job_artifact_registry
.not_expired
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment