Commit e0622d3f authored by Stan Hu's avatar Stan Hu

Make FDW Geo download code DRY

parent 4f418a1c
# This module is intended to centralize all database access to the secondary
# tracking database for Geo.
# tracking database for Geo.
module Geo
class TrackingBase < ActiveRecord::Base
self.abstract_class = true
......
......@@ -33,24 +33,40 @@ module Geo
.pluck(:file_id, :file_type)
end
def selective_sync
current_node.restricted_project_ids
end
def find_lfs_object_ids
# Selective project replication adds a wrinkle to FDW queries, so
# we fallback to the legacy version for now.
if Gitlab::Geo.fdw? && !current_node.restricted_project_ids
fdw_find_lfs_object_ids
else
legacy_find_lfs_object_ids
end
relation =
if Gitlab::Geo.fdw? && !selective_sync
fdw_find_lfs_object_ids
else
legacy_find_lfs_object_ids
end
relation
.limit(db_retrieve_batch_size)
.pluck(:id)
.map { |id| [id, :lfs] }
end
def find_upload_object_ids
# Selective project replication adds a wrinkle to FDW queries, so
# we fallback to the legacy version for now.
if Gitlab::Geo.fdw? && !current_node.restricted_project_ids
fdw_find_upload_object_ids
else
legacy_find_upload_object_ids
end
relation =
if Gitlab::Geo.fdw? && !selective_sync
fdw_find_upload_object_ids
else
legacy_find_upload_object_ids
end
relation
.limit(db_retrieve_batch_size)
.pluck(:id, :uploader)
.map { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] }
end
def fdw_find_lfs_object_ids
......@@ -60,10 +76,6 @@ module Geo
Geo::Fdw::LfsObject.joins("LEFT OUTER JOIN file_registry ON file_registry.file_id = #{fdw_table}.id AND file_registry.file_type = 'lfs'")
.where("#{fdw_table}.file_store IS NULL OR #{fdw_table}.file_store = #{LfsObjectUploader::LOCAL_STORE}")
.where('file_registry.file_id IS NULL')
.order(created_at: :desc)
.limit(db_retrieve_batch_size)
.pluck(:id)
.map { |id| [id, :lfs] }
end
def fdw_find_upload_object_ids
......@@ -73,35 +85,22 @@ module Geo
Geo::Fdw::Upload.joins("LEFT OUTER JOIN file_registry ON file_registry.file_id = #{fdw_table}.id AND file_registry.file_type IN (#{obj_types})")
.where('file_registry.file_id IS NULL')
.order(created_at: :desc)
.limit(db_retrieve_batch_size)
.pluck(:id, :uploader)
.map { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] }
end
def legacy_find_upload_object_ids
unsynced_downloads = legacy_filter_registry_ids(
legacy_filter_registry_ids(
current_node.uploads,
Geo::FileService::DEFAULT_OBJECT_TYPES,
Upload.table_name
)
unsynced_downloads
.limit(db_retrieve_batch_size)
.pluck(:id, :uploader)
.map { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] }
end
def legacy_find_lfs_object_ids
unsynced_downloads = legacy_filter_registry_ids(
legacy_filter_registry_ids(
current_node.lfs_objects,
[:lfs],
LfsObject.table_name
)
unsynced_downloads
.limit(db_retrieve_batch_size)
.pluck(:id)
.map { |id| [id, :lfs] }
end
# This query requires data from two different databases, and unavoidably
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment