Add postgres_fdw support to find projects updated recently

parent 118ac9f1
...@@ -19,6 +19,19 @@ module Geo ...@@ -19,6 +19,19 @@ module Geo
relation.limit(batch_size) relation.limit(batch_size)
end end
def find_projects_updated_recently(batch_size:)
# Selective project replication adds a wrinkle to FDW queries, so
# we fallback to the legacy version for now.
relation =
if Gitlab::Geo.fdw? && !selective_sync
fdw_find_projects_updated_recently
else
legacy_find_projects_updated_recently
end
relation.limit(batch_size)
end
protected protected
def selective_sync def selective_sync
...@@ -32,7 +45,16 @@ module Geo ...@@ -32,7 +45,16 @@ module Geo
def fdw_find_unsynced_projects def fdw_find_unsynced_projects
fdw_table = Geo::Fdw::Project.table_name fdw_table = Geo::Fdw::Project.table_name
Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id").where('project_registry.project_id IS NULL') Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
.where('project_registry.project_id IS NULL')
end
def fdw_find_projects_updated_recently
fdw_table = Geo::Fdw::Project.table_name
Geo::Fdw::Project.joins("INNER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
.merge(Geo::ProjectRegistry.dirty)
.merge(Geo::ProjectRegistry.retry_due)
end end
# #
...@@ -52,5 +74,19 @@ module Geo ...@@ -52,5 +74,19 @@ module Geo
joined_relation.where(project_registry: { registry_present: [nil, false] }) joined_relation.where(project_registry: { registry_present: [nil, false] })
end end
def legacy_find_projects_updated_recently
registry_project_ids = current_node.project_registries.dirty.retry_due.pluck(:project_id)
return Project.none if registry_project_ids.empty?
joined_relation = current_node.projects.joins(<<~SQL)
INNER JOIN
(VALUES #{registry_project_ids.map { |id| "(#{id})" }.join(',')})
project_registry(project_id)
ON projects.id = project_registry.project_id
SQL
joined_relation
end
end end
end end
...@@ -34,12 +34,9 @@ module Geo ...@@ -34,12 +34,9 @@ module Geo
end end
def find_project_ids_updated_recently(batch_size:) def find_project_ids_updated_recently(batch_size:)
current_node.project_registries healthy_shards_restriction(finder.find_projects_updated_recently(batch_size: batch_size))
.dirty
.retry_due
.order(Gitlab::Database.nulls_first_order(:last_repository_synced_at, :desc)) .order(Gitlab::Database.nulls_first_order(:last_repository_synced_at, :desc))
.limit(batch_size) .pluck(:id)
.pluck(:project_id)
end end
def healthy_shards_restriction(relation) def healthy_shards_restriction(relation)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment