Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
e0622d3f
Commit
e0622d3f
authored
Oct 18, 2017
by
Stan Hu
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Make FDW Geo download code DRY
parent
4f418a1c
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
29 additions
and
30 deletions
+29
-30
app/models/geo/tracking_base.rb
app/models/geo/tracking_base.rb
+1
-1
app/workers/geo/file_download_dispatch_worker.rb
app/workers/geo/file_download_dispatch_worker.rb
+28
-29
No files found.
app/models/geo/tracking_base.rb
View file @
e0622d3f
# This module is intended to centralize all database access to the secondary
# tracking database for Geo.
# tracking database for Geo.
module
Geo
class
TrackingBase
<
ActiveRecord
::
Base
self
.
abstract_class
=
true
...
...
app/workers/geo/file_download_dispatch_worker.rb
View file @
e0622d3f
...
...
@@ -33,24 +33,40 @@ module Geo
.
pluck
(
:file_id
,
:file_type
)
end
def
selective_sync
current_node
.
restricted_project_ids
end
def
find_lfs_object_ids
# Selective project replication adds a wrinkle to FDW queries, so
# we fallback to the legacy version for now.
if
Gitlab
::
Geo
.
fdw?
&&
!
current_node
.
restricted_project_ids
fdw_find_lfs_object_ids
else
legacy_find_lfs_object_ids
end
relation
=
if
Gitlab
::
Geo
.
fdw?
&&
!
selective_sync
fdw_find_lfs_object_ids
else
legacy_find_lfs_object_ids
end
relation
.
limit
(
db_retrieve_batch_size
)
.
pluck
(
:id
)
.
map
{
|
id
|
[
id
,
:lfs
]
}
end
def
find_upload_object_ids
# Selective project replication adds a wrinkle to FDW queries, so
# we fallback to the legacy version for now.
if
Gitlab
::
Geo
.
fdw?
&&
!
current_node
.
restricted_project_ids
fdw_find_upload_object_ids
else
legacy_find_upload_object_ids
end
relation
=
if
Gitlab
::
Geo
.
fdw?
&&
!
selective_sync
fdw_find_upload_object_ids
else
legacy_find_upload_object_ids
end
relation
.
limit
(
db_retrieve_batch_size
)
.
pluck
(
:id
,
:uploader
)
.
map
{
|
id
,
uploader
|
[
id
,
uploader
.
sub
(
/Uploader\z/
,
''
).
underscore
]
}
end
def
fdw_find_lfs_object_ids
...
...
@@ -60,10 +76,6 @@ module Geo
Geo
::
Fdw
::
LfsObject
.
joins
(
"LEFT OUTER JOIN file_registry ON file_registry.file_id =
#{
fdw_table
}
.id AND file_registry.file_type = 'lfs'"
)
.
where
(
"
#{
fdw_table
}
.file_store IS NULL OR
#{
fdw_table
}
.file_store =
#{
LfsObjectUploader
::
LOCAL_STORE
}
"
)
.
where
(
'file_registry.file_id IS NULL'
)
.
order
(
created_at: :desc
)
.
limit
(
db_retrieve_batch_size
)
.
pluck
(
:id
)
.
map
{
|
id
|
[
id
,
:lfs
]
}
end
def
fdw_find_upload_object_ids
...
...
@@ -73,35 +85,22 @@ module Geo
Geo
::
Fdw
::
Upload
.
joins
(
"LEFT OUTER JOIN file_registry ON file_registry.file_id =
#{
fdw_table
}
.id AND file_registry.file_type IN (
#{
obj_types
}
)"
)
.
where
(
'file_registry.file_id IS NULL'
)
.
order
(
created_at: :desc
)
.
limit
(
db_retrieve_batch_size
)
.
pluck
(
:id
,
:uploader
)
.
map
{
|
id
,
uploader
|
[
id
,
uploader
.
sub
(
/Uploader\z/
,
''
).
underscore
]
}
end
def
legacy_find_upload_object_ids
unsynced_downloads
=
legacy_filter_registry_ids
(
legacy_filter_registry_ids
(
current_node
.
uploads
,
Geo
::
FileService
::
DEFAULT_OBJECT_TYPES
,
Upload
.
table_name
)
unsynced_downloads
.
limit
(
db_retrieve_batch_size
)
.
pluck
(
:id
,
:uploader
)
.
map
{
|
id
,
uploader
|
[
id
,
uploader
.
sub
(
/Uploader\z/
,
''
).
underscore
]
}
end
def
legacy_find_lfs_object_ids
unsynced_downloads
=
legacy_filter_registry_ids
(
legacy_filter_registry_ids
(
current_node
.
lfs_objects
,
[
:lfs
],
LfsObject
.
table_name
)
unsynced_downloads
.
limit
(
db_retrieve_batch_size
)
.
pluck
(
:id
)
.
map
{
|
id
|
[
id
,
:lfs
]
}
end
# This query requires data from two different databases, and unavoidably
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment