Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
ccddf64f
Commit
ccddf64f
authored
Dec 01, 2017
by
Stan Hu
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Revert "Geo: Prepare for shard-specific repository sync worker"
This reverts commit
83b31c0e
.
parent
83b31c0e
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
48 additions
and
63 deletions
+48
-63
app/workers/geo/repository_shard_sync_worker.rb
app/workers/geo/repository_shard_sync_worker.rb
+0
-54
app/workers/geo/repository_sync_worker.rb
app/workers/geo/repository_sync_worker.rb
+46
-7
spec/workers/geo/repository_sync_worker_spec.rb
spec/workers/geo/repository_sync_worker_spec.rb
+2
-2
No files found.
app/workers/geo/repository_shard_sync_worker.rb
deleted
100644 → 0
View file @
83b31c0e
module
Geo
class
RepositoryShardSyncWorker
<
Geo
::
BaseSchedulerWorker
attr_accessor
:shard_name
def
initialize
(
shard_name
)
@shard_name
=
shard_name
super
()
end
private
def
max_capacity
current_node
.
repos_max_capacity
end
def
schedule_job
(
project_id
)
job_id
=
Geo
::
ProjectSyncWorker
.
perform_async
(
project_id
,
Time
.
now
)
{
id:
project_id
,
job_id:
job_id
}
if
job_id
end
def
finder
@finder
||=
ProjectRegistryFinder
.
new
(
current_node:
current_node
)
end
def
load_pending_resources
resources
=
find_project_ids_not_synced
(
batch_size:
db_retrieve_batch_size
)
remaining_capacity
=
db_retrieve_batch_size
-
resources
.
size
if
remaining_capacity
.
zero?
resources
else
resources
+
find_project_ids_updated_recently
(
batch_size:
remaining_capacity
)
end
end
def
find_project_ids_not_synced
(
batch_size
:)
shard_restriction
(
finder
.
find_unsynced_projects
(
batch_size:
batch_size
))
.
reorder
(
last_repository_updated_at: :desc
)
.
pluck
(
:id
)
end
def
find_project_ids_updated_recently
(
batch_size
:)
shard_restriction
(
finder
.
find_projects_updated_recently
(
batch_size:
batch_size
))
.
order
(
Gitlab
::
Database
.
nulls_first_order
(
:last_repository_updated_at
,
:desc
))
.
pluck
(
:id
)
end
def
shard_restriction
(
relation
)
relation
.
where
(
repository_storage:
shard_name
)
end
end
end
app/workers/geo/repository_sync_worker.rb
View file @
ccddf64f
module
Geo
module
Geo
class
RepositorySyncWorker
class
RepositorySyncWorker
<
Geo
::
BaseSchedulerWorker
def
perform
private
return
unless
Gitlab
::
Geo
.
geo_database_configured?
return
unless
Gitlab
::
Geo
.
secondary?
shards
=
healthy_shards
def
max_capacity
current_node
.
repos_max_capacity
end
def
schedule_job
(
project_id
)
job_id
=
Geo
::
ProjectSyncWorker
.
perform_async
(
project_id
,
Time
.
now
)
{
id:
project_id
,
job_id:
job_id
}
if
job_id
end
shards
.
each
do
|
shard_name
|
def
finder
RepositoryShardSyncWorker
.
perform_async
(
shard_name
,
Time
.
now
)
@finder
||=
ProjectRegistryFinder
.
new
(
current_node:
current_node
)
end
def
load_pending_resources
resources
=
find_project_ids_not_synced
(
batch_size:
db_retrieve_batch_size
)
remaining_capacity
=
db_retrieve_batch_size
-
resources
.
size
if
remaining_capacity
.
zero?
resources
else
resources
+
find_project_ids_updated_recently
(
batch_size:
remaining_capacity
)
end
end
end
def
find_project_ids_not_synced
(
batch_size
:)
healthy_shards_restriction
(
finder
.
find_unsynced_projects
(
batch_size:
batch_size
))
.
reorder
(
last_repository_updated_at: :desc
)
.
pluck
(
:id
)
end
def
find_project_ids_updated_recently
(
batch_size
:)
healthy_shards_restriction
(
finder
.
find_projects_updated_recently
(
batch_size:
batch_size
))
.
order
(
Gitlab
::
Database
.
nulls_first_order
(
:last_repository_updated_at
,
:desc
))
.
pluck
(
:id
)
end
def
healthy_shards_restriction
(
relation
)
configured
=
Gitlab
.
config
.
repositories
.
storages
.
keys
referenced
=
Project
.
distinct
(
:repository_storage
).
pluck
(
:repository_storage
)
healthy
=
healthy_shards
known
=
configured
|
referenced
return
relation
if
(
known
-
healthy
).
empty?
relation
.
where
(
repository_storage:
healthy
)
end
end
def
healthy_shards
def
healthy_shards
...
...
spec/workers/geo/repository_s
hard_s
ync_worker_spec.rb
→
spec/workers/geo/repository_sync_worker_spec.rb
View file @
ccddf64f
...
@@ -2,7 +2,7 @@ require 'spec_helper'
...
@@ -2,7 +2,7 @@ require 'spec_helper'
# Disable transactions via :truncate method because a foreign table
# Disable transactions via :truncate method because a foreign table
# can't see changes inside a transaction of a different connection.
# can't see changes inside a transaction of a different connection.
describe
Geo
::
RepositoryS
hardS
yncWorker
,
:geo
,
:truncate
do
describe
Geo
::
RepositorySyncWorker
,
:geo
,
:truncate
do
include
::
EE
::
GeoHelpers
include
::
EE
::
GeoHelpers
let!
(
:primary
)
{
create
(
:geo_node
,
:primary
)
}
let!
(
:primary
)
{
create
(
:geo_node
,
:primary
)
}
...
@@ -11,7 +11,7 @@ describe Geo::RepositoryShardSyncWorker, :geo, :truncate do
...
@@ -11,7 +11,7 @@ describe Geo::RepositoryShardSyncWorker, :geo, :truncate do
let!
(
:project_in_synced_group
)
{
create
(
:project
,
group:
synced_group
)
}
let!
(
:project_in_synced_group
)
{
create
(
:project
,
group:
synced_group
)
}
let!
(
:unsynced_project
)
{
create
(
:project
)
}
let!
(
:unsynced_project
)
{
create
(
:project
)
}
subject
{
described_class
.
new
(
Gitlab
.
config
.
repositories
.
storages
.
keys
.
first
)
}
subject
{
described_class
.
new
}
before
do
before
do
stub_current_geo_node
(
secondary
)
stub_current_geo_node
(
secondary
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment