Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
5f784e2a
Commit
5f784e2a
authored
Sep 04, 2019
by
Douglas Barbosa Alexandre
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Extract a helper method to schedule jobs
parent
54ec4cae
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
58 additions
and
54 deletions
+58
-54
ee/app/workers/concerns/geo/secondary/backfill_worker.rb
ee/app/workers/concerns/geo/secondary/backfill_worker.rb
+16
-0
ee/app/workers/geo/secondary/repository_backfill_worker.rb
ee/app/workers/geo/secondary/repository_backfill_worker.rb
+42
-54
No files found.
ee/app/workers/concerns/geo/secondary/backfill_worker.rb
View file @
5f784e2a
...
@@ -30,6 +30,22 @@ module Geo
...
@@ -30,6 +30,22 @@ module Geo
end
end
end
end
def
initialize
@scheduled_jobs
=
[]
@loops
=
0
end
def
perform
(
shard_name
)
@shard_name
=
shard_name
@start_time
=
Time
.
now
.
utc
return
unless
healthy_node?
try_obtain_lease
do
schedule_jobs
end
end
private
private
def
base_log_data
(
message
)
def
base_log_data
(
message
)
...
...
ee/app/workers/geo/secondary/repository_backfill_worker.rb
View file @
5f784e2a
...
@@ -5,56 +5,6 @@ module Geo
...
@@ -5,56 +5,6 @@ module Geo
class
RepositoryBackfillWorker
class
RepositoryBackfillWorker
include
Geo
::
Secondary
::
BackfillWorker
include
Geo
::
Secondary
::
BackfillWorker
def
initialize
@scheduled_jobs
=
[]
end
def
perform
(
shard_name
)
@shard_name
=
shard_name
@start_time
=
Time
.
now
.
utc
@loops
=
0
return
unless
healthy_node?
try_obtain_lease
do
log_info
(
'Repository backfilling started'
)
reason
=
:unknown
begin
connection
.
send_query
(
"
#{
projects_ids_unsynced
.
to_sql
}
;
#{
project_ids_updated_recently
.
to_sql
}
"
)
connection
.
set_single_row_mode
reason
=
loop
do
break
:node_disabled
unless
node_enabled?
break
:over_time
if
over_time?
break
:lease_lost
unless
renew_lease!
update_jobs_in_progress
unless
over_capacity?
# This will stream the results one by one
# until there are no more results to fetch.
result
=
connection
.
get_result
break
:complete
if
result
.
nil?
result
.
check
result
.
each
do
|
row
|
schedule_job
(
row
[
'id'
])
end
else
sleep
(
1
)
end
end
rescue
=>
error
reason
=
:error
log_error
(
'Repository backfilling error'
,
error
)
raise
error
ensure
log_info
(
'Repository backfilling finished'
,
total_loops:
loops
,
duration:
Time
.
now
.
utc
-
start_time
,
reason:
reason
)
end
end
end
private
private
attr_reader
:scheduled_jobs
attr_reader
:scheduled_jobs
...
@@ -83,6 +33,44 @@ module Geo
...
@@ -83,6 +33,44 @@ module Geo
scheduled_jobs
.
size
>=
max_capacity
scheduled_jobs
.
size
>=
max_capacity
end
end
def
schedule_jobs
log_info
(
'Repository backfilling started'
)
reason
=
:unknown
begin
connection
.
send_query
(
"
#{
projects_ids_unsynced
.
to_sql
}
;
#{
project_ids_updated_recently
.
to_sql
}
"
)
connection
.
set_single_row_mode
reason
=
loop
do
break
:node_disabled
unless
node_enabled?
break
:over_time
if
over_time?
break
:lease_lost
unless
renew_lease!
update_jobs_in_progress
unless
over_capacity?
# This will stream the results one by one
# until there are no more results to fetch.
result
=
connection
.
get_result
break
:complete
if
result
.
nil?
result
.
check
result
.
each
do
|
row
|
schedule_job
(
row
[
'id'
])
end
else
sleep
(
1
)
end
end
rescue
=>
error
reason
=
:error
log_error
(
'Repository backfilling error'
,
error
)
raise
error
ensure
log_info
(
'Repository backfilling finished'
,
total_loops:
loops
,
duration:
Time
.
now
.
utc
-
start_time
,
reason:
reason
)
end
end
def
schedule_job
(
project_id
)
def
schedule_job
(
project_id
)
job_id
=
Geo
::
ProjectSyncWorker
.
perform_async
(
project_id
,
sync_repository:
true
,
sync_wiki:
true
)
job_id
=
Geo
::
ProjectSyncWorker
.
perform_async
(
project_id
,
sync_repository:
true
,
sync_wiki:
true
)
...
@@ -94,6 +82,10 @@ module Geo
...
@@ -94,6 +82,10 @@ module Geo
end
end
end
end
def
scheduled_job_ids
scheduled_jobs
.
map
{
|
data
|
data
[
:job_id
]
}
end
def
update_jobs_in_progress
def
update_jobs_in_progress
job_ids
=
scheduled_job_ids
job_ids
=
scheduled_job_ids
return
if
job_ids
.
empty?
return
if
job_ids
.
empty?
...
@@ -106,10 +98,6 @@ module Geo
...
@@ -106,10 +98,6 @@ module Geo
end
end
end
end
def
scheduled_job_ids
scheduled_jobs
.
map
{
|
data
|
data
[
:job_id
]
}
end
# rubocop: disable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def
projects_ids_unsynced
def
projects_ids_unsynced
Geo
::
ProjectUnsyncedFinder
Geo
::
ProjectUnsyncedFinder
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment