Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
4f55ef53
Commit
4f55ef53
authored
Oct 05, 2021
by
drew cimino
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Use ExclusiveLeaseGuard in Ci::StuckBuilds::DropRunningWorker
parent
f407e01e
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
12 additions
and
65 deletions
+12
-65
app/workers/ci/stuck_builds/drop_running_worker.rb
app/workers/ci/stuck_builds/drop_running_worker.rb
+4
-13
app/workers/stuck_ci_jobs_worker.rb
app/workers/stuck_ci_jobs_worker.rb
+0
-2
spec/workers/ci/stuck_builds/drop_running_worker_spec.rb
spec/workers/ci/stuck_builds/drop_running_worker_spec.rb
+8
-50
No files found.
app/workers/ci/stuck_builds/drop_running_worker.rb
View file @
4f55ef53
...
@@ -4,6 +4,7 @@ module Ci
...
@@ -4,6 +4,7 @@ module Ci
module
StuckBuilds
module
StuckBuilds
class
DropRunningWorker
class
DropRunningWorker
include
ApplicationWorker
include
ApplicationWorker
include
ExclusiveLeaseGuard
idempotent!
idempotent!
...
@@ -17,26 +18,16 @@ module Ci
...
@@ -17,26 +18,16 @@ module Ci
feature_category
:continuous_integration
feature_category
:continuous_integration
EXCLUSIVE_LEASE_KEY
=
'ci_stuck_builds_drop_running_worker_lease'
def
perform
def
perform
return
unless
try_obtain_lease
try_obtain_lease
do
begin
Ci
::
StuckBuilds
::
DropRunningService
.
new
.
execute
Ci
::
StuckBuilds
::
DropRunningService
.
new
.
execute
ensure
remove_lease
end
end
end
end
private
private
def
try_obtain_lease
def
lease_timeout
@uuid
=
Gitlab
::
ExclusiveLease
.
new
(
EXCLUSIVE_LEASE_KEY
,
timeout:
30
.
minutes
).
try_obtain
30
.
minutes
end
def
remove_lease
Gitlab
::
ExclusiveLease
.
cancel
(
EXCLUSIVE_LEASE_KEY
,
@uuid
)
end
end
end
end
end
end
...
...
app/workers/stuck_ci_jobs_worker.rb
View file @
4f55ef53
...
@@ -15,8 +15,6 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
...
@@ -15,8 +15,6 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
feature_category
:continuous_integration
feature_category
:continuous_integration
worker_resource_boundary
:cpu
worker_resource_boundary
:cpu
EXCLUSIVE_LEASE_KEY
=
'stuck_ci_builds_worker_lease'
def
perform
def
perform
Ci
::
StuckBuilds
::
DropRunningWorker
.
perform_in
(
20
.
minutes
)
Ci
::
StuckBuilds
::
DropRunningWorker
.
perform_in
(
20
.
minutes
)
Ci
::
StuckBuilds
::
DropScheduledWorker
.
perform_in
(
40
.
minutes
)
Ci
::
StuckBuilds
::
DropScheduledWorker
.
perform_in
(
40
.
minutes
)
...
...
spec/workers/ci/stuck_builds/drop_running_worker_spec.rb
View file @
4f55ef53
...
@@ -5,66 +5,24 @@ require 'spec_helper'
...
@@ -5,66 +5,24 @@ require 'spec_helper'
RSpec
.
describe
Ci
::
StuckBuilds
::
DropRunningWorker
do
RSpec
.
describe
Ci
::
StuckBuilds
::
DropRunningWorker
do
include
ExclusiveLeaseHelpers
include
ExclusiveLeaseHelpers
let
(
:worker_lease_key
)
{
Ci
::
StuckBuilds
::
DropRunningWorker
::
EXCLUSIVE_LEASE_KEY
}
let
(
:worker
)
{
described_class
.
new
}
let
(
:worker_lease_uuid
)
{
SecureRandom
.
uuid
}
let
(
:lease_uuid
)
{
SecureRandom
.
uuid
}
let
(
:worker2
)
{
described_class
.
new
}
subject
(
:worker
)
{
described_class
.
new
}
before
do
stub_exclusive_lease
(
worker_lease_key
,
worker_lease_uuid
)
end
describe
'#perform'
do
describe
'#perform'
do
subject
{
worker
.
perform
}
it_behaves_like
'an idempotent worker'
it_behaves_like
'an idempotent worker'
it
'executes an instance of Ci::StuckBuilds::DropRunningService'
do
it
'executes an instance of Ci::StuckBuilds::DropRunningService'
do
expect_to_obtain_exclusive_lease
(
worker
.
lease_key
,
lease_uuid
)
expect_next_instance_of
(
Ci
::
StuckBuilds
::
DropRunningService
)
do
|
service
|
expect_next_instance_of
(
Ci
::
StuckBuilds
::
DropRunningService
)
do
|
service
|
expect
(
service
).
to
receive
(
:execute
).
exactly
(
:once
)
expect
(
service
).
to
receive
(
:execute
).
exactly
(
:once
)
end
end
worker
.
perform
expect_to_cancel_exclusive_lease
(
worker
.
lease_key
,
lease_uuid
)
end
context
'with an exclusive lease'
do
it
'does not execute concurrently'
do
expect
(
worker
).
to
receive
(
:remove_lease
).
exactly
(
:once
)
expect
(
worker2
).
not_to
receive
(
:remove_lease
)
worker
.
perform
stub_exclusive_lease_taken
(
worker_lease_key
)
worker2
.
perform
end
it
'can execute in sequence'
do
expect
(
worker
).
to
receive
(
:remove_lease
).
at_least
(
:once
)
expect
(
worker2
).
to
receive
(
:remove_lease
).
at_least
(
:once
)
worker
.
perform
subject
worker2
.
perform
end
it
'cancels exclusive leases after worker perform'
do
expect_to_cancel_exclusive_lease
(
worker_lease_key
,
worker_lease_uuid
)
worker
.
perform
end
context
'when the DropRunningService fails'
do
it
'ensures cancellation of the exclusive lease'
do
expect_to_cancel_exclusive_lease
(
worker_lease_key
,
worker_lease_uuid
)
allow_next_instance_of
(
Ci
::
StuckBuilds
::
DropRunningService
)
do
|
service
|
expect
(
service
).
to
receive
(
:execute
)
do
raise
'The query timed out'
end
end
expect
{
worker
.
perform
}.
to
raise_error
(
/The query timed out/
)
end
end
end
end
end
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment