Commit 4f55ef53 authored by drew cimino's avatar drew cimino

Use ExclusiveLeaseGuard in Ci::StuckBuilds::DropRunningWorker

parent f407e01e
...@@ -4,6 +4,7 @@ module Ci ...@@ -4,6 +4,7 @@ module Ci
module StuckBuilds module StuckBuilds
class DropRunningWorker class DropRunningWorker
include ApplicationWorker include ApplicationWorker
include ExclusiveLeaseGuard
idempotent! idempotent!
...@@ -17,26 +18,16 @@ module Ci ...@@ -17,26 +18,16 @@ module Ci
feature_category :continuous_integration feature_category :continuous_integration
EXCLUSIVE_LEASE_KEY = 'ci_stuck_builds_drop_running_worker_lease'
def perform def perform
return unless try_obtain_lease try_obtain_lease do
begin
Ci::StuckBuilds::DropRunningService.new.execute Ci::StuckBuilds::DropRunningService.new.execute
ensure
remove_lease
end end
end end
private private
def try_obtain_lease def lease_timeout
@uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain 30.minutes
end
def remove_lease
Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
end end
end end
end end
......
...@@ -15,8 +15,6 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker ...@@ -15,8 +15,6 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
feature_category :continuous_integration feature_category :continuous_integration
worker_resource_boundary :cpu worker_resource_boundary :cpu
EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease'
def perform def perform
Ci::StuckBuilds::DropRunningWorker.perform_in(20.minutes) Ci::StuckBuilds::DropRunningWorker.perform_in(20.minutes)
Ci::StuckBuilds::DropScheduledWorker.perform_in(40.minutes) Ci::StuckBuilds::DropScheduledWorker.perform_in(40.minutes)
......
...@@ -5,66 +5,24 @@ require 'spec_helper' ...@@ -5,66 +5,24 @@ require 'spec_helper'
RSpec.describe Ci::StuckBuilds::DropRunningWorker do RSpec.describe Ci::StuckBuilds::DropRunningWorker do
include ExclusiveLeaseHelpers include ExclusiveLeaseHelpers
let(:worker_lease_key) { Ci::StuckBuilds::DropRunningWorker::EXCLUSIVE_LEASE_KEY } let(:worker) { described_class.new }
let(:worker_lease_uuid) { SecureRandom.uuid } let(:lease_uuid) { SecureRandom.uuid }
let(:worker2) { described_class.new }
subject(:worker) { described_class.new }
before do
stub_exclusive_lease(worker_lease_key, worker_lease_uuid)
end
describe '#perform' do describe '#perform' do
subject { worker.perform }
it_behaves_like 'an idempotent worker' it_behaves_like 'an idempotent worker'
it 'executes an instance of Ci::StuckBuilds::DropRunningService' do it 'executes an instance of Ci::StuckBuilds::DropRunningService' do
expect_to_obtain_exclusive_lease(worker.lease_key, lease_uuid)
expect_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service| expect_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
expect(service).to receive(:execute).exactly(:once) expect(service).to receive(:execute).exactly(:once)
end end
worker.perform expect_to_cancel_exclusive_lease(worker.lease_key, lease_uuid)
end
context 'with an exclusive lease' do
it 'does not execute concurrently' do
expect(worker).to receive(:remove_lease).exactly(:once)
expect(worker2).not_to receive(:remove_lease)
worker.perform
stub_exclusive_lease_taken(worker_lease_key)
worker2.perform
end
it 'can execute in sequence' do
expect(worker).to receive(:remove_lease).at_least(:once)
expect(worker2).to receive(:remove_lease).at_least(:once)
worker.perform subject
worker2.perform
end
it 'cancels exclusive leases after worker perform' do
expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
worker.perform
end
context 'when the DropRunningService fails' do
it 'ensures cancellation of the exclusive lease' do
expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
allow_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
expect(service).to receive(:execute) do
raise 'The query timed out'
end
end
expect { worker.perform }.to raise_error(/The query timed out/)
end
end
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment