Commit 4f55ef53 authored by drew cimino's avatar drew cimino

Use ExclusiveLeaseGuard in Ci::StuckBuilds::DropRunningWorker

parent f407e01e
......@@ -4,6 +4,7 @@ module Ci
module StuckBuilds
class DropRunningWorker
include ApplicationWorker
include ExclusiveLeaseGuard
idempotent!
......@@ -17,26 +18,16 @@ module Ci
feature_category :continuous_integration
EXCLUSIVE_LEASE_KEY = 'ci_stuck_builds_drop_running_worker_lease'
def perform
return unless try_obtain_lease
begin
try_obtain_lease do
Ci::StuckBuilds::DropRunningService.new.execute
ensure
remove_lease
end
end
private
def try_obtain_lease
@uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain
end
def remove_lease
Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
def lease_timeout
30.minutes
end
end
end
......
......@@ -15,8 +15,6 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
feature_category :continuous_integration
worker_resource_boundary :cpu
EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease'
def perform
Ci::StuckBuilds::DropRunningWorker.perform_in(20.minutes)
Ci::StuckBuilds::DropScheduledWorker.perform_in(40.minutes)
......
......@@ -5,66 +5,24 @@ require 'spec_helper'
RSpec.describe Ci::StuckBuilds::DropRunningWorker do
include ExclusiveLeaseHelpers
let(:worker_lease_key) { Ci::StuckBuilds::DropRunningWorker::EXCLUSIVE_LEASE_KEY }
let(:worker_lease_uuid) { SecureRandom.uuid }
let(:worker2) { described_class.new }
subject(:worker) { described_class.new }
before do
stub_exclusive_lease(worker_lease_key, worker_lease_uuid)
end
let(:worker) { described_class.new }
let(:lease_uuid) { SecureRandom.uuid }
describe '#perform' do
subject { worker.perform }
it_behaves_like 'an idempotent worker'
it 'executes an instance of Ci::StuckBuilds::DropRunningService' do
expect_to_obtain_exclusive_lease(worker.lease_key, lease_uuid)
expect_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
expect(service).to receive(:execute).exactly(:once)
end
worker.perform
end
context 'with an exclusive lease' do
it 'does not execute concurrently' do
expect(worker).to receive(:remove_lease).exactly(:once)
expect(worker2).not_to receive(:remove_lease)
worker.perform
stub_exclusive_lease_taken(worker_lease_key)
worker2.perform
end
it 'can execute in sequence' do
expect(worker).to receive(:remove_lease).at_least(:once)
expect(worker2).to receive(:remove_lease).at_least(:once)
worker.perform
worker2.perform
end
it 'cancels exclusive leases after worker perform' do
expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
expect_to_cancel_exclusive_lease(worker.lease_key, lease_uuid)
worker.perform
end
context 'when the DropRunningService fails' do
it 'ensures cancellation of the exclusive lease' do
expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
allow_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
expect(service).to receive(:execute) do
raise 'The query timed out'
end
end
expect { worker.perform }.to raise_error(/The query timed out/)
end
end
subject
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment