Commit 8d1cdc94 authored by Tomasz Maczukin's avatar Tomasz Maczukin

Update stuck and outdated builds cleanup worker

parent e5bafed8
......@@ -2,18 +2,42 @@ class StuckCiBuildsWorker
include Sidekiq::Worker
include CronjobQueue
BUILD_STUCK_TIMEOUT = 1.day
BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
BUILD_PENDING_STUCK_TIMEOUT = 1.hour
def perform
Rails.logger.info 'Cleaning stuck builds'
builds = Ci::Build.joins(:project).running_or_pending.where('ci_builds.updated_at < ?', BUILD_STUCK_TIMEOUT.ago)
builds.find_each(batch_size: 50).each do |build|
Rails.logger.debug "Dropping stuck #{build.status} build #{build.id} for runner #{build.runner_id}"
build.drop
drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT
drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT
drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT
end
private
def drop(status, timeout)
search(status, timeout) do |build|
drop_build :outdated, build, status, timeout
end
end
def drop_stuck(status, timeout)
search(status, timeout) do |build|
return unless build.stuck?
drop_build :stuck, build, status, timeout
end
end
def search(status, timeout)
builds = Ci::Build.where(status: status).where('ci_builds.updated_at < ?', timeout.ago)
builds.joins(:project).find_each(batch_size: 50).each do |build|
yield(build)
end
end
# Update builds that failed to drop
builds.update_all(status: 'failed')
def drop_build(type, build, status, timeout)
Rails.logger.info "#{self.class}: Dropping #{type.to_s} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})"
build.drop
end
end
require "spec_helper"
require 'spec_helper'
describe StuckCiBuildsWorker do
let!(:build) { create :ci_build }
let!(:runner) { create :ci_runner }
let!(:build) { create :ci_build, runner: runner }
let(:worker) { described_class.new }
subject do
......@@ -9,47 +10,88 @@ describe StuckCiBuildsWorker do
build.status
end
%w(pending running).each do |status|
context "#{status} build" do
before do
build.update!(status: status)
before { build.update!(status: status, updated_at: updated_at) }
shared_examples 'build is dropped' do
it 'changes status' do
worker.perform
is_expected.to eq('failed')
end
end
shared_examples 'build is unchanged' do
it "doesn't change status" do
worker.perform
is_expected.to eq(status)
end
end
context 'when build is pending' do
let(:status) { 'pending' }
context 'when build is not stuck' do
before { allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(false) }
context 'when build was not updated for more than 1 day ago' do
let(:updated_at) { 2.days.ago }
it_behaves_like 'build is dropped'
end
it 'gets dropped if it was updated over 2 days ago' do
build.update!(updated_at: 2.days.ago)
worker.perform
is_expected.to eq('failed')
context 'when build was updated in less than 1 day ago' do
let(:updated_at) { 6.hours.ago }
it_behaves_like 'build is unchanged'
end
it "is still #{status}" do
build.update!(updated_at: 1.minute.ago)
worker.perform
is_expected.to eq(status)
context 'when build was not updated for more than 1 hour ago' do
let(:updated_at) { 2.hours.ago }
it_behaves_like 'build is unchanged'
end
end
end
%w(success failed canceled).each do |status|
context "#{status} build" do
before do
build.update!(status: status)
context 'when build is stuck' do
before { allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(true) }
context 'when build was not updated for more than 1 hour ago' do
let(:updated_at) { 2.hours.ago }
it_behaves_like 'build is dropped'
end
it "is still #{status}" do
build.update!(updated_at: 2.days.ago)
worker.perform
is_expected.to eq(status)
context 'when build was updated in less than 1 hour ago' do
let(:updated_at) { 30.minutes.ago }
it_behaves_like 'build is unchanged'
end
end
end
context "for deleted project" do
before do
build.update!(status: :running, updated_at: 2.days.ago)
build.project.update(pending_delete: true)
context 'when build is running' do
let(:status) { 'running' }
context 'when build was not updated for more than 1 hour ago' do
let(:updated_at) { 2.hours.ago }
it_behaves_like 'build is dropped'
end
it "does not drop build" do
context 'when build was updated in less than 1 hour ago' do
let(:updated_at) { 30.minutes.ago }
it_behaves_like 'build is unchanged'
end
end
%w(success skipped failed canceled).each do |status|
context "when build is #{status}" do
let(:status) { status }
let(:updated_at) { 2.days.ago }
it_behaves_like 'build is unchanged'
end
end
context 'for deleted project' do
let(:status) { 'running' }
let(:updated_at) { 2.days.ago }
before { build.project.update(pending_delete: true) }
it 'does not drop build' do
expect_any_instance_of(Ci::Build).not_to receive(:drop)
worker.perform
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment