Update stuck and outdated builds cleanup worker

app/workers/stuck_ci_builds_worker.rb

@@ -2,18 +2,42 @@ class StuckCiBuildsWorker
   include Sidekiq::Worker
   include CronjobQueue
 
-  BUILD_STUCK_TIMEOUT = 1.day
+  BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
+  BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
+  BUILD_PENDING_STUCK_TIMEOUT = 1.hour
 
   def perform
     Rails.logger.info 'Cleaning stuck builds'
 
-    builds = Ci::Build.joins(:project).running_or_pending.where('ci_builds.updated_at < ?', BUILD_STUCK_TIMEOUT.ago)
-    builds.find_each(batch_size: 50).each do |build|
-      Rails.logger.debug "Dropping stuck #{build.status} build #{build.id} for runner #{build.runner_id}"
-      build.drop
+    drop       :running, BUILD_RUNNING_OUTDATED_TIMEOUT
+    drop       :pending, BUILD_PENDING_OUTDATED_TIMEOUT
+    drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT
+  end
+
+  private
+
+  def drop(status, timeout)
+    search(status, timeout) do |build|
+      drop_build :outdated, build, status, timeout
+    end
+  end
+
+  def drop_stuck(status, timeout)
+    search(status, timeout) do |build|
+      return unless build.stuck?
+      drop_build :stuck, build, status, timeout
     end
+  end
+
+  def search(status, timeout)
+    builds = Ci::Build.where(status: status).where('ci_builds.updated_at < ?', timeout.ago)
+    builds.joins(:project).find_each(batch_size: 50).each do |build|
+      yield(build)
+    end
+  end
 
-    # Update builds that failed to drop
-    builds.update_all(status: 'failed')
+  def drop_build(type, build, status, timeout)
+    Rails.logger.info "#{self.class}: Dropping #{type.to_s} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})"
+    build.drop
   end
 end

spec/workers/stuck_ci_builds_worker_spec.rb

-require "spec_helper"
+require 'spec_helper'
 
 describe StuckCiBuildsWorker do
-  let!(:build) { create :ci_build }
+  let!(:runner) { create :ci_runner }
+  let!(:build) { create :ci_build, runner: runner }
   let(:worker) { described_class.new }
 
   subject do
@@ -9,47 +10,88 @@ describe StuckCiBuildsWorker do
     build.status
   end
 
-  %w(pending running).each do |status|
-    context "#{status} build" do
-      before do
-        build.update!(status: status)
+  before { build.update!(status: status, updated_at: updated_at) }
+
+  shared_examples 'build is dropped' do
+    it 'changes status' do
+      worker.perform
+      is_expected.to eq('failed')
+    end
+  end
+
+  shared_examples 'build is unchanged' do
+    it "doesn't change status" do
+      worker.perform
+      is_expected.to eq(status)
+    end
+  end
+
+  context 'when build is pending' do
+    let(:status) { 'pending' }
+
+    context 'when build is not stuck' do
+      before { allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(false) }
+
+      context 'when build was not updated for more than 1 day ago' do
+        let(:updated_at) { 2.days.ago }
+        it_behaves_like 'build is dropped'
       end
 
-      it 'gets dropped if it was updated over 2 days ago' do
-        build.update!(updated_at: 2.days.ago)
-        worker.perform
-        is_expected.to eq('failed')
+      context 'when build was updated in less than 1 day ago' do
+        let(:updated_at) { 6.hours.ago }
+        it_behaves_like 'build is unchanged'
       end
 
-      it "is still #{status}" do
-        build.update!(updated_at: 1.minute.ago)
-        worker.perform
-        is_expected.to eq(status)
+      context 'when build was not updated for more than 1 hour ago' do
+        let(:updated_at) { 2.hours.ago }
+        it_behaves_like 'build is unchanged'
       end
     end
-  end
 
-  %w(success failed canceled).each do |status|
-    context "#{status} build" do
-      before do
-        build.update!(status: status)
+    context 'when build is stuck' do
+      before { allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(true) }
+
+      context 'when build was not updated for more than 1 hour ago' do
+        let(:updated_at) { 2.hours.ago }
+        it_behaves_like 'build is dropped'
       end
 
-      it "is still #{status}" do
-        build.update!(updated_at: 2.days.ago)
-        worker.perform
-        is_expected.to eq(status)
+      context 'when build was updated in less than 1 hour ago' do
+        let(:updated_at) { 30.minutes.ago }
+        it_behaves_like 'build is unchanged'
       end
     end
   end
 
-  context "for deleted project" do
-    before do
-      build.update!(status: :running, updated_at: 2.days.ago)
-      build.project.update(pending_delete: true)
+  context 'when build is running' do
+    let(:status) { 'running' }
+
+    context 'when build was not updated for more than 1 hour ago' do
+      let(:updated_at) { 2.hours.ago }
+      it_behaves_like 'build is dropped'
     end
 
-    it "does not drop build" do
+    context 'when build was updated in less than 1 hour ago' do
+      let(:updated_at) { 30.minutes.ago }
+      it_behaves_like 'build is unchanged'
+    end
+  end
+
+  %w(success skipped failed canceled).each do |status|
+    context "when build is #{status}" do
+      let(:status) { status }
+      let(:updated_at) { 2.days.ago }
+      it_behaves_like 'build is unchanged'
+    end
+  end
+
+  context 'for deleted project' do
+    let(:status) { 'running' }
+    let(:updated_at) { 2.days.ago }
+
+    before { build.project.update(pending_delete: true) }
+
+    it 'does not drop build' do
       expect_any_instance_of(Ci::Build).not_to receive(:drop)
       worker.perform
     end