Add cleanup mechanizm for stale scheduled jobs

app/models/commit_status.rb

@@ -49,7 +49,8 @@ class CommitStatus < ActiveRecord::Base
     stuck_or_timeout_failure: 3,
     runner_system_failure: 4,
     missing_dependency_failure: 5,
-    runner_unsupported: 6
+    runner_unsupported: 6,
+    schedule_expired: 7
   }
 
   ##

app/workers/stuck_ci_jobs_worker.rb

@@ -8,6 +8,7 @@ class StuckCiJobsWorker
 
   BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
   BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
+  BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
   BUILD_PENDING_STUCK_TIMEOUT = 1.hour
 
   def perform
@@ -15,9 +16,10 @@ class StuckCiJobsWorker
 
     Rails.logger.info "#{self.class}: Cleaning stuck builds"
 
-    drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT
-    drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT
-    drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT
+    drop :running, :updated_at, BUILD_RUNNING_OUTDATED_TIMEOUT, :stuck_or_timeout_failure
+    drop :pending, :updated_at, BUILD_PENDING_OUTDATED_TIMEOUT, :stuck_or_timeout_failure
+    drop :scheduled, :scheduled_at, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired
+    drop_stuck :pending, :updated_at, BUILD_PENDING_STUCK_TIMEOUT, :stuck_or_timeout_failure
 
     remove_lease
   end
@@ -32,25 +34,27 @@ class StuckCiJobsWorker
     Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
   end
 
-  def drop(status, timeout)
-    search(status, timeout) do |build|
-      drop_build :outdated, build, status, timeout
+  def drop(status, column, timeout, reason)
+    search(status, column, timeout) do |build|
+      drop_build :outdated, build, status, timeout, reason
     end
   end
 
-  def drop_stuck(status, timeout)
-    search(status, timeout) do |build|
+  def drop_stuck(status, column, timeout, reason)
+    search(status, column, timeout) do |build|
       break unless build.stuck?
 
-      drop_build :stuck, build, status, timeout
+      drop_build :stuck, build, status, timeout, reason
     end
   end
 
   # rubocop: disable CodeReuse/ActiveRecord
-  def search(status, timeout)
+  def search(status, column, timeout)
+    quoted_column = ActiveRecord::Base.connection.quote_column_name(column)
+
     loop do
       jobs = Ci::Build.where(status: status)
-        .where('ci_builds.updated_at < ?', timeout.ago)
+        .where("#{quoted_column} < ?", timeout.ago)
         .includes(:tags, :runner, project: :namespace)
         .limit(100)
         .to_a
@@ -63,10 +67,10 @@ class StuckCiJobsWorker
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
-  def drop_build(type, build, status, timeout)
+  def drop_build(type, build, status, timeout, reason)
     Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})"
     Gitlab::OptimisticLocking.retry_lock(build, 3) do |b|
-      b.drop(:stuck_or_timeout_failure)
+      b.drop(reason)
     end
   end
 end