Fail batch-aborted pipelines with reason

Cancellation should not be used for automated
abandonment of pipelines. We should fail them
instead with a reason string.

app/models/concerns/enums/ci/commit_status.rb

@@ -20,6 +20,8 @@ module Enums
           scheduler_failure: 11,
           data_integrity_failure: 12,
           forward_deployment_failure: 13,
+          user_blocked: 14,
+          project_deleted: 15,
           insufficient_bridge_permissions: 1_001,
           downstream_bridge_project_not_found: 1_002,
           invalid_bridge_trigger: 1_003,

app/models/concerns/enums/ci/pipeline.rb

@@ -13,7 +13,9 @@ module Enums
           activity_limit_exceeded: 20,
           size_limit_exceeded: 21,
           job_activity_limit_exceeded: 22,
-          deployments_limit_exceeded: 23
+          deployments_limit_exceeded: 23,
+          user_blocked: 24,
+          project_deleted: 25
         }
       end
 

app/models/user.rb

@@ -354,7 +354,7 @@ class User < ApplicationRecord
     # this state transition object in order to do a rollback.
     # For this reason the tradeoff is to disable this cop.
     after_transition any => :blocked do |user|
-      Ci::AbortPipelinesService.new.execute(user.pipelines)
+      Ci::AbortPipelinesService.new.execute(user.pipelines, :user_blocked)
       Ci::DisableUserPipelineSchedulesService.new.execute(user)
     end
     # rubocop: enable CodeReuse/ServiceClass

app/presenters/ci/pipeline_presenter.rb

@@ -14,7 +14,9 @@ module Ci
         activity_limit_exceeded: 'Pipeline activity limit exceeded!',
         size_limit_exceeded: 'Pipeline size limit exceeded!',
         job_activity_limit_exceeded: 'Pipeline job activity limit exceeded!',
-        deployments_limit_exceeded: 'Pipeline deployments limit exceeded!' }
+        deployments_limit_exceeded: 'Pipeline deployments limit exceeded!',
+        project_deleted: 'The associated project was deleted',
+        user_blocked: 'The user who created this pipeline is blocked' }
     end
 
     presents :pipeline

app/presenters/commit_status_presenter.rb

@@ -21,7 +21,9 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
     bridge_pipeline_is_child_pipeline: 'This job belongs to a child pipeline and cannot create further child pipelines',
     downstream_pipeline_creation_failed: 'The downstream pipeline could not be created',
     secrets_provider_not_found: 'The secrets provider can not be found',
-    reached_max_descendant_pipelines_depth: 'Maximum child pipeline depth has been reached'
+    reached_max_descendant_pipelines_depth: 'You reached the maximum depth of child pipelines',
+    project_deleted: 'The job belongs to a deleted project',
+    user_blocked: 'The user who created this job is blocked'
   }.freeze
 
   private_constant :CALLOUT_FAILURE_MESSAGES

app/services/ci/abort_pipelines_service.rb

@@ -2,28 +2,27 @@
 
 module Ci
   class AbortPipelinesService
-    # Danger: Cancels in bulk without callbacks
+    # NOTE: This call fails pipelines in bulk without running callbacks.
     # Only for pipeline abandonment scenarios (examples: project delete, user block)
-    def execute(pipelines)
-      @time = Time.current
+    def execute(pipelines, failure_reason)
+      pipelines.cancelable.each_batch(of: 100) do |pipeline_batch|
+        now = Time.current
 
-      bulk_abort!(pipelines.cancelable, { status: :canceled })
+        basic_attributes = { status: :failed }
+        all_attributes = basic_attributes.merge(failure_reason: failure_reason, finished_at: now)
 
-      ServiceResponse.success(message: 'Pipelines canceled')
-    end
-
-    private
-
-    def bulk_abort!(pipelines, attributes)
-      pipelines.each_batch(of: 100) do |pipeline_batch|
-        update_status_for(Ci::Stage, pipeline_batch, attributes)
-        update_status_for(CommitStatus, pipeline_batch, attributes.merge(finished_at: @time))
+        bulk_fail_for(Ci::Stage, pipeline_batch, basic_attributes)
+        bulk_fail_for(CommitStatus, pipeline_batch, all_attributes)
 
-        pipeline_batch.update_all(attributes.merge(finished_at: @time))
+        pipeline_batch.update_all(all_attributes)
       end
+
+      ServiceResponse.success(message: 'Pipelines stopped')
     end
 
-    def update_status_for(klass, pipelines, attributes)
+    private
+
+    def bulk_fail_for(klass, pipelines, attributes)
       klass.in_pipelines(pipelines)
         .cancelable
         .in_batches(of: 150) # rubocop:disable Cop/InBatches

app/services/projects/destroy_service.rb

@@ -28,7 +28,7 @@ module Projects
       flush_caches(project)
 
       if Feature.enabled?(:abort_deleted_project_pipelines, default_enabled: :yaml)
-        ::Ci::AbortPipelinesService.new.execute(project.all_pipelines)
+        ::Ci::AbortPipelinesService.new.execute(project.all_pipelines, :project_deleted)
       end
 
       Projects::UnlinkForkService.new(project, current_user).execute

changelogs/unreleased/324646-bulk-fail-with-reason.yml

+---
+title: Fail batch-aborted pipelines with reason
+merge_request: 57838
+author:
+type: changed

lib/gitlab/ci/status/build/failed.rb

@@ -26,7 +26,9 @@ module Gitlab
             bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline',
             downstream_pipeline_creation_failed: 'downstream pipeline can not be created',
             secrets_provider_not_found: 'secrets provider can not be found',
-            reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines'
+            reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines',
+            project_deleted: 'pipeline project was deleted',
+            user_blocked: 'pipeline user was blocked'
           }.freeze
 
           private_constant :REASONS

spec/models/user_spec.rb

@@ -1804,7 +1804,7 @@ RSpec.describe User do
       it 'aborts all running pipelines and related jobs' do
         expect(user).to receive(:pipelines).and_return(pipelines)
         expect(Ci::AbortPipelinesService).to receive(:new).and_return(service)
-        expect(service).to receive(:execute).with(pipelines)
+        expect(service).to receive(:execute).with(pipelines, :user_blocked)
 
         user.block
       end

spec/services/ci/abort_pipelines_service_spec.rb

@@ -17,33 +17,38 @@ RSpec.describe Ci::AbortPipelinesService do
   describe '#execute' do
     def expect_correct_cancellations
       expect(cancelable_pipeline.finished_at).not_to be_nil
-      expect(cancelable_pipeline).to be_canceled
-      expect(cancelable_pipeline.stages - [non_cancelable_stage]).to all(be_canceled)
-      expect(cancelable_build).to be_canceled
-
-      expect(manual_pipeline).not_to be_canceled
-      expect(non_cancelable_stage).not_to be_canceled
-      expect(non_cancelable_build).not_to be_canceled
+      expect(cancelable_pipeline.status).to eq('failed')
+      expect((cancelable_pipeline.stages - [non_cancelable_stage]).map(&:status)).to all(eq('failed'))
+      expect(cancelable_build.status).to eq('failed')
+      expect(cancelable_build.finished_at).not_to be_nil
+
+      expect(manual_pipeline.status).not_to eq('failed')
+      expect(non_cancelable_stage.status).not_to eq('failed')
+      expect(non_cancelable_build.status).not_to eq('failed')
     end
 
     context 'with project pipelines' do
-      it 'cancels all running pipelines and related jobs' do
-        expect(described_class.new.execute(project.all_pipelines)).to be_success
+      def abort_project_pipelines
+        described_class.new.execute(project.all_pipelines, :project_deleted)
+      end
+
+      it 'fails all running pipelines and related jobs' do
+        expect(abort_project_pipelines).to be_success
 
         expect_correct_cancellations
 
-        expect(other_users_pipeline).to be_canceled
-        expect(other_users_pipeline.stages).to all(be_canceled)
+        expect(other_users_pipeline.status).to eq('failed')
+        expect(other_users_pipeline.failure_reason).to eq('project_deleted')
+        expect(other_users_pipeline.stages.map(&:status)).to all(eq('failed'))
       end
 
       it 'avoids N+1 queries' do
-        project_pipelines = project.all_pipelines
-        control_count = ActiveRecord::QueryRecorder.new { described_class.new.execute(project_pipelines) }.count
+        control_count = ActiveRecord::QueryRecorder.new { abort_project_pipelines }.count
 
         pipelines = create_list(:ci_pipeline, 5, :running, project: project)
         create_list(:ci_build, 5, :running, pipeline: pipelines.first)
 
-        expect { described_class.new.execute(project_pipelines) }.not_to exceed_query_limit(control_count)
+        expect { abort_project_pipelines }.not_to exceed_query_limit(control_count)
       end
 
       context 'with live build logs' do
@@ -51,11 +56,11 @@ RSpec.describe Ci::AbortPipelinesService do
           create(:ci_build_trace_chunk, build: cancelable_build)
         end
 
-        it 'makes canceled builds with stale trace visible' do
+        it 'makes failed builds with stale trace visible' do
           expect(Ci::Build.with_stale_live_trace.count).to eq 0
 
           travel_to(2.days.ago) do
-            described_class.new.execute(project.all_pipelines)
+            abort_project_pipelines
           end
 
           expect(Ci::Build.with_stale_live_trace.count).to eq 1
@@ -64,22 +69,25 @@ RSpec.describe Ci::AbortPipelinesService do
     end
 
     context 'with user pipelines' do
-      it 'cancels all running pipelines and related jobs' do
-        expect(described_class.new.execute(user.pipelines)).to be_success
+      def abort_user_pipelines
+        described_class.new.execute(user.pipelines, :user_blocked)
+      end
+
+      it 'fails all running pipelines and related jobs' do
+        expect(abort_user_pipelines).to be_success
 
         expect_correct_cancellations
 
-        expect(other_users_pipeline).not_to be_canceled
+        expect(other_users_pipeline.status).not_to eq('failed')
       end
 
       it 'avoids N+1 queries' do
-        user_pipelines = user.pipelines
-        control_count = ActiveRecord::QueryRecorder.new { described_class.new.execute(user_pipelines) }.count
+        control_count = ActiveRecord::QueryRecorder.new { abort_user_pipelines }.count
 
         pipelines = create_list(:ci_pipeline, 5, :running, project: project, user: user)
         create_list(:ci_build, 5, :running, pipeline: pipelines.first)
 
-        expect { described_class.new.execute(user_pipelines) }.not_to exceed_query_limit(control_count)
+        expect { abort_user_pipelines }.not_to exceed_query_limit(control_count)
       end
     end
   end

spec/services/projects/destroy_service_spec.rb

@@ -109,7 +109,7 @@ RSpec.describe Projects::DestroyService, :aggregate_failures do
       pipelines = build_list(:ci_pipeline, 3, :running)
       allow(project).to receive(:all_pipelines).and_return(pipelines)
 
-      expect(::Ci::AbortPipelinesService).to receive_message_chain(:new, :execute).with(pipelines)
+      expect(::Ci::AbortPipelinesService).to receive_message_chain(:new, :execute).with(pipelines, :project_deleted)
 
       destroy_project(project, user, {})
     end