Create Deployment in a separate transaction

This commit addresses the Cross-DB transaction
issue on the deployment creation in the
pipeline creation service.

app/models/ci/pipeline.rb

@@ -1287,6 +1287,12 @@ module Ci
       end
     end
 
+    def create_deployment_in_separate_transaction?
+      strong_memoize(:create_deployment_in_separate_transaction) do
+        ::Feature.enabled?(:create_deployment_in_separate_transaction, project, default_enabled: :yaml)
+      end
+    end
+
     private
 
     def add_message(severity, content)

app/services/ci/create_pipeline_service.rb

@@ -28,7 +28,10 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Validate::External,
                 Gitlab::Ci::Pipeline::Chain::Populate,
                 Gitlab::Ci::Pipeline::Chain::StopDryRun,
+                Gitlab::Ci::Pipeline::Chain::EnsureEnvironments,
+                Gitlab::Ci::Pipeline::Chain::EnsureResourceGroups,
                 Gitlab::Ci::Pipeline::Chain::Create,
+                Gitlab::Ci::Pipeline::Chain::CreateDeployments,
                 Gitlab::Ci::Pipeline::Chain::CreateCrossDatabaseAssociations,
                 Gitlab::Ci::Pipeline::Chain::Limit::Activity,
                 Gitlab::Ci::Pipeline::Chain::Limit::JobActivity,

config/feature_flags/development/create_deployment_in_separate_transaction.yml

+---
+name: create_deployment_in_separate_transaction
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75604
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/346879
+milestone: '14.6'
+type: development
+group: group::release
+default_enabled: false

lib/gitlab/ci/pipeline/chain/build.rb

@@ -21,6 +21,10 @@ module Gitlab
               merge_request: @command.merge_request,
               external_pull_request: @command.external_pull_request,
               locked: @command.project.default_pipeline_lock)
+
+            # Initialize the feature flag at the beginning of the pipeline creation process
+            # so that the flag references in the latter chains return the same value.
+            @pipeline.create_deployment_in_separate_transaction?
           end
 
           def break?

lib/gitlab/ci/pipeline/chain/create_deployments.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class CreateDeployments < Chain::Base
+          DeploymentCreationError = Class.new(StandardError)
+
+          def perform!
+            return unless pipeline.create_deployment_in_separate_transaction?
+
+            create_deployments!
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          def create_deployments!
+            pipeline.stages.map(&:statuses).flatten.map(&method(:create_deployment))
+          end
+
+          def create_deployment(build)
+            return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present?
+
+            deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment
+              .new(build, build.persisted_environment).to_resource
+
+            return unless deployment
+
+            deployment.deployable = build
+            deployment.save!
+          rescue ActiveRecord::RecordInvalid => e
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(
+              DeploymentCreationError.new(e.message), build_id: build.id)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/ensure_environments.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class EnsureEnvironments < Chain::Base
+          def perform!
+            return unless pipeline.create_deployment_in_separate_transaction?
+
+            pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_environment))
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          def ensure_environment(build)
+            return unless build.instance_of?(::Ci::Build) && build.has_environment?
+
+            environment = ::Gitlab::Ci::Pipeline::Seed::Environment.new(build).to_resource
+
+            if environment.persisted?
+              build.persisted_environment = environment
+              build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name })
+            else
+              build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure)
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class EnsureResourceGroups < Chain::Base
+          def perform!
+            return unless pipeline.create_deployment_in_separate_transaction?
+
+            pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_resource_group))
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          def ensure_resource_group(processable)
+            return unless processable.is_a?(::Ci::Processable)
+
+            key = processable.options.delete(:resource_group_key)
+
+            resource_group = ::Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup
+              .new(processable, key).to_resource
+
+            processable.resource_group = resource_group
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/seed/build.rb

@@ -78,7 +78,7 @@ module Gitlab
           def to_resource
             strong_memoize(:resource) do
               processable = initialize_processable
-              assign_resource_group(processable)
+              assign_resource_group(processable) unless @pipeline.create_deployment_in_separate_transaction?
               processable
             end
           end
@@ -88,10 +88,12 @@ module Gitlab
               ::Ci::Bridge.new(attributes)
             else
               ::Ci::Build.new(attributes).tap do |build|
+                unless @pipeline.create_deployment_in_separate_transaction?
                   build.assign_attributes(self.class.deployment_attributes_for(build))
                 end
               end
             end
+          end
 
           def assign_resource_group(processable)
             processable.resource_group =

lib/gitlab/ci/yaml_processor/result.rb

@@ -92,6 +92,7 @@ module Gitlab
               script: job[:script],
               after_script: job[:after_script],
               environment: job[:environment],
+              resource_group_key: job[:resource_group],
               retry: job[:retry],
               parallel: job[:parallel],
               instance: job[:instance],

spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Pipeline::Chain::CreateDeployments do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
+
+  let(:stage) { build(:ci_stage_entity, project: project, statuses: [job]) }
+  let(:pipeline) { create(:ci_pipeline, project: project, stages: [stage]) }
+
+  let(:command) do
+    Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user)
+  end
+
+  let(:step) { described_class.new(pipeline, command) }
+
+  describe '#perform!' do
+    subject { step.perform! }
+
+    before do
+      job.pipeline = pipeline
+    end
+
+    context 'when a pipeline contains a deployment job' do
+      let!(:job) { build(:ci_build, :start_review_app, project: project) }
+      let!(:environment) { create(:environment, project: project, name: job.expanded_environment_name) }
+
+      it 'creates a deployment record' do
+        expect { subject }.to change { Deployment.count }.by(1)
+
+        job.reset
+        expect(job.deployment.project).to eq(job.project)
+        expect(job.deployment.ref).to eq(job.ref)
+        expect(job.deployment.sha).to eq(job.sha)
+        expect(job.deployment.deployable).to eq(job)
+        expect(job.deployment.deployable_type).to eq('CommitStatus')
+        expect(job.deployment.environment).to eq(job.persisted_environment)
+      end
+
+      context 'when creation failure occures' do
+        before do
+          allow_next_instance_of(Deployment) do |deployment|
+            allow(deployment).to receive(:save!) { raise ActiveRecord::RecordInvalid }
+          end
+        end
+
+        it 'trackes the exception' do
+          expect { subject }.to raise_error(described_class::DeploymentCreationError)
+
+          expect(Deployment.count).to eq(0)
+        end
+      end
+
+      context 'when the corresponding environment does not exist' do
+        let!(:environment) { }
+
+        it 'does not create a deployment record' do
+          expect { subject }.not_to change { Deployment.count }
+
+          expect(job.deployment).to be_nil
+        end
+      end
+
+      context 'when create_deployment_in_separate_transaction feature flag is disabled' do
+        before do
+          stub_feature_flags(create_deployment_in_separate_transaction: false)
+        end
+
+        it 'does not create a deployment record' do
+          expect { subject }.not_to change { Deployment.count }
+
+          expect(job.deployment).to be_nil
+        end
+      end
+    end
+
+    context 'when a pipeline contains a teardown job' do
+      let!(:job) { build(:ci_build, :stop_review_app, project: project) }
+      let!(:environment) { create(:environment, name: job.expanded_environment_name) }
+
+      it 'does not create a deployment record' do
+        expect { subject }.not_to change { Deployment.count }
+
+        expect(job.deployment).to be_nil
+      end
+    end
+
+    context 'when a pipeline does not contain a deployment job' do
+      let!(:job) { build(:ci_build, project: project) }
+
+      it 'does not create any deployments' do
+        expect { subject }.not_to change { Deployment.count }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/chain/ensure_environments_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Pipeline::Chain::EnsureEnvironments do
+  let(:project) { create(:project) }
+  let(:user) { create(:user) }
+  let(:stage) { build(:ci_stage_entity, project: project, statuses: [job]) }
+  let(:pipeline) { build(:ci_pipeline, project: project, stages: [stage]) }
+
+  let(:command) do
+    Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user)
+  end
+
+  let(:step) { described_class.new(pipeline, command) }
+
+  describe '#perform!' do
+    subject { step.perform! }
+
+    before do
+      job.pipeline = pipeline
+    end
+
+    context 'when a pipeline contains a deployment job' do
+      let!(:job) { build(:ci_build, :start_review_app, project: project) }
+
+      it 'ensures environment existence for the job' do
+        expect { subject }.to change { Environment.count }.by(1)
+
+        expect(project.environments.find_by_name('review/master')).to be_present
+        expect(job.persisted_environment.name).to eq('review/master')
+        expect(job.metadata.expanded_environment_name).to eq('review/master')
+      end
+
+      context 'when an environment has already been existed' do
+        before do
+          create(:environment, project: project, name: 'review/master')
+        end
+
+        it 'ensures environment existence for the job' do
+          expect { subject }.not_to change { Environment.count }
+
+          expect(project.environments.find_by_name('review/master')).to be_present
+          expect(job.persisted_environment.name).to eq('review/master')
+          expect(job.metadata.expanded_environment_name).to eq('review/master')
+        end
+      end
+
+      context 'when an environment name contains an invalid character' do
+        let(:pipeline) { build(:ci_pipeline, ref: '!!!', project: project, stages: [stage]) }
+
+        it 'sets the failure status' do
+          expect { subject }.not_to change { Environment.count }
+
+          expect(job).to be_failed
+          expect(job).to be_environment_creation_failure
+          expect(job.persisted_environment).to be_nil
+        end
+      end
+
+      context 'when create_deployment_in_separate_transaction feature flag is disabled' do
+        before do
+          stub_feature_flags(create_deployment_in_separate_transaction: false)
+        end
+
+        it 'does not create any environments' do
+          expect { subject }.not_to change { Environment.count }
+
+          expect(job.persisted_environment).to be_nil
+        end
+      end
+    end
+
+    context 'when a pipeline contains a teardown job' do
+      let!(:job) { build(:ci_build, :stop_review_app, project: project) }
+
+      it 'ensures environment existence for the job' do
+        expect { subject }.to change { Environment.count }.by(1)
+
+        expect(project.environments.find_by_name('review/master')).to be_present
+        expect(job.persisted_environment.name).to eq('review/master')
+        expect(job.metadata.expanded_environment_name).to eq('review/master')
+      end
+    end
+
+    context 'when a pipeline does not contain a deployment job' do
+      let!(:job) { build(:ci_build, project: project) }
+
+      it 'does not create any environments' do
+        expect { subject }.not_to change { Environment.count }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/chain/ensure_resource_groups_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Pipeline::Chain::EnsureResourceGroups do
+  let(:project)  { create(:project) }
+  let(:user)     { create(:user) }
+  let(:stage) { build(:ci_stage_entity, project: project, statuses: [job]) }
+  let(:pipeline) { build(:ci_pipeline, project: project, stages: [stage]) }
+  let!(:environment) { create(:environment, name: 'production', project: project) }
+
+  let(:command) do
+    Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user)
+  end
+
+  let(:step) { described_class.new(pipeline, command) }
+
+  describe '#perform!' do
+    subject { step.perform! }
+
+    before do
+      job.pipeline = pipeline
+    end
+
+    context 'when a pipeline contains a job that requires a resource group' do
+      let!(:job) do
+        build(:ci_build, project: project, environment: 'production', options: { resource_group_key: '$CI_ENVIRONMENT_NAME' })
+      end
+
+      it 'ensures the resource group existence' do
+        expect { subject }.to change { Ci::ResourceGroup.count }.by(1)
+
+        expect(project.resource_groups.find_by_key('production')).to be_present
+        expect(job.resource_group.key).to eq('production')
+        expect(job.options[:resource_group_key]).to be_nil
+      end
+
+      context 'when a resource group has already been existed' do
+        before do
+          create(:ci_resource_group, project: project, key: 'production')
+        end
+
+        it 'ensures the resource group existence' do
+          expect { subject }.not_to change { Ci::ResourceGroup.count }
+
+          expect(project.resource_groups.find_by_key('production')).to be_present
+          expect(job.resource_group.key).to eq('production')
+          expect(job.options[:resource_group_key]).to be_nil
+        end
+      end
+
+      context 'when a resource group key contains an invalid character' do
+        let!(:job) do
+          build(:ci_build, project: project, environment: '!!!', options: { resource_group_key: '$CI_ENVIRONMENT_NAME' })
+        end
+
+        it 'does not create any resource groups' do
+          expect { subject }.not_to change { Ci::ResourceGroup.count }
+
+          expect(job.resource_group).to be_nil
+        end
+      end
+
+      context 'when create_deployment_in_separate_transaction feature flag is disabled' do
+        before do
+          stub_feature_flags(create_deployment_in_separate_transaction: false)
+        end
+
+        it 'does not create any resource groups' do
+          expect { subject }.not_to change { Ci::ResourceGroup.count }
+
+          expect(job.resource_group).to be_nil
+        end
+      end
+    end
+
+    context 'when a pipeline does not contain a job that requires a resource group' do
+      let!(:job) { build(:ci_build, project: project) }
+
+      it 'does not create any resource groups' do
+        expect { subject }.not_to change { Ci::ResourceGroup.count }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/seed/build_spec.rb

@@ -393,6 +393,10 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do
   describe '#to_resource' do
     subject { seed_build.to_resource }
 
+    before do
+      stub_feature_flags(create_deployment_in_separate_transaction: false)
+    end
+
     context 'when job is Ci::Build' do
       it { is_expected.to be_a(::Ci::Build) }
       it { is_expected.to be_valid }
@@ -443,6 +447,18 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do
         it_behaves_like 'deployment job'
         it_behaves_like 'ensures environment existence'
 
+        context 'when create_deployment_in_separate_transaction feature flag is enabled' do
+          before do
+            stub_feature_flags(create_deployment_in_separate_transaction: true)
+          end
+
+          it 'does not create any deployments nor environments' do
+            expect(subject.deployment).to be_nil
+            expect(Environment.count).to eq(0)
+            expect(Deployment.count).to eq(0)
+          end
+        end
+
         context 'when the environment name is invalid' do
           let(:attributes) { { name: 'deploy', ref: 'master', environment: '!!!' } }
 
@@ -496,6 +512,18 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do
         it 'returns a job with resource group' do
           expect(subject.resource_group).not_to be_nil
           expect(subject.resource_group.key).to eq('iOS')
+          expect(Ci::ResourceGroup.count).to eq(1)
+        end
+
+        context 'when create_deployment_in_separate_transaction feature flag is enabled' do
+          before do
+            stub_feature_flags(create_deployment_in_separate_transaction: true)
+          end
+
+          it 'does not create any resource groups' do
+            expect(subject.resource_group).to be_nil
+            expect(Ci::ResourceGroup.count).to eq(0)
+          end
         end
 
         context 'when resource group has $CI_ENVIRONMENT_NAME in it' do