Move persistent_ref.create into run_after_commit

Previously, persistent ref creation would hold transaction
causing significant delays and increase in 409 errors,
and in general slowness when picking CI jobs.

This moves that into run_after_commit hook that still
allows us to execute it, but releasing DB lock.

app/models/ci/build.rb

@@ -247,10 +247,11 @@ module Ci
       end
 
       after_transition pending: :running do |build|
-        build.pipeline.persistent_ref.create
         build.deployment&.run
 
         build.run_after_commit do
+          build.pipeline.persistent_ref.create
+
           BuildHooksWorker.perform_async(id)
         end
       end

changelogs/unreleased/create-persistent-ref-outside-of-transaction.yml

+---
+title: Move persistent_ref.create into run_after_commit
+merge_request: 20422
+author:
+type: fixed

spec/models/ci/build_spec.rb

@@ -3086,10 +3086,20 @@ describe Ci::Build do
     rescue StateMachines::InvalidTransition
     end
 
-    it 'ensures pipeline ref existence' do
-      expect(job.pipeline.persistent_ref).to receive(:create).once
+    context 'for pipeline ref existence' do
+      it 'ensures pipeline ref creation' do
+        expect(job.pipeline.persistent_ref).to receive(:create).once
 
-      run_job_without_exception
+        run_job_without_exception
+      end
+
+      it 'ensures that it is not run in database transaction' do
+        expect(job.pipeline.persistent_ref).to receive(:create) do
+          expect(Gitlab::Database).not_to be_inside_transaction
+        end
+
+        run_job_without_exception
+      end
     end
 
     shared_examples 'saves data on transition' do

spec/requests/api/runner_spec.rb

@@ -513,6 +513,16 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
             expect(json_response['features']).to eq(expected_features)
           end
 
+          it 'creates persistent ref' do
+            expect_any_instance_of(Ci::PersistentRef).to receive(:create_ref)
+              .with(job.sha, "refs/#{Repository::REF_PIPELINES}/#{job.commit_id}")
+
+            request_job info: { platform: :darwin }
+
+            expect(response).to have_gitlab_http_status(201)
+            expect(json_response['id']).to eq(job.id)
+          end
+
           context 'when job is made for tag' do
             let!(:job) { create(:ci_build, :tag, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) }