Merge branch 'dedicated-pipeline-ref' into 'master'

Create a persistent ref per pipeline and let runners fetch it

See merge request gitlab-org/gitlab!17043

app/models/ci/build.rb

@@ -234,6 +234,7 @@ module Ci
       end
 
       after_transition pending: :running do |build|
+        build.pipeline.persistent_ref.create
         build.deployment&.run
 
         build.run_after_commit do

app/models/ci/persistent_ref.rb

+# frozen_string_literal: true
+
+module Ci
+  ##
+  # The persistent pipeline ref to ensure runners can safely fetch source code
+  # even if force-push/source-branch-deletion happens.
+  class PersistentRef
+    include ActiveModel::Model
+
+    attr_accessor :pipeline
+
+    delegate :project, :sha, to: :pipeline
+    delegate :repository, to: :project
+    delegate :ref_exists?, :create_ref, :delete_refs, to: :repository
+
+    def exist?
+      return unless enabled?
+
+      ref_exists?(path)
+    rescue
+      false
+    end
+
+    def create
+      return unless enabled? && !exist?
+
+      create_ref(sha, path)
+    rescue => e
+      Gitlab::Sentry
+        .track_acceptable_exception(e, extra: { pipeline_id: pipeline.id })
+    end
+
+    def delete
+      return unless enabled?
+
+      delete_refs(path)
+    rescue Gitlab::Git::Repository::NoRepository
+      # no-op
+    rescue => e
+      Gitlab::Sentry
+        .track_acceptable_exception(e, extra: { pipeline_id: pipeline.id })
+    end
+
+    def path
+      "refs/#{Repository::REF_PIPELINES}/#{pipeline.id}"
+    end
+
+    private
+
+    def enabled?
+      Feature.enabled?(:depend_on_persistent_pipeline_ref, project)
+    end
+  end
+end

app/models/ci/pipeline.rb

@@ -174,6 +174,8 @@ module Ci
 
       after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline|
         pipeline.run_after_commit do
+          pipeline.persistent_ref.delete
+
           pipeline.all_merge_requests.each do |merge_request|
             next unless merge_request.auto_merge_enabled?
 
@@ -853,6 +855,10 @@ module Ci
       end
     end
 
+    def persistent_ref
+      @persistent_ref ||= PersistentRef.new(pipeline: self)
+    end
+
     private
 
     def ci_yaml_from_repo

app/models/repository.rb

@@ -6,6 +6,7 @@ class Repository
   REF_MERGE_REQUEST = 'merge-requests'
   REF_KEEP_AROUND = 'keep-around'
   REF_ENVIRONMENTS = 'environments'
+  REF_PIPELINES = 'pipelines'
 
   ARCHIVE_CACHE_TIME = 60 # Cache archives referred to by a (mutable) ref for 1 minute
   ARCHIVE_CACHE_TIME_IMMUTABLE = 3600 # Cache archives referred to by an immutable reference for 1 hour
@@ -16,7 +17,7 @@ class Repository
     replace
     #{REF_ENVIRONMENTS}
     #{REF_KEEP_AROUND}
-    #{REF_ENVIRONMENTS}
+    #{REF_PIPELINES}
   ].freeze
 
   include Gitlab::RepositoryCacheAdapter

app/presenters/ci/build_runner_presenter.rb

@@ -34,7 +34,8 @@ module Ci
 
     def refspecs
       specs = []
-      specs << refspec_for_merge_request_ref if merge_request_ref?
+      specs << refspec_for_pipeline_ref if merge_request_ref?
+      specs << refspec_for_persistent_ref if persistent_ref_exist?
 
       if git_depth > 0
         specs << refspec_for_branch(ref) if branch? || legacy_detached_merge_request_pipeline?
@@ -86,10 +87,22 @@ module Ci
       "+#{Gitlab::Git::TAG_REF_PREFIX}#{ref}:#{RUNNER_REMOTE_TAG_PREFIX}#{ref}"
     end
 
-    def refspec_for_merge_request_ref
+    def refspec_for_pipeline_ref
       "+#{ref}:#{ref}"
     end
 
+    def refspec_for_persistent_ref
+      "+#{persistent_ref_path}:#{persistent_ref_path}"
+    end
+
+    def persistent_ref_exist?
+      pipeline.persistent_ref.exist?
+    end
+
+    def persistent_ref_path
+      pipeline.persistent_ref.path
+    end
+
     def git_depth_variable
       strong_memoize(:git_depth_variable) do
         variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }

changelogs/unreleased/dedicated-pipeline-ref.yml

+---
+title: Create a persistent ref per pipeline for keeping pipelines run from force-push
+  and merged results
+merge_request: 17043
+author:
+type: fixed

doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md

@@ -93,6 +93,17 @@ To check these feature flag values, please ask administrator to execute the foll
 > Feature.enable(:ci_use_merge_request_ref)         # Enable the feature flag.
 ```
 
+### Intermittently pipelines fail by `fatal: reference is not a tree:` error
+
+Since pipelines for merged results are a run on a merge ref of a merge request
+(`refs/merge-requests/<iid>/merge`), the git-reference could be overwritten at an
+unexpected timing, for example, when a source or target branch is advanced.
+In this case, the pipeline fails because of `fatal: reference is not a tree:` error,
+which indicates that the checkout-SHA is not found in the merge ref.
+
+This behavior was improved at GitLab 12.4 by introducing [Persistent pipeline refs](../../pipelines.md#persistent-pipeline-refs).
+You should be able to create pipelines at any timings without concerning the error.
+
 ## Using Merge Trains **(PREMIUM)**
 
 By enabling [Pipelines for merged results](#pipelines-for-merged-results-premium),

doc/ci/pipelines.md

@@ -405,3 +405,44 @@ branches, avoiding untrusted code to be executed on the protected runner and
 preserving deployment keys and other credentials from being unintentionally
 accessed. In order to ensure that jobs intended to be executed on protected
 runners will not use regular runners, they must be tagged accordingly.
+
+## Persistent pipeline refs
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/17043) in GitLab 12.4.
+
+Previously, you'd have encountered unexpected pipeline failures when you force-pushed
+a branch to its remote repository. To illustrate the problem, suppose you've had the current workflow:
+
+1. A user creates a feature branch named `example` and pushes it to a remote repository.
+1. A new pipeline starts running on the `example` branch.
+1. A user rebases the `example` branch on the latest `master` branch and force-pushes it to its remote repository.
+1. A new pipeline starts running on the `example` branch again, however,
+   the previous pipeline (2) fails because of `fatal: reference is not a tree:` error.
+
+This is because the previous pipeline cannot find a checkout-SHA (which associated with the pipeline record)
+from the `example` branch that the commit history has already been overwritten by the force-push.
+Similarly, [Pipelines for merged results](merge_request_pipelines/pipelines_for_merged_results/index.md)
+might have failed intermittently due to [the same reason](merge_request_pipelines/pipelines_for_merged_results/index.md#intermittently-pipelines-fail-by-fatal-reference-is-not-a-tree-error).
+
+As of GitLab 12.4, we've improved this behavior by persisting pipeline refs exclusively.
+To illustrate its life cycle:
+
+1. A pipeline is created on a feature branch named `example`.
+1. A persistent pipeline ref is created at `refs/pipelines/<pipeline-id>`,
+   which retains the checkout-SHA of the associated pipeline record.
+   This persistent ref stays intact during the pipeline execution,
+   even if the commit history of the `example` branch has been overwritten by force-push.
+1. GitLab Runner fetches the persistent pipeline ref and gets source code from the checkout-SHA.
+1. When the pipeline finished, its persistent ref is cleaned up in a background process.
+
+NOTE: **NOTE**: At this moment, this feature is off dy default and can be manually enabled
+by enabling `depend_on_persistent_pipeline_ref` feature flag, however, we'd remove this
+feature flag and make it enabled by deafult by the day we release 12.4 _if we don't find any issues_.
+If you'd be interested in manually turning on this behavior, please ask the administrator
+to execute the following commands in rails console.
+
+```shell
+> sudo gitlab-rails console                                        # Login to Rails console of GitLab instance.
+> project = Project.find_by_full_path('namespace/project-name')    # Get the project instance.
+> Feature.enable(:depend_on_persistent_pipeline_ref, project)      # Enable the feature flag.
+```

spec/models/ci/build_spec.rb

@@ -3079,6 +3079,12 @@ describe Ci::Build do
     rescue StateMachines::InvalidTransition
     end
 
+    it 'ensures pipeline ref existence' do
+      expect(job.pipeline.persistent_ref).to receive(:create).once
+
+      run_job_without_exception
+    end
+
     shared_examples 'saves data on transition' do
       it 'saves timeout' do
         expect { job.run! }.to change { job.reload.ensure_metadata.timeout }.from(nil).to(expected_timeout)

spec/models/ci/persistent_ref_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::PersistentRef do
+  it 'cleans up persistent refs after pipeline finished' do
+    pipeline = create(:ci_pipeline, :running)
+
+    expect(pipeline.persistent_ref).to receive(:delete).once
+
+    pipeline.succeed!
+  end
+
+  context '#exist?' do
+    subject { pipeline.persistent_ref.exist? }
+
+    let(:pipeline) { create(:ci_pipeline, sha: sha, project: project) }
+    let(:project) { create(:project, :repository) }
+    let(:sha) { project.repository.commit.sha }
+
+    context 'when a persistent ref does not exist' do
+      it { is_expected.to eq(false) }
+    end
+
+    context 'when a persistent ref exists' do
+      before do
+        pipeline.persistent_ref.create
+      end
+
+      it { is_expected.to eq(true) }
+    end
+  end
+
+  context '#create' do
+    subject { pipeline.persistent_ref.create }
+
+    let(:pipeline) { create(:ci_pipeline, sha: sha, project: project) }
+    let(:project) { create(:project, :repository) }
+    let(:sha) { project.repository.commit.sha }
+
+    context 'when a persistent ref does not exist' do
+      it 'creates a persistent ref' do
+        subject
+
+        expect(pipeline.persistent_ref).to be_exist
+      end
+
+      context 'when depend_on_persistent_pipeline_ref feature flag is disabled' do
+        before do
+          stub_feature_flags(depend_on_persistent_pipeline_ref: false)
+        end
+
+        it 'does not create a persistent ref' do
+          expect(project.repository).not_to receive(:create_ref)
+
+          subject
+        end
+      end
+
+      context 'when sha does not exist in the repository' do
+        let(:sha) { 'not-exist' }
+
+        it 'fails to create a persistent ref' do
+          subject
+
+          expect(pipeline.persistent_ref).not_to be_exist
+        end
+      end
+    end
+
+    context 'when a persistent ref already exists' do
+      before do
+        pipeline.persistent_ref.create
+      end
+
+      it 'does not create a persistent ref' do
+        expect(project.repository).not_to receive(:create_ref)
+
+        subject
+      end
+    end
+  end
+
+  context '#delete' do
+    subject { pipeline.persistent_ref.delete }
+
+    let(:pipeline) { create(:ci_pipeline, sha: sha, project: project) }
+    let(:project) { create(:project, :repository) }
+    let(:sha) { project.repository.commit.sha }
+
+    context 'when a persistent ref exists' do
+      before do
+        pipeline.persistent_ref.create
+      end
+
+      it 'deletes the ref' do
+        expect { subject }.to change { pipeline.persistent_ref.exist? }
+                          .from(true).to(false)
+      end
+    end
+
+    context 'when a persistent ref does not exist' do
+      it 'does not raise an error' do
+        expect { subject }.not_to raise_error
+      end
+    end
+  end
+end

spec/models/ci/pipeline_spec.rb

@@ -1318,6 +1318,16 @@ describe Ci::Pipeline, :mailer do
     let(:build_b) { create_build('build2', queued_at: 0) }
     let(:build_c) { create_build('build3', queued_at: 0) }
 
+    %w[succeed! drop! cancel! skip!].each do |action|
+      context "when the pipeline recieved #{action} event" do
+        it 'deletes a persistent ref' do
+          expect(pipeline.persistent_ref).to receive(:delete).once
+
+          pipeline.public_send(action)
+        end
+      end
+    end
+
     describe '#duration' do
       context 'when multiple builds are finished' do
         before do

spec/presenters/ci/build_runner_presenter_spec.rb

@@ -207,5 +207,22 @@ describe Ci::BuildRunnerPresenter do
         end
       end
     end
+
+    context 'when persistent pipeline ref exists' do
+      let(:project) { create(:project, :repository) }
+      let(:sha) { project.repository.commit.sha }
+      let(:pipeline) { create(:ci_pipeline, sha: sha, project: project) }
+      let(:build) { create(:ci_build, pipeline: pipeline) }
+
+      before do
+        pipeline.persistent_ref.create
+      end
+
+      it 'exposes the persistent pipeline ref' do
+        is_expected
+          .to contain_exactly("+refs/pipelines/#{pipeline.id}:refs/pipelines/#{pipeline.id}",
+                              "+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
+      end
+    end
   end
 end

spec/services/ci/process_pipeline_service_spec.rb

@@ -422,6 +422,18 @@ describe Ci::ProcessPipelineService, '#execute' do
     end
   end
 
+  context 'when an exception is raised during a persistent ref creation' do
+    before do
+      successful_build('test', stage_idx: 0)
+
+      allow_any_instance_of(Ci::PersistentRef).to receive(:delete_refs) { raise ArgumentError }
+    end
+
+    it 'process the pipeline' do
+      expect { process_pipeline }.not_to raise_error
+    end
+  end
+
   context 'when there are manual action in earlier stages' do
     context 'when first stage has only optional manual actions' do
       before do
@@ -907,6 +919,10 @@ describe Ci::ProcessPipelineService, '#execute' do
     create(:ci_build, :created, pipeline: pipeline, name: name, **opts)
   end
 
+  def successful_build(name, **opts)
+    create(:ci_build, :success, pipeline: pipeline, name: name, **opts)
+  end
+
   def delayed_options
     { when: 'delayed', options: { script: %w(echo), start_in: '1 minute' } }
   end

spec/services/ci/register_job_service_spec.rb

@@ -501,6 +501,22 @@ module Ci
           expect(pending_job).to be_archived_failure
         end
       end
+
+      context 'when an exception is raised during a persistent ref creation' do
+        before do
+          allow_any_instance_of(Ci::PersistentRef).to receive(:exist?) { false }
+          allow_any_instance_of(Ci::PersistentRef).to receive(:create_ref) { raise ArgumentError }
+        end
+
+        subject { execute(specific_runner, {}) }
+
+        it 'picks the build' do
+          expect(subject).to eq(pending_job)
+
+          pending_job.reload
+          expect(pending_job).to be_running
+        end
+      end
     end
 
     describe '#register_success' do