Add project level git depth setting

Introduce default_git_depth in project's CI/CD settings and set it to
50. Use it if there is no GIT_DEPTH variable specified. Apply this
default only to newly created projects and keep it nil for old ones
in order to not break pipelines that rely on non-shallow clones.

default_git_depth can be updated from CI/CD Settings in the UI, must be
either nil or integer between 0 and 1000 (incl).

Inherit default_git_depth from the origin project when forking projects.

MR pipelines are run on a MR ref (refs/merge-requests/:iid/merge) and it
contains unique commit (i.e. merge commit) which doesn't exist in the
other branch/tags refs. We need to add it cause otherwise it may break
pipelines for old projects that have already enabled Pipelines for merge
results and have git depth 0.

Document new default_git_depth project CI/CD setting

app/controllers/projects/settings/ci_cd_controller.rb

@@ -50,7 +50,8 @@ module Projects
           :runners_token, :builds_enabled, :build_allow_git_fetch,
           :build_timeout_human_readable, :build_coverage_regex, :public_builds,
           :auto_cancel_pending_pipelines, :ci_config_path,
-          auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy]
+          auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy],
+          ci_cd_settings_attributes: [:default_git_depth]
         )
       end
 

app/models/project.rb

@@ -292,6 +292,7 @@ class Project < ApplicationRecord
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :import_data
   accepts_nested_attributes_for :auto_devops, update_only: true
+  accepts_nested_attributes_for :ci_cd_settings, update_only: true
 
   accepts_nested_attributes_for :remote_mirrors,
                                 allow_destroy: true,
@@ -310,6 +311,7 @@ class Project < ApplicationRecord
   delegate :root_ancestor, to: :namespace, allow_nil: true
   delegate :last_pipeline, to: :commit, allow_nil: true
   delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true
+  delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings
 
   # Validations
   validates :creator, presence: true, on: :create

app/models/project_ci_cd_setting.rb

@@ -6,6 +6,18 @@ class ProjectCiCdSetting < ApplicationRecord
   # The version of the schema that first introduced this model/table.
   MINIMUM_SCHEMA_VERSION = 20180403035759
 
+  DEFAULT_GIT_DEPTH = 50
+
+  before_create :set_default_git_depth, unless: :default_git_depth?
+
+  validates :default_git_depth,
+    numericality: {
+      only_integer: true,
+      greater_than_or_equal_to: 0,
+      less_than_or_equal_to: 1000
+    },
+    allow_nil: true
+
   def self.available?
     @available ||=
       ActiveRecord::Migrator.current_version >= MINIMUM_SCHEMA_VERSION
@@ -15,4 +27,10 @@ class ProjectCiCdSetting < ApplicationRecord
     @available = nil
     super
   end
+
+  private
+
+  def set_default_git_depth
+    self.default_git_depth = DEFAULT_GIT_DEPTH
+  end
 end

app/presenters/ci/build_runner_presenter.rb

@@ -25,14 +25,16 @@ module Ci
     end
 
     def git_depth
-      strong_memoize(:git_depth) do
-        git_depth = variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }&.dig(:value)
-        git_depth.to_i
-      end
+      if git_depth_variable
+        git_depth_variable[:value]
+      else
+        project.default_git_depth
+      end.to_i
     end
 
     def refspecs
       specs = []
+      specs << refspec_for_merge_request_ref if merge_request_ref?
 
       if git_depth > 0
         specs << refspec_for_branch(ref) if branch? || legacy_detached_merge_request_pipeline?
@@ -42,8 +44,6 @@ module Ci
         specs << refspec_for_tag
       end
 
-      specs << refspec_for_merge_request_ref if merge_request_ref?
-
       specs
     end
 
@@ -89,5 +89,11 @@ module Ci
     def refspec_for_merge_request_ref
       "+#{ref}:#{ref}"
     end
+
+    def git_depth_variable
+      strong_memoize(:git_depth_variable) do
+        variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }
+      end
+    end
   end
 end

app/services/projects/fork_service.rb

@@ -36,18 +36,19 @@ module Projects
 
     def fork_new_project
       new_params = {
-        visibility_level:       allowed_visibility_level,
-        description:            @project.description,
-        name:                   target_name,
-        path:                   target_path,
-        shared_runners_enabled: @project.shared_runners_enabled,
-        namespace_id:           target_namespace.id,
-        fork_network:           fork_network,
+        visibility_level:          allowed_visibility_level,
+        description:               @project.description,
+        name:                      target_name,
+        path:                      target_path,
+        shared_runners_enabled:    @project.shared_runners_enabled,
+        namespace_id:              target_namespace.id,
+        fork_network:              fork_network,
+        ci_cd_settings_attributes: { default_git_depth: @project.default_git_depth },
         # We need to assign the fork network membership after the project has
         # been instantiated to avoid ActiveRecord trying to create it when
         # initializing the project, as that would cause a foreign key constraint
         # exception.
-        relations_block:        -> (project) { build_fork_network_member(project) }
+        relations_block:           -> (project) { build_fork_network_member(project) }
       }
 
       if @project.avatar.present? && @project.avatar.image?
@@ -56,7 +57,10 @@ module Projects
 
       new_params.merge!(@project.object_pool_params)
 
-      new_project = CreateService.new(current_user, new_params).execute
+      new_project = CreateService.new(current_user, new_params).execute do |p|
+        p.build_ci_cd_settings(default_git_depth: @project.default_git_depth)
+      end
+
       return new_project unless new_project.persisted?
 
       # Set the forked_from_project relation after saving to avoid having to

app/views/projects/settings/ci_cd/_form.html.haml

@@ -24,6 +24,14 @@
               %span.descr
                 = _("Faster as it re-uses the project workspace (falling back to clone if it doesn't exist)")
 
+        %hr
+        .form-group
+          = f.fields_for :ci_cd_settings_attributes, @project.ci_cd_settings do |form|
+            = form.label :default_git_depth, _('Git shallow clone'), class: 'label-bold'
+            = form.number_field :default_git_depth, { class: 'form-control', min: 0, max: 1000 }
+            %p.form-text.text-muted
+              = _('The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time.')
+
         %hr
         .form-group
           = f.label :build_timeout_human_readable, _('Timeout'), class: 'label-bold'

changelogs/unreleased/62418-project-default-git-depth.yml

+---
+title: Add project level git depth CI/CD setting
+merge_request: 28919
+author:
+type: added

db/migrate/20190530042141_add_default_git_depth_to_ci_cd_settings.rb

+# frozen_string_literal: true
+
+class AddDefaultGitDepthToCiCdSettings < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :project_ci_cd_settings, :default_git_depth, :integer
+  end
+end

db/schema.rb

@@ -1644,6 +1644,7 @@ ActiveRecord::Schema.define(version: 20190530154715) do
     t.boolean "group_runners_enabled", default: true, null: false
     t.boolean "merge_pipelines_enabled"
     t.boolean "merge_trains_enabled", default: false, null: false
+    t.integer "default_git_depth"
     t.index ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
   end
 

doc/user/project/pipelines/settings.md

@@ -20,6 +20,22 @@ There are two options. Using:
 The default Git strategy can be overridden by the [GIT_STRATEGY variable](../../../ci/yaml/README.md#git-strategy)
 in `.gitlab-ci.yml`.
 
+## Git shallow clone
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/28919) in GitLab 12.0.
+
+NOTE: **Note**: As of GitLab 12.0, newly created projects will automaticallyl have a default 
+`git depth` value of `50`.
+
+It is possible to limit the number of changes that GitLab CI/CD will fetch when cloning 
+a repository. Setting a limit to `git depth` can speed up Pipelines execution. Maximum
+allowed value is `1000`.
+
+To disable shallow clone and make GitLab CI/CD fetch all branches and tags each time,
+keep the value empty or set to `0`.
+
+This value can also be [overridden by `GIT_DEPTH`](../../../ci/large_repositories/index.md#shallow-cloning) variable in `.gitlab-ci.yml` file.
+
 ## Timeout
 
 Timeout defines the maximum amount of time in minutes that a job is able run.

locale/gitlab.pot

@@ -4698,6 +4698,9 @@ msgstr ""
 msgid "Git revision"
 msgstr ""
 
+msgid "Git shallow clone"
+msgstr ""
+
 msgid "Git strategy for pipelines"
 msgstr ""
 
@@ -9978,6 +9981,9 @@ msgstr ""
 msgid "The name %{entryName} is already taken in this directory."
 msgstr ""
 
+msgid "The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time."
+msgstr ""
+
 msgid "The number of times an upload record could not find its file"
 msgstr ""
 

spec/controllers/projects/settings/ci_cd_controller_spec.rb

@@ -200,6 +200,21 @@ describe Projects::Settings::CiCdController do
           expect(response).to redirect_to(namespace_project_settings_ci_cd_path)
         end
       end
+
+      context 'when default_git_depth is not specified' do
+        let(:params) { { ci_cd_settings_attributes: { default_git_depth: 10 } } }
+
+        before do
+          project.ci_cd_settings.update!(default_git_depth: nil)
+        end
+
+        it 'set specified git depth' do
+          subject
+
+          project.reload
+          expect(project.default_git_depth).to eq(10)
+        end
+      end
     end
   end
 end

spec/lib/gitlab/background_migration/delete_diff_files_spec.rb

@@ -3,6 +3,12 @@ require 'spec_helper'
 # rubocop:disable RSpec/FactoriesInMigrationSpecs
 describe Gitlab::BackgroundMigration::DeleteDiffFiles, :migration, :sidekiq, schema: 20180619121030 do
   describe '#perform' do
+    before do
+      # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+      allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
+      allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
+    end
+
     context 'when diff files can be deleted' do
       let(:merge_request) { create(:merge_request, :merged) }
       let!(:merge_request_diff) do

spec/lib/gitlab/background_migration/populate_external_pipeline_source_spec.rb

@@ -9,6 +9,9 @@ describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration
   before do
     # This migration was created before we introduced metadata configs
     stub_feature_flags(ci_build_metadata_config: false)
+    # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
   end
 
   let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }

spec/migrations/remove_orphaned_label_links_spec.rb

@@ -10,6 +10,12 @@ describe RemoveOrphanedLabelLinks, :migration do
   let(:project) { create(:project) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
   let(:label) { create_label }
 
+  before do
+    # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
+  end
+
   context 'add foreign key on label_id' do
     let!(:label_link_with_label) { create_label_link(label_id: label.id) }
     let!(:label_link_without_label) { create_label_link(label_id: nil) }

spec/models/project_ci_cd_setting_spec.rb

@@ -21,4 +21,26 @@ describe ProjectCiCdSetting do
       2.times { described_class.available? }
     end
   end
+
+  describe 'validations' do
+    it { is_expected.to validate_numericality_of(:default_git_depth).only_integer.is_greater_than_or_equal_to(0).is_less_than_or_equal_to(1000).allow_nil }
+  end
+
+  describe '#default_git_depth' do
+    let(:default_value) { described_class::DEFAULT_GIT_DEPTH }
+
+    it 'sets default value for new records' do
+      project = create(:project)
+
+      expect(project.ci_cd_settings.default_git_depth).to eq(default_value)
+    end
+
+    it 'does not set default value if present' do
+      project = build(:project)
+      project.build_ci_cd_settings(default_git_depth: 42)
+      project.save!
+
+      expect(project.reload.ci_cd_settings.default_git_depth).to eq(42)
+    end
+  end
 end

spec/presenters/ci/build_runner_presenter_spec.rb

@@ -119,23 +119,23 @@ describe Ci::BuildRunnerPresenter do
   end
 
   describe '#git_depth' do
-    subject { presenter.git_depth }
-
     let(:build) { create(:ci_build) }
 
-    it 'returns the correct git depth' do
-      is_expected.to eq(0)
-    end
+    subject(:git_depth) { presenter.git_depth }
 
     context 'when GIT_DEPTH variable is specified' do
       before do
         create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
       end
 
-      it 'returns the correct git depth' do
-        is_expected.to eq(1)
+      it 'returns its value' do
+        expect(git_depth).to eq(1)
       end
     end
+
+    it 'defaults to git depth setting for the project' do
+      expect(git_depth).to eq(build.project.default_git_depth)
+    end
   end
 
   describe '#refspecs' do
@@ -144,24 +144,24 @@ describe Ci::BuildRunnerPresenter do
     let(:build) { create(:ci_build) }
 
     it 'returns the correct refspecs' do
-      is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
-                                     '+refs/heads/*:refs/remotes/origin/*')
+      is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
     end
 
-    context 'when GIT_DEPTH variable is specified' do
-      before do
-        create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
-      end
+    context 'when ref is tag' do
+      let(:build) { create(:ci_build, :tag) }
 
       it 'returns the correct refspecs' do
-        is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
+        is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}")
       end
 
-      context 'when ref is tag' do
-        let(:build) { create(:ci_build, :tag) }
+      context 'when GIT_DEPTH is zero' do
+        before do
+          create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
+        end
 
         it 'returns the correct refspecs' do
-          is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}")
+          is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
+                                         '+refs/heads/*:refs/remotes/origin/*')
         end
       end
     end
@@ -173,17 +173,27 @@ describe Ci::BuildRunnerPresenter do
 
       it 'returns the correct refspecs' do
         is_expected
-          .to contain_exactly('+refs/heads/*:refs/remotes/origin/*',
-                              '+refs/tags/*:refs/tags/*',
-                              '+refs/merge-requests/1/head:refs/merge-requests/1/head')
+          .to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head')
+      end
+
+      context 'when GIT_DEPTH is zero' do
+        before do
+          create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
+        end
+
+        it 'returns the correct refspecs' do
+          is_expected
+            .to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head',
+                                '+refs/heads/*:refs/remotes/origin/*',
+                                '+refs/tags/*:refs/tags/*')
+        end
       end
 
       context 'when pipeline is legacy detached merge request pipeline' do
         let(:merge_request) { create(:merge_request, :with_legacy_detached_merge_request_pipeline) }
 
         it 'returns the correct refspecs' do
-          is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
-                                         '+refs/heads/*:refs/remotes/origin/*')
+          is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
         end
       end
     end

spec/requests/api/runner_spec.rb

@@ -444,8 +444,8 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               'sha' => job.sha,
               'before_sha' => job.before_sha,
               'ref_type' => 'branch',
-              'refspecs' => %w[+refs/heads/*:refs/remotes/origin/* +refs/tags/*:refs/tags/*],
-              'depth' => 0 }
+              'refspecs' => ["+refs/heads/#{job.ref}:refs/remotes/origin/#{job.ref}"],
+              'depth' => project.default_git_depth }
           end
 
           let(:expected_steps) do
@@ -531,7 +531,11 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               end
             end
 
-            context 'when GIT_DEPTH is not specified' do
+            context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
+              before do
+                project.update!(default_git_depth: nil)
+              end
+
               it 'specifies refspecs' do
                 request_job
 
@@ -587,7 +591,11 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               end
             end
 
-            context 'when GIT_DEPTH is not specified' do
+            context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
+              before do
+                project.update!(default_git_depth: nil)
+              end
+
               it 'specifies refspecs' do
                 request_job
 

spec/services/projects/fork_service_spec.rb

@@ -145,6 +145,14 @@ describe Projects::ForkService do
         end
       end
 
+      context "CI/CD settings" do
+        it "inherits default_git_depth from the origin project" do
+          @from_project.update(default_git_depth: 42)
+          @to_project = fork_project(@from_project, @to_user)
+          expect(@to_project.default_git_depth).to eq(42)
+        end
+      end
+
       context "when project has restricted visibility level" do
         context "and only one visibility level is restricted" do
           before do