Automatic merge of gitlab-org/gitlab-ce master

app/controllers/projects/settings/ci_cd_controller.rb

@@ -50,7 +50,8 @@ module Projects
           :runners_token, :builds_enabled, :build_allow_git_fetch,
           :build_timeout_human_readable, :build_coverage_regex, :public_builds,
           :auto_cancel_pending_pipelines, :ci_config_path,
-          auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy]
+          auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy],
+          ci_cd_settings_attributes: [:default_git_depth]
         )
       end
 

app/models/project.rb

@@ -292,6 +292,7 @@ class Project < ApplicationRecord
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :import_data
   accepts_nested_attributes_for :auto_devops, update_only: true
+  accepts_nested_attributes_for :ci_cd_settings, update_only: true
 
   accepts_nested_attributes_for :remote_mirrors,
                                 allow_destroy: true,
@@ -310,6 +311,7 @@ class Project < ApplicationRecord
   delegate :root_ancestor, to: :namespace, allow_nil: true
   delegate :last_pipeline, to: :commit, allow_nil: true
   delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true
+  delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings
 
   # Validations
   validates :creator, presence: true, on: :create

app/models/project_ci_cd_setting.rb

@@ -6,6 +6,18 @@ class ProjectCiCdSetting < ApplicationRecord
   # The version of the schema that first introduced this model/table.
   MINIMUM_SCHEMA_VERSION = 20180403035759
 
+  DEFAULT_GIT_DEPTH = 50
+
+  before_create :set_default_git_depth
+
+  validates :default_git_depth,
+    numericality: {
+      only_integer: true,
+      greater_than_or_equal_to: 0,
+      less_than_or_equal_to: 1000
+    },
+    allow_nil: true
+
   def self.available?
     @available ||=
       ActiveRecord::Migrator.current_version >= MINIMUM_SCHEMA_VERSION
@@ -15,6 +27,12 @@ class ProjectCiCdSetting < ApplicationRecord
     @available = nil
     super
   end
+
+  private
+
+  def set_default_git_depth
+    self.default_git_depth ||= DEFAULT_GIT_DEPTH
+  end
 end
 
 ProjectCiCdSetting.prepend(EE::ProjectCiCdSetting)

app/presenters/ci/build_runner_presenter.rb

@@ -25,14 +25,16 @@ module Ci
     end
 
     def git_depth
-      strong_memoize(:git_depth) do
-        git_depth = variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }&.dig(:value)
-        git_depth.to_i
-      end
+      if git_depth_variable
+        git_depth_variable[:value]
+      else
+        project.default_git_depth
+      end.to_i
     end
 
     def refspecs
       specs = []
+      specs << refspec_for_merge_request_ref if merge_request_ref?
 
       if git_depth > 0
         specs << refspec_for_branch(ref) if branch? || legacy_detached_merge_request_pipeline?
@@ -42,8 +44,6 @@ module Ci
         specs << refspec_for_tag
       end
 
-      specs << refspec_for_merge_request_ref if merge_request_ref?
-
       specs
     end
 
@@ -89,5 +89,11 @@ module Ci
     def refspec_for_merge_request_ref
       "+#{ref}:#{ref}"
     end
+
+    def git_depth_variable
+      strong_memoize(:git_depth_variable) do
+        variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }
+      end
+    end
   end
 end

app/services/projects/fork_service.rb

@@ -36,18 +36,22 @@ module Projects
 
     def fork_new_project
       new_params = {
-        visibility_level:       allowed_visibility_level,
-        description:            @project.description,
-        name:                   target_name,
-        path:                   target_path,
-        shared_runners_enabled: @project.shared_runners_enabled,
-        namespace_id:           target_namespace.id,
-        fork_network:           fork_network,
+        visibility_level:          allowed_visibility_level,
+        description:               @project.description,
+        name:                      target_name,
+        path:                      target_path,
+        shared_runners_enabled:    @project.shared_runners_enabled,
+        namespace_id:              target_namespace.id,
+        fork_network:              fork_network,
+        # We need to set default_git_depth to 0 for the forked project when
+        # @project.default_git_depth is nil in order to keep the same behaviour
+        # and not get ProjectCiCdSetting::DEFAULT_GIT_DEPTH set on create
+        ci_cd_settings_attributes: { default_git_depth: @project.default_git_depth || 0 },
         # We need to assign the fork network membership after the project has
         # been instantiated to avoid ActiveRecord trying to create it when
         # initializing the project, as that would cause a foreign key constraint
         # exception.
-        relations_block:        -> (project) { build_fork_network_member(project) }
+        relations_block:           -> (project) { build_fork_network_member(project) }
       }
 
       if @project.avatar.present? && @project.avatar.image?

app/views/projects/settings/ci_cd/_form.html.haml

@@ -24,6 +24,14 @@
               %span.descr
                 = _("Faster as it re-uses the project workspace (falling back to clone if it doesn't exist)")
 
+        %hr
+        .form-group
+          = f.fields_for :ci_cd_settings_attributes, @project.ci_cd_settings do |form|
+            = form.label :default_git_depth, _('Git shallow clone'), class: 'label-bold'
+            = form.number_field :default_git_depth, { class: 'form-control', min: 0, max: 1000 }
+            %p.form-text.text-muted
+              = _('The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time.')
+
         %hr
         .form-group
           = f.label :build_timeout_human_readable, _('Timeout'), class: 'label-bold'

changelogs/unreleased/50070-legacy-attachments.yml

+---
+title: Migrate legacy uploads out of deprecated paths
+merge_request: 24679
+author:
+type: other

changelogs/unreleased/62418-project-default-git-depth.yml

+---
+title: Add project level git depth CI/CD setting
+merge_request: 28919
+author:
+type: added

db/migrate/20190114184258_migrate_legacy_attachments.rb

+# frozen_string_literal: true
+
+class MigrateLegacyAttachments < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  MIGRATION = 'MigrateLegacyUploads'.freeze
+  BATCH_SIZE = 5000
+  DELAY_INTERVAL = 5.minutes.to_i
+
+  class Upload < ActiveRecord::Base
+    self.table_name = 'uploads'
+
+    include ::EachBatch
+  end
+
+  def up
+    Upload.where(uploader: 'AttachmentUploader').each_batch(of: BATCH_SIZE) do |relation, index|
+      start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+      delay = index * DELAY_INTERVAL
+
+      BackgroundMigrationWorker.perform_in(delay, MIGRATION, [start_id, end_id])
+    end
+  end
+
+  # not needed
+  def down
+  end
+end

db/migrate/20190530042141_add_default_git_depth_to_ci_cd_settings.rb

+# frozen_string_literal: true
+
+class AddDefaultGitDepthToCiCdSettings < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :project_ci_cd_settings, :default_git_depth, :integer
+  end
+end

db/schema.rb

@@ -2368,6 +2368,7 @@ ActiveRecord::Schema.define(version: 20190603124955) do
     t.boolean "group_runners_enabled", default: true, null: false
     t.boolean "merge_pipelines_enabled"
     t.boolean "merge_trains_enabled", default: false, null: false
+    t.integer "default_git_depth"
     t.index ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
   end
 

doc/administration/troubleshooting/migration.md

+# Migrations problems
+
+## Legacy upload migration
+
+> Introduced in GitLab 12.0.
+
+  The migration takes all attachments uploaded by legacy `AttachmentUploader` and
+  migrate them to the path that current uploaders expect.
+
+Although it should not usually happen there could possibly be some attachments belonging to
+LegacyDiffNotes. These attachments can't be seen before running the migration by users and
+they should not be present in your instance.
+
+However, if you have some of them, you will need to handle them manually.
+You can find the ids of failed notes in logs as "MigrateLegacyUploads: LegacyDiffNote"
+
+1. Run a Rails console:
+
+    ```sh
+    sudo gitlab-rails console production
+    ```
+
+    or for source installs:
+
+    ```sh
+    bundle exec rails console production
+    ```
+
+ 1. Check the failed upload and find the note (you can see their ids in the logs)
+
+    ```ruby
+    upload = Upload.find(upload_id)
+    note = Note.find(note_id)
+    ```
+
+
+ 1. Check the path - it should contain `system/note/attachment`
+
+    ```ruby
+    upload.absolut_path
+    ```
+    
+ 1. Check the path in the uploader - it should differ from the upload path and should contain  `system/legacy_diff_note`
+ 
+    ```ruby
+    uploader = upload.build_uploader
+    uploader.file
+    ```
+    
+ 1. First, you need to move the file to the path that is expected from the uploader
+
+    ```ruby
+    old_path = upload.absolute_path
+    new_path = upload.absolute_path.sub('-/system/note/attachment', '-/system/legacy_diff_note')
+    new_dir = File.dirname(new_path)
+    FileUtils.mkdir_p(new_dir)
+    
+    FileUtils.mv(old_path, new_path)
+    ```
+
+ 1. You then need to move the file to the `FileUploader` and create a new `Upload` object
+ 
+    ```ruby
+    file_uploader = UploadService.new(note.project, File.read(new_path)).execute
+    ```
+ 
+ 1. And update the legacy note to contain the file.
+ 
+    ```ruby
+    new_text = "#{note.note} \n #{file_uploader.markdown_link}"
+    note.update!(
+      note: new_text
+    )
+    ```
+
+ 1. And finally, you can remove the old upload
+ 
+    ```ruby
+    upload.destroy
+    ```
+
+If you have any problems feel free to contact [GitLab Support](https://about.gitlab.com/support/).

doc/user/application_security/dependency_scanning/index.md

@@ -225,6 +225,17 @@ vulnerabilities in your groups and projects. Read more about the
 Once a vulnerability is found, you can interact with it. Read more on how to
 [interact with the vulnerabilities](../index.md#interacting-with-the-vulnerabilities).
 
+## Dependency List
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/10075)
+in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.0.
+
+An additional benefit of Dependency Scanning is the ability to get a list of your project's dependencies with their versions. 
+
+This list can be generated only for [supported languages and package managers](#supported-languages-and-package-managers).
+
+To see the generated dependency list, navigate to the Dependency List page under your project's left sidebar menu **Project > Dependency List**.
+
 ## Contributing to the vulnerability database
 
 You can search the [gemnasium-db](https://gitlab.com/gitlab-org/security-products/gemnasium-db) project

doc/user/project/pipelines/settings.md

@@ -20,6 +20,22 @@ There are two options. Using:
 The default Git strategy can be overridden by the [GIT_STRATEGY variable](../../../ci/yaml/README.md#git-strategy)
 in `.gitlab-ci.yml`.
 
+## Git shallow clone
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/28919) in GitLab 12.0.
+
+NOTE: **Note**: As of GitLab 12.0, newly created projects will automaticallyl have a default 
+`git depth` value of `50`.
+
+It is possible to limit the number of changes that GitLab CI/CD will fetch when cloning 
+a repository. Setting a limit to `git depth` can speed up Pipelines execution. Maximum
+allowed value is `1000`.
+
+To disable shallow clone and make GitLab CI/CD fetch all branches and tags each time,
+keep the value empty or set to `0`.
+
+This value can also be [overridden by `GIT_DEPTH`](../../../ci/large_repositories/index.md#shallow-cloning) variable in `.gitlab-ci.yml` file.
+
 ## Timeout
 
 Timeout defines the maximum amount of time in minutes that a job is able run.

lib/gitlab/background_migration/migrate_legacy_uploads.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This migration takes all legacy uploads (that were uploaded using AttachmentUploader)
+    # and migrate them to the new (FileUploader) location (=under projects).
+    #
+    # We have dependencies (uploaders) in this migration because extracting code would add a lot of complexity
+    # and possible errors could appear as the logic in the uploaders is not trivial.
+    #
+    # This migration will be removed in 12.4 in order to get rid of a migration that depends on
+    # the application code.
+    class MigrateLegacyUploads
+      include Database::MigrationHelpers
+      include ::Gitlab::Utils::StrongMemoize
+
+      # This class takes a legacy upload and migrates it to the correct location
+      class UploadMover
+        include Gitlab::Utils::StrongMemoize
+
+        attr_reader :upload, :project, :note
+
+        def initialize(upload)
+          @upload = upload
+          @note = Note.find_by(id: upload.model_id)
+          @project = note&.project
+        end
+
+        def execute
+          return unless upload
+
+          if !project
+            # if we don't have models associated with the upload we can not move it
+            say "MigrateLegacyUploads: Deleting upload due to model not found: #{upload.inspect}"
+            destroy_legacy_upload
+          elsif note.is_a?(LegacyDiffNote)
+            handle_legacy_note_upload
+          elsif !legacy_file_exists?
+            # if we can not find the file we just remove the upload record
+            say "MigrateLegacyUploads: Deleting upload due to file not found: #{upload.inspect}"
+            destroy_legacy_upload
+          else
+            migrate_upload
+          end
+        end
+
+        private
+
+        def migrate_upload
+          return unless copy_upload_to_project
+
+          add_upload_link_to_note_text
+          destroy_legacy_file
+          destroy_legacy_upload
+        end
+
+        # we should proceed and log whenever one upload copy fails, no matter the reasons
+        # rubocop: disable Lint/RescueException
+        def copy_upload_to_project
+          @uploader = FileUploader.copy_to(legacy_file_uploader, project)
+
+          say "MigrateLegacyUploads: Copied file #{legacy_file_uploader.file.path} -> #{@uploader.file.path}"
+          true
+        rescue Exception => e
+          say "MigrateLegacyUploads: File #{legacy_file_uploader.file.path} couldn't be copied to project uploads. Error: #{e.message}"
+          false
+        end
+        # rubocop: enable Lint/RescueException
+
+        def destroy_legacy_upload
+          note.remove_attachment = true
+          note.save
+
+          if upload.destroy
+            say "MigrateLegacyUploads: Upload #{upload.inspect} was destroyed."
+          else
+            say "MigrateLegacyUploads: Upload #{upload.inspect} destroy failed."
+          end
+        end
+
+        def destroy_legacy_file
+          legacy_file_uploader.file.delete
+        end
+
+        def add_upload_link_to_note_text
+          new_text = "#{note.note} \n #{@uploader.markdown_link}"
+          note.update!(
+            note: new_text
+          )
+        end
+
+        def legacy_file_uploader
+          strong_memoize(:legacy_file_uploader) do
+            uploader = upload.build_uploader
+            uploader.retrieve_from_store!(File.basename(upload.path))
+            uploader
+          end
+        end
+
+        def legacy_file_exists?
+          legacy_file_uploader.file.exists?
+        end
+
+        def handle_legacy_note_upload
+          note.note += "\n \n Attachment ##{upload.id} with URL \"#{note.attachment.url}\" failed to migrate \
+               for model class #{note.class}. See #{help_doc_link}."
+          note.save
+
+          say "MigrateLegacyUploads: LegacyDiffNote ##{note.id} found, can't move the file: #{upload.inspect} for upload ##{upload.id}. See #{help_doc_link}."
+        end
+
+        def say(message)
+          Rails.logger.info(message)
+        end
+
+        def help_doc_link
+          'https://docs.gitlab.com/ee/administration/troubleshooting/migrations.html#legacy-upload-migration'
+        end
+      end
+
+      def perform(start_id, end_id)
+        Upload.where(id: start_id..end_id, uploader: 'AttachmentUploader').find_each do |upload|
+          UploadMover.new(upload).execute
+        end
+      end
+    end
+  end
+end

spec/controllers/projects/settings/ci_cd_controller_spec.rb

@@ -200,6 +200,21 @@ describe Projects::Settings::CiCdController do
           expect(response).to redirect_to(namespace_project_settings_ci_cd_path)
         end
       end
+
+      context 'when default_git_depth is not specified' do
+        let(:params) { { ci_cd_settings_attributes: { default_git_depth: 10 } } }
+
+        before do
+          project.ci_cd_settings.update!(default_git_depth: nil)
+        end
+
+        it 'set specified git depth' do
+          subject
+
+          project.reload
+          expect(project.default_git_depth).to eq(10)
+        end
+      end
     end
   end
 end

spec/factories/uploads.rb

@@ -54,10 +54,7 @@ FactoryBot.define do
     end
 
     trait :attachment_upload do
-      transient do
-        mount_point :attachment
-      end
-
+      mount_point :attachment
       model { build(:note) }
       uploader "AttachmentUploader"
     end

spec/lib/gitlab/background_migration/delete_diff_files_spec.rb

@@ -3,6 +3,12 @@ require 'spec_helper'
 # rubocop:disable RSpec/FactoriesInMigrationSpecs
 describe Gitlab::BackgroundMigration::DeleteDiffFiles, :migration, :sidekiq, schema: 20180619121030 do
   describe '#perform' do
+    before do
+      # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+      allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth=).and_return(0)
+      allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
+    end
+
     context 'when diff files can be deleted' do
       let(:merge_request) { create(:merge_request, :merged) }
       let!(:merge_request_diff) do

spec/lib/gitlab/background_migration/migrate_legacy_uploads_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateLegacyUploads, :migration, schema: 20190103140724 do
+  let(:test_dir) { FileUploader.options['storage_path'] }
+
+  # rubocop: disable RSpec/FactoriesInMigrationSpecs
+  let!(:namespace) { create(:namespace) }
+  let!(:project) { create(:project, :legacy_storage, namespace: namespace) }
+  let!(:issue) { create(:issue, project: project) }
+
+  let!(:note1) { create(:note, note: 'some note text awesome', project: project, noteable: issue) }
+  let!(:note2) { create(:note, note: 'some note', project: project, noteable: issue) }
+
+  let!(:hashed_project) { create(:project, namespace: namespace) }
+  let!(:issue_hashed_project) { create(:issue, project: hashed_project) }
+  let!(:note_hashed_project) { create(:note, note: 'some note', project: hashed_project, attachment: 'text.pdf', noteable: issue_hashed_project) }
+
+  let!(:standard_upload) do
+    create(:upload,
+      path: "secretabcde/image.png",
+      model_id: create(:project).id, model_type: 'Project', uploader: 'FileUploader')
+  end
+
+  def create_remote_upload(model, filename)
+    create(:upload, :attachment_upload,
+           path: "note/attachment/#{model.id}/#{filename}", secret: nil,
+           store: ObjectStorage::Store::REMOTE, model: model)
+  end
+
+  def create_upload(model, filename, with_file = true)
+    params = {
+      path: "uploads/-/system/note/attachment/#{model.id}/#{filename}",
+      model: model,
+      store: ObjectStorage::Store::LOCAL
+    }
+
+    upload = if with_file
+               create(:upload, :with_file, :attachment_upload, params)
+             else
+               create(:upload, :attachment_upload, params)
+             end
+
+    model.update(attachment: upload.build_uploader)
+    model.attachment.upload
+  end
+
+  let(:start_id) { 1 }
+  let(:end_id) { 10000 }
+
+  def new_upload_legacy
+    Upload.find_by(model_id: project.id, model_type: 'Project')
+  end
+
+  def new_upload_hashed
+    Upload.find_by(model_id: hashed_project.id, model_type: 'Project')
+  end
+
+  shared_examples 'migrates files correctly' do
+    before do
+      described_class.new.perform(start_id, end_id)
+    end
+
+    it 'removes all the legacy upload records' do
+      expect(Upload.where(uploader: 'AttachmentUploader')).to be_empty
+
+      expect(standard_upload.reload).to eq(standard_upload)
+    end
+
+    it 'creates new upload records correctly' do
+      expect(new_upload_legacy.secret).not_to be_nil
+      expect(new_upload_legacy.path).to end_with("#{new_upload_legacy.secret}/image.png")
+      expect(new_upload_legacy.model_id).to eq(project.id)
+      expect(new_upload_legacy.model_type).to eq('Project')
+      expect(new_upload_legacy.uploader).to eq('FileUploader')
+
+      expect(new_upload_hashed.secret).not_to be_nil
+      expect(new_upload_hashed.path).to end_with("#{new_upload_hashed.secret}/text.pdf")
+      expect(new_upload_hashed.model_id).to eq(hashed_project.id)
+      expect(new_upload_hashed.model_type).to eq('Project')
+      expect(new_upload_hashed.uploader).to eq('FileUploader')
+    end
+
+    it 'updates the legacy upload notes so that they include the file references in the markdown' do
+      expected_path = File.join('/uploads', new_upload_legacy.secret, 'image.png')
+      expected_markdown = "some note text awesome \n ![image](#{expected_path})"
+      expect(note1.reload.note).to eq(expected_markdown)
+
+      expected_path = File.join('/uploads', new_upload_hashed.secret, 'text.pdf')
+      expected_markdown = "some note \n [text.pdf](#{expected_path})"
+      expect(note_hashed_project.reload.note).to eq(expected_markdown)
+    end
+
+    it 'removed the attachments from the note model' do
+      expect(note1.reload.attachment.file).to be_nil
+      expect(note2.reload.attachment.file).to be_nil
+      expect(note_hashed_project.reload.attachment.file).to be_nil
+    end
+  end
+
+  context 'when legacy uploads are stored in local storage' do
+    let!(:legacy_upload1) { create_upload(note1, 'image.png') }
+    let!(:legacy_upload_not_found) { create_upload(note2, 'image.png', false) }
+    let!(:legacy_upload_hashed) { create_upload(note_hashed_project, 'text.pdf', with_file: true) }
+
+    shared_examples 'removes legacy local files' do
+      it 'removes all the legacy upload records' do
+        expect(File.exist?(legacy_upload1.absolute_path)).to be_truthy
+        expect(File.exist?(legacy_upload_hashed.absolute_path)).to be_truthy
+
+        described_class.new.perform(start_id, end_id)
+
+        expect(File.exist?(legacy_upload1.absolute_path)).to be_falsey
+        expect(File.exist?(legacy_upload_hashed.absolute_path)).to be_falsey
+      end
+    end
+
+    context 'when object storage is disabled for FileUploader' do
+      it_behaves_like 'migrates files correctly'
+      it_behaves_like 'removes legacy local files'
+
+      it 'moves legacy uploads to the correct location' do
+        described_class.new.perform(start_id, end_id)
+
+        expected_path1 = File.join(test_dir, 'uploads', namespace.path, project.path, new_upload_legacy.secret, 'image.png')
+        expected_path2 = File.join(test_dir, 'uploads', hashed_project.disk_path, new_upload_hashed.secret, 'text.pdf')
+
+        expect(File.exist?(expected_path1)).to be_truthy
+        expect(File.exist?(expected_path2)).to be_truthy
+      end
+
+      context 'when the upload move fails' do
+        it 'does not remove old uploads' do
+          expect(FileUploader).to receive(:copy_to).twice.and_raise('failed')
+
+          described_class.new.perform(start_id, end_id)
+
+          expect(legacy_upload1.reload).to eq(legacy_upload1)
+          expect(legacy_upload_hashed.reload).to eq(legacy_upload_hashed)
+          expect(standard_upload.reload).to eq(standard_upload)
+        end
+      end
+    end
+
+    context 'when object storage is enabled for FileUploader' do
+      before do
+        stub_uploads_object_storage(FileUploader)
+      end
+
+      it_behaves_like 'migrates files correctly'
+      it_behaves_like 'removes legacy local files'
+
+      # The process of migrating to object storage is a manual one,
+      # so it would go against expectations to automatically migrate these files
+      # to object storage during this migration.
+      # After this migration, these files should be able to successfully migrate to object storage.
+      it 'stores files locally' do
+        described_class.new.perform(start_id, end_id)
+
+        expected_path1 = File.join(test_dir, 'uploads', namespace.path, project.path, new_upload_legacy.secret, 'image.png')
+        expected_path2 = File.join(test_dir, 'uploads', hashed_project.disk_path, new_upload_hashed.secret, 'text.pdf')
+
+        expect(File.exist?(expected_path1)).to be_truthy
+        expect(File.exist?(expected_path2)).to be_truthy
+      end
+    end
+
+    context 'with legacy_diff_note upload' do
+      let!(:merge_request) { create(:merge_request, source_project: project) }
+      let!(:legacy_diff_note) { create(:legacy_diff_note_on_merge_request, note: 'some note', project: project, noteable: merge_request) }
+      let!(:legacy_upload_diff_note) do
+        create(:upload, :with_file, :attachment_upload,
+               path: "uploads/-/system/note/attachment/#{legacy_diff_note.id}/some_legacy.pdf", model: legacy_diff_note)
+      end
+
+      before do
+        described_class.new.perform(start_id, end_id)
+      end
+
+      it 'does not remove legacy diff note file' do
+        expect(File.exist?(legacy_upload_diff_note.absolute_path)).to be_truthy
+      end
+
+      it 'removes all the legacy upload records except for the one with legacy_diff_note' do
+        expect(Upload.where(uploader: 'AttachmentUploader')).to eq([legacy_upload_diff_note])
+      end
+
+      it 'adds link to the troubleshooting documentation to the note' do
+        help_doc_link = 'https://docs.gitlab.com/ee/administration/troubleshooting/migrations.html#legacy-upload-migration'
+
+        expect(legacy_diff_note.reload.note).to include(help_doc_link)
+      end
+    end
+  end
+
+  context 'when legacy uploads are stored in object storage' do
+    let!(:legacy_upload1) { create_remote_upload(note1, 'image.png') }
+    let!(:legacy_upload_not_found) { create_remote_upload(note2, 'non-existing.pdf') }
+    let!(:legacy_upload_hashed) { create_remote_upload(note_hashed_project, 'text.pdf') }
+    let(:remote_files) do
+      [
+        { key: "#{legacy_upload1.path}" },
+        { key: "#{legacy_upload_hashed.path}" }
+      ]
+    end
+    let(:connection) { ::Fog::Storage.new(FileUploader.object_store_credentials) }
+    let(:bucket) { connection.directories.create(key: 'uploads') }
+
+    def create_remote_files
+      remote_files.each { |file| bucket.files.create(file) }
+    end
+
+    before do
+      stub_uploads_object_storage(FileUploader)
+      create_remote_files
+    end
+
+    it_behaves_like 'migrates files correctly'
+
+    it 'moves legacy uploads to the correct remote location' do
+      described_class.new.perform(start_id, end_id)
+
+      connection = ::Fog::Storage.new(FileUploader.object_store_credentials)
+      expect(connection.get_object('uploads', new_upload_legacy.path)[:status]).to eq(200)
+      expect(connection.get_object('uploads', new_upload_hashed.path)[:status]).to eq(200)
+    end
+
+    it 'removes all the legacy upload records' do
+      described_class.new.perform(start_id, end_id)
+
+      remote_files.each do |remote_file|
+        expect(bucket.files.get(remote_file[:key])).to be_nil
+      end
+    end
+  end
+  # rubocop: enable RSpec/FactoriesInMigrationSpecs
+end

spec/lib/gitlab/background_migration/populate_external_pipeline_source_spec.rb

@@ -9,6 +9,9 @@ describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration
   before do
     # This migration was created before we introduced metadata configs
     stub_feature_flags(ci_build_metadata_config: false)
+    # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth=).and_return(0)
   end
 
   let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }

spec/migrations/remove_orphaned_label_links_spec.rb

@@ -10,6 +10,12 @@ describe RemoveOrphanedLabelLinks, :migration do
   let(:project) { create(:project) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
   let(:label) { create_label }
 
+  before do
+    # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth=).and_return(0)
+  end
+
   context 'add foreign key on label_id' do
     let!(:label_link_with_label) { create_label_link(label_id: label.id) }
     let!(:label_link_without_label) { create_label_link(label_id: nil) }

spec/models/project_ci_cd_setting_spec.rb

@@ -21,4 +21,32 @@ describe ProjectCiCdSetting do
       2.times { described_class.available? }
     end
   end
+
+  describe 'validations' do
+    it 'validates default_git_depth is between 0 and 1000 or nil' do
+      expect(subject).to validate_numericality_of(:default_git_depth)
+        .only_integer
+        .is_greater_than_or_equal_to(0)
+        .is_less_than_or_equal_to(1000)
+        .allow_nil
+    end
+  end
+
+  describe '#default_git_depth' do
+    let(:default_value) { described_class::DEFAULT_GIT_DEPTH }
+
+    it 'sets default value for new records' do
+      project = create(:project)
+
+      expect(project.ci_cd_settings.default_git_depth).to eq(default_value)
+    end
+
+    it 'does not set default value if present' do
+      project = build(:project)
+      project.build_ci_cd_settings(default_git_depth: 0)
+      project.save!
+
+      expect(project.reload.ci_cd_settings.default_git_depth).to eq(0)
+    end
+  end
 end

spec/presenters/ci/build_runner_presenter_spec.rb

@@ -119,23 +119,23 @@ describe Ci::BuildRunnerPresenter do
   end
 
   describe '#git_depth' do
-    subject { presenter.git_depth }
-
     let(:build) { create(:ci_build) }
 
-    it 'returns the correct git depth' do
-      is_expected.to eq(0)
-    end
+    subject(:git_depth) { presenter.git_depth }
 
     context 'when GIT_DEPTH variable is specified' do
       before do
         create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
       end
 
-      it 'returns the correct git depth' do
-        is_expected.to eq(1)
+      it 'returns its value' do
+        expect(git_depth).to eq(1)
       end
     end
+
+    it 'defaults to git depth setting for the project' do
+      expect(git_depth).to eq(build.project.default_git_depth)
+    end
   end
 
   describe '#refspecs' do
@@ -144,24 +144,24 @@ describe Ci::BuildRunnerPresenter do
     let(:build) { create(:ci_build) }
 
     it 'returns the correct refspecs' do
-      is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
-                                     '+refs/heads/*:refs/remotes/origin/*')
+      is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
     end
 
-    context 'when GIT_DEPTH variable is specified' do
-      before do
-        create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
-      end
+    context 'when ref is tag' do
+      let(:build) { create(:ci_build, :tag) }
 
       it 'returns the correct refspecs' do
-        is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
+        is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}")
       end
 
-      context 'when ref is tag' do
-        let(:build) { create(:ci_build, :tag) }
+      context 'when GIT_DEPTH is zero' do
+        before do
+          create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
+        end
 
         it 'returns the correct refspecs' do
-          is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}")
+          is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
+                                         '+refs/heads/*:refs/remotes/origin/*')
         end
       end
     end
@@ -173,17 +173,27 @@ describe Ci::BuildRunnerPresenter do
 
       it 'returns the correct refspecs' do
         is_expected
-          .to contain_exactly('+refs/heads/*:refs/remotes/origin/*',
-                              '+refs/tags/*:refs/tags/*',
-                              '+refs/merge-requests/1/head:refs/merge-requests/1/head')
+          .to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head')
+      end
+
+      context 'when GIT_DEPTH is zero' do
+        before do
+          create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
+        end
+
+        it 'returns the correct refspecs' do
+          is_expected
+            .to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head',
+                                '+refs/heads/*:refs/remotes/origin/*',
+                                '+refs/tags/*:refs/tags/*')
+        end
       end
 
       context 'when pipeline is legacy detached merge request pipeline' do
         let(:merge_request) { create(:merge_request, :with_legacy_detached_merge_request_pipeline) }
 
         it 'returns the correct refspecs' do
-          is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
-                                         '+refs/heads/*:refs/remotes/origin/*')
+          is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
         end
       end
     end

spec/requests/api/runner_spec.rb

@@ -444,8 +444,8 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               'sha' => job.sha,
               'before_sha' => job.before_sha,
               'ref_type' => 'branch',
-              'refspecs' => %w[+refs/heads/*:refs/remotes/origin/* +refs/tags/*:refs/tags/*],
-              'depth' => 0 }
+              'refspecs' => ["+refs/heads/#{job.ref}:refs/remotes/origin/#{job.ref}"],
+              'depth' => project.default_git_depth }
           end
 
           let(:expected_steps) do
@@ -531,7 +531,11 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               end
             end
 
-            context 'when GIT_DEPTH is not specified' do
+            context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
+              before do
+                project.update!(default_git_depth: nil)
+              end
+
               it 'specifies refspecs' do
                 request_job
 
@@ -587,7 +591,11 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               end
             end
 
-            context 'when GIT_DEPTH is not specified' do
+            context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
+              before do
+                project.update!(default_git_depth: nil)
+              end
+
               it 'specifies refspecs' do
                 request_job
 

spec/services/projects/fork_service_spec.rb

@@ -145,6 +145,30 @@ describe Projects::ForkService do
         end
       end
 
+      context "CI/CD settings" do
+        let(:to_project) { fork_project(@from_project, @to_user) }
+
+        context "when origin has git depth specified" do
+          before do
+            @from_project.update(default_git_depth: 42)
+          end
+
+          it "inherits default_git_depth from the origin project" do
+            expect(to_project.default_git_depth).to eq(42)
+          end
+        end
+
+        context "when origin does not define git depth" do
+          before do
+            @from_project.update!(default_git_depth: nil)
+          end
+
+          it "the fork has git depth set to 0" do
+            expect(to_project.default_git_depth).to eq(0)
+          end
+        end
+      end
+
       context "when project has restricted visibility level" do
         context "and only one visibility level is restricted" do
           before do