Merge branch 'live-trace-v2' into live-trace-v2-efficient-destroy-all

.gitlab-ci.yml

@@ -289,7 +289,6 @@ stages:
 # Trigger a package build in omnibus-gitlab repository
 #
 package-and-qa:
-  <<: *dedicated-runner
   image: ruby:2.4-alpine
   before_script: []
   stage: build

app/assets/stylesheets/framework/mobile.scss

@@ -40,10 +40,6 @@
   .project-home-panel {
     padding-left: 0 !important;
 
-    .project-avatar {
-      display: block;
-    }
-
     .project-repo-buttons,
     .git-clone-holder {
       display: none;

app/assets/stylesheets/framework/secondary_navigation_elements.scss

@@ -241,8 +241,6 @@
 }
 
 .scrolling-tabs-container {
-  position: relative;
-
   .merge-request-tabs-container & {
     overflow: hidden;
   }
@@ -272,8 +270,6 @@
 }
 
 .inner-page-scroll-tabs {
-  position: relative;
-
   .fade-right {
     @include fade(left, $white-light);
     right: 0;

app/assets/stylesheets/pages/labels.scss

@@ -314,6 +314,10 @@
   display: inline-flex;
   vertical-align: top;
 
+  &:hover .color-label {
+    text-decoration: underline;
+  }
+
   .label {
     vertical-align: inherit;
     font-size: $label-font-size;

app/helpers/projects_helper.rb

@@ -400,7 +400,8 @@ module ProjectsHelper
     exports_path = File.join(Settings.shared['path'], 'tmp/project_exports')
     filtered_message = message.strip.gsub(exports_path, "[REPO EXPORT PATH]")
 
-    filtered_message.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
+    disk_path = Gitlab.config.repositories.storages[project.repository_storage].legacy_disk_path
+    filtered_message.gsub(disk_path.chomp('/'), "[REPOS PATH]")
   end
 
   def project_child_container_class(view_path)

app/models/ci/build.rb

@@ -19,14 +19,18 @@ module Ci
 
     has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
     has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
+    has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
     has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
     has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
     has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
 
+<<<<<<< HEAD
     has_many :chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id
 
+=======
+>>>>>>> live-trace-v2
     has_one :metadata, class_name: 'Ci::BuildMetadata'
     delegate :timeout, to: :metadata, prefix: true, allow_nil: true
     delegate :gitlab_deploy_token, to: :project

app/models/ci/job_trace_chunk.rb → app/models/ci/build_trace_chunk.rb

 module Ci
-  class JobTraceChunk < ActiveRecord::Base
+  class BuildTraceChunk < ActiveRecord::Base
     extend Gitlab::Ci::Model
 
-    belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
+    belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
 
     default_value_for :data_store, :redis
 
@@ -10,15 +10,16 @@ module Ci
 
     CHUNK_SIZE = 128.kilobytes
     CHUNK_REDIS_TTL = 1.week
-    LOCK_RETRY = 100
-    LOCK_SLEEP = 1
-    LOCK_TTL = 5.minutes
+    WRITE_LOCK_RETRY = 100
+    WRITE_LOCK_SLEEP = 1
+    WRITE_LOCK_TTL = 5.minutes
 
     enum data_store: {
       redis: 1,
       db: 2
     }
 
+<<<<<<< HEAD:app/models/ci/job_trace_chunk.rb
     def self.delayed_cleanup_blk
       ids = all.redis.pluck(:job_id, :chunk_index).map do |data|
         "gitlab:ci:trace:#{data.first}:chunks:#{data.second}:data"
@@ -43,36 +44,16 @@ module Ci
       end
     end
 
+=======
+    ##
+    # Data is memoized for optimizing #size and #end_offset
+>>>>>>> live-trace-v2:app/models/ci/build_trace_chunk.rb
     def data
-      if redis?
-        redis_data
-      elsif db?
-        raw_data
-      else
-        raise 'Unsupported data store'
-      end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default
-    end
-
-    def set_data(value)
-      raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE
-
-      in_lock do
-        if redis?
-          redis_set_data(value)
-        elsif db?
-          self.raw_data = value
-        else
-          raise 'Unsupported data store'
-        end
-
-        save! if changed?
-      end
-
-      schedule_to_db if fullfilled?
+      @data ||= get_data
     end
 
     def truncate(offset = 0)
-      self.append("", offset)
+      self.append("", offset) if offset < size
     end
 
     def append(new_data, offset)
@@ -80,7 +61,7 @@ module Ci
       raise ArgumentError, 'Offset is out of bound' if offset > current_data.bytesize || offset < 0
       raise ArgumentError, 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize
 
-      self.set_data(current_data.byteslice(0, offset) + new_data)
+      set_data(current_data.byteslice(0, offset) + new_data)
     end
 
     def size
@@ -111,6 +92,36 @@ module Ci
 
     private
 
+    def get_data
+      if redis?
+        redis_data
+      elsif db?
+        raw_data
+      else
+        raise 'Unsupported data store'
+      end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default
+    end
+
+    def set_data(value)
+      raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE
+
+      in_lock do
+        if redis?
+          redis_set_data(value)
+        elsif db?
+          self.raw_data = value
+        else
+          raise 'Unsupported data store'
+        end
+
+        @data = value
+
+        save! if changed?
+      end
+
+      schedule_to_db if fullfilled?
+    end
+
     def schedule_to_db
       return if db?
 
@@ -140,22 +151,22 @@ module Ci
     end
 
     def redis_data_key
-      "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:data"
+      "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:data"
     end
 
     def redis_lock_key
-      "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:lock"
+      "trace_write:#{build_id}:chunks:#{chunk_index}"
     end
 
     def in_lock
-      lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: LOCK_TTL)
+      lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: WRITE_LOCK_TTL)
       retry_count = 0
 
       until uuid = lease.try_obtain
         # Keep trying until we obtain the lease. To prevent hammering Redis too
         # much we'll wait for a bit between retries.
-        sleep(LOCK_SLEEP)
-        break if LOCK_RETRY < (retry_count += 1)
+        sleep(WRITE_LOCK_SLEEP)
+        break if WRITE_LOCK_RETRY < (retry_count += 1)
       end
 
       raise WriteError, 'Failed to obtain write lock' unless uuid

app/models/concerns/storage/legacy_namespace.rb

@@ -45,25 +45,25 @@ module Storage
 
     # Hooks
 
-    # Save the storage paths before the projects are destroyed to use them on after destroy
+    # Save the storages before the projects are destroyed to use them on after destroy
     def prepare_for_destroy
-      old_repository_storage_paths
+      old_repository_storages
     end
 
     private
 
     def move_repositories
-      # Move the namespace directory in all storage paths used by member projects
-      repository_storage_paths.each do |repository_storage_path|
+      # Move the namespace directory in all storages used by member projects
+      repository_storages.each do |repository_storage|
         # Ensure old directory exists before moving it
-        gitlab_shell.add_namespace(repository_storage_path, full_path_was)
+        gitlab_shell.add_namespace(repository_storage, full_path_was)
 
         # Ensure new directory exists before moving it (if there's a parent)
-        gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent
+        gitlab_shell.add_namespace(repository_storage, parent.full_path) if parent
 
-        unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
+        unless gitlab_shell.mv_namespace(repository_storage, full_path_was, full_path)
 
-          Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
+          Rails.logger.error "Exception moving path #{repository_storage} from #{full_path_was} to #{full_path}"
 
           # if we cannot move namespace directory we should rollback
           # db changes in order to prevent out of sync between db and fs
@@ -72,33 +72,33 @@ module Storage
       end
     end
 
-    def old_repository_storage_paths
-      @old_repository_storage_paths ||= repository_storage_paths
+    def old_repository_storages
+      @old_repository_storage_paths ||= repository_storages
     end
 
-    def repository_storage_paths
+    def repository_storages
       # We need to get the storage paths for all the projects, even the ones that are
       # pending delete. Unscoping also get rids of the default order, which causes
       # problems with SELECT DISTINCT.
       Project.unscoped do
-        all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
+        all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage)
       end
     end
 
     def rm_dir
       # Remove the namespace directory in all storages paths used by member projects
-      old_repository_storage_paths.each do |repository_storage_path|
+      old_repository_storages.each do |repository_storage|
         # Move namespace directory into trash.
         # We will remove it later async
         new_path = "#{full_path}+#{id}+deleted"
 
-        if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path)
+        if gitlab_shell.mv_namespace(repository_storage, full_path, new_path)
           Gitlab::AppLogger.info %Q(Namespace directory "#{full_path}" moved to "#{new_path}")
 
           # Remove namespace directroy async with delay so
           # GitLab has time to remove all projects first
           run_after_commit do
-            GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
+            GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage, new_path)
           end
         end
       end

app/models/merge_request_diff.rb

@@ -197,10 +197,6 @@ class MergeRequestDiff < ActiveRecord::Base
     CompareService.new(project, head_commit_sha).execute(project, sha, straight: true)
   end
 
-  def commits_count
-    super || merge_request_diff_commits.size
-  end
-
   private
 
   def create_merge_request_diff_files(diffs)

app/models/project.rb

@@ -527,10 +527,6 @@ class Project < ActiveRecord::Base
     repository.empty?
   end
 
-  def repository_storage_path
-    Gitlab.config.repositories.storages[repository_storage]&.legacy_disk_path
-  end
-
   def team
     @team ||= ProjectTeam.new(self)
   end
@@ -1114,7 +1110,7 @@ class Project < ActiveRecord::Base
   # Check if repository already exists on disk
   def check_repository_path_availability
     return true if skip_disk_validation
-    return false unless repository_storage_path
+    return false unless repository_storage
 
     expires_full_path_cache # we need to clear cache to validate renames correctly
 
@@ -1919,14 +1915,14 @@ class Project < ActiveRecord::Base
   def check_repository_absence!
     return if skip_disk_validation
 
-    if repository_storage_path.blank? || repository_with_same_path_already_exists?
+    if repository_storage.blank? || repository_with_same_path_already_exists?
       errors.add(:base, 'There is already a repository with that name on disk')
       throw :abort
     end
   end
 
   def repository_with_same_path_already_exists?
-    gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git")
+    gitlab_shell.exists?(repository_storage, "#{disk_path}.git")
   end
 
   # set last_activity_at to the same as created_at

app/models/project_wiki.rb

@@ -21,7 +21,7 @@ class ProjectWiki
   end
 
   delegate :empty?, to: :pages
-  delegate :repository_storage_path, :hashed_storage?, to: :project
+  delegate :repository_storage, :hashed_storage?, to: :project
 
   def path
     @project.path + '.wiki'

app/models/repository.rb

@@ -84,9 +84,14 @@ class Repository
 
   # Return absolute path to repository
   def path_to_repo
-    @path_to_repo ||= File.expand_path(
-      File.join(repository_storage_path, disk_path + '.git')
-    )
+    @path_to_repo ||=
+      begin
+        storage = Gitlab.config.repositories.storages[@project.repository_storage]
+
+        File.expand_path(
+          File.join(storage.legacy_disk_path, disk_path + '.git')
+        )
+      end
   end
 
   def inspect
@@ -915,10 +920,6 @@ class Repository
     raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
   end
 
-  def repository_storage_path
-    @project.repository_storage_path
-  end
-
   def rebase(user, merge_request)
     raw.rebase(user, merge_request.id, branch: merge_request.source_branch,
                                        branch_sha: merge_request.source_branch_sha,

app/models/storage/hashed_project.rb

 module Storage
   class HashedProject
     attr_accessor :project
-    delegate :gitlab_shell, :repository_storage_path, to: :project
+    delegate :gitlab_shell, :repository_storage, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
 
@@ -24,7 +24,7 @@ module Storage
     end
 
     def ensure_storage_path_exists
-      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+      gitlab_shell.add_namespace(repository_storage, base_dir)
     end
 
     def rename_repo

app/models/storage/legacy_project.rb

 module Storage
   class LegacyProject
     attr_accessor :project
-    delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project
+    delegate :namespace, :gitlab_shell, :repository_storage, to: :project
 
     def initialize(project)
       @project = project
@@ -24,18 +24,18 @@ module Storage
     def ensure_storage_path_exists
       return unless namespace
 
-      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+      gitlab_shell.add_namespace(repository_storage, base_dir)
     end
 
     def rename_repo
       new_full_path = project.build_full_path
 
-      if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path)
+      if gitlab_shell.mv_repository(repository_storage, project.full_path_was, new_full_path)
         # If repository moved successfully we need to send update instructions to users.
         # However we cannot allow rollback since we moved repository
         # So we basically we mute exceptions in next actions
         begin
-          gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
+          gitlab_shell.mv_repository(repository_storage, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
           return true
         rescue => e
           Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}"

app/services/projects/destroy_service.rb

@@ -91,7 +91,7 @@ module Projects
 
         project.run_after_commit do
           # self is now project
-          GitlabShellWorker.perform_in(5.minutes, :remove_repository, self.repository_storage_path, new_path)
+          GitlabShellWorker.perform_in(5.minutes, :remove_repository, self.repository_storage, new_path)
         end
       else
         false
@@ -100,9 +100,9 @@ module Projects
 
     def mv_repository(from_path, to_path)
       # There is a possibility project does not have repository or wiki
-      return true unless gitlab_shell.exists?(project.repository_storage_path, from_path + '.git')
+      return true unless gitlab_shell.exists?(project.repository_storage, from_path + '.git')
 
-      gitlab_shell.mv_repository(project.repository_storage_path, from_path, to_path)
+      gitlab_shell.mv_repository(project.repository_storage, from_path, to_path)
     end
 
     def attempt_rollback(project, message)

app/services/projects/hashed_storage/migrate_repository_service.rb

@@ -47,8 +47,8 @@ module Projects
       private
 
       def move_repository(from_name, to_name)
-        from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
-        to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
+        from_exists = gitlab_shell.exists?(project.repository_storage, "#{from_name}.git")
+        to_exists = gitlab_shell.exists?(project.repository_storage, "#{to_name}.git")
 
         # If we don't find the repository on either original or target we should log that as it could be an issue if the
         # project was not originally empty.
@@ -60,7 +60,7 @@ module Projects
           return true
         end
 
-        gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+        gitlab_shell.mv_repository(project.repository_storage, from_name, to_name)
       end
 
       def rollback_folder_move

app/services/projects/transfer_service.rb

@@ -127,7 +127,7 @@ module Projects
     end
 
     def move_repo_folder(from_name, to_name)
-      gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+      gitlab_shell.mv_repository(project.repository_storage, from_name, to_name)
     end
 
     def execute_system_hooks

app/views/groups/edit.html.haml

 - breadcrumb_title "General Settings"
+- @content_class = "limit-container-width" unless fluid_layout
+
 .panel.panel-default.prepend-top-default
   .panel-heading
     Group settings

app/views/projects/forks/new.html.haml

@@ -12,7 +12,7 @@
     - if @namespaces.present?
       .fork-thumbnail-container.js-fork-content
         %h5.prepend-top-0.append-bottom-0.prepend-left-default.append-right-default
-          Click to fork the project
+          = _("Select a namespace to fork the project")
         - @namespaces.each do |namespace|
           = render 'fork_button', namespace: namespace
     - else

app/views/shared/_group_form.html.haml

@@ -32,6 +32,13 @@
       required: true,
       title: 'You can choose a descriptive name different from the path.'
 
+- if @group.persisted?
+  .form-group.group-name-holder
+    = f.label :id, class: 'control-label' do
+      = _("Group ID")
+    .col-sm-10
+      = f.text_field :id, class: 'form-control', readonly: true
+
 .form-group.group-description-holder
   = f.label :description, class: 'control-label'
   .col-sm-10

app/workers/build_trace_swap_chunk_worker.rb

@@ -4,9 +4,9 @@ class BuildTraceSwapChunkWorker
 
   queue_namespace :pipeline_processing
 
-  def perform(job_trace_chunk_id)
-    Ci::JobTraceChunk.find_by(id: job_trace_chunk_id).try do |job_trace_chunk|
-      job_trace_chunk.use_database!
+  def perform(build_trace_chunk_id)
+    Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk|
+      build_trace_chunk.use_database!
     end
   end
 end

app/workers/repository_fork_worker.rb

@@ -13,7 +13,9 @@ class RepositoryForkWorker
     # See https://gitlab.com/gitlab-org/gitaly/issues/1110
     if args.empty?
       source_project = target_project.forked_from_project
-      return target_project.mark_import_as_failed('Source project cannot be found.') unless source_project
+      unless source_project
+        return target_project.mark_import_as_failed('Source project cannot be found.')
+      end
 
       fork_repository(target_project, source_project.repository_storage, source_project.disk_path)
     else

changelogs/unreleased/45398-fix-rss-button.yml

+---
+title: Fix tabs container styles to make RSS button clickable
+merge_request: 18559
+author:
+type: fixed

changelogs/unreleased/accessible-text.yml

+---
+title: Replace "Click" with "Select" to be more inclusive of people with accessibility
+  requirements
+merge_request: 18386
+author: Mark Lapierre
+type: other

changelogs/unreleased/align-project-avatar-on-small-viewports.yml

+---
+title: Align project avatar on small viewports
+merge_request: 18513
+author: George Tsiolis
+type: changed

changelogs/unreleased/restore-label-underline-color.yml

+---
+title: Restore label underline color
+merge_request: 18407
+author: George Tsiolis
+type: fixed

changelogs/unreleased/show-group-id-in-group-settings.yml

+---
+title: Show group id in group settings
+merge_request: 18482
+author: George Tsiolis
+type: added

config/initializers/1_settings.rb

-require_dependency File.expand_path('../../lib/gitlab', __dir__) # Load Gitlab as soon as possible
+require_relative '../settings'
 
 # Default settings
 Settings['ldap'] ||= Settingslogic.new({})

config/initializers/2_gitlab.rb

+require_relative '../../lib/gitlab'

config/initializers/deprecations.rb

 deprecator = ActiveSupport::Deprecation.new('11.0', 'GitLab')
 
-if Gitlab.com? || Rails.env.development?
+if Gitlab.dev_env_or_com?
   ActiveSupport::Deprecation.deprecate_methods(Gitlab::GitalyClient::StorageSettings, :legacy_disk_path, deprecator: deprecator)
 end

db/migrate/20161220141214_remove_dot_git_from_group_names.rb

@@ -59,17 +59,17 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration
   end
 
   def move_namespace(group_id, path_was, path)
-    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
-      Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
+    repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
+      row['repository_storage']
     end.compact
 
     # Move the namespace directory in all storages paths used by member projects
-    repository_storage_paths.each do |repository_storage_path|
+    repository_storages.each do |repository_storage|
       # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, path_was)
+      gitlab_shell.add_namespace(repository_storage, path_was)
 
-      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
-        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+      unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
+        Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
 
         # if we cannot move namespace directory we should rollback
         # db changes in order to prevent out of sync between db and fs

db/migrate/20161226122833_remove_dot_git_from_usernames.rb

@@ -53,8 +53,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
     select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(path)}'").present?
   end
 
-  def path_exists?(path, repository_storage_path)
-    repository_storage_path && gitlab_shell.exists?(repository_storage_path, path)
+  def path_exists?(shard, repository_storage_path)
+    repository_storage_path && gitlab_shell.exists?(shard, repository_storage_path)
   end
 
   # Accepts invalid path like test.git and returns test_git or
@@ -70,8 +70,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
   def check_routes(base, counter, path)
     route_exists = route_exists?(path)
 
-    Gitlab.config.repositories.storages.each_value do |storage|
-      if route_exists || path_exists?(path, storage.legacy_disk_path)
+    Gitlab.config.repositories.storages.each do |shard, storage|
+      if route_exists || path_exists?(shard, storage.legacy_disk_path)
         counter += 1
         path = "#{base}#{counter}"
 
@@ -83,17 +83,17 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
   end
 
   def move_namespace(namespace_id, path_was, path)
-    repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
-      Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
+    repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
+      row['repository_storage']
     end.compact
 
-    # Move the namespace directory in all storages paths used by member projects
-    repository_storage_paths.each do |repository_storage_path|
+    # Move the namespace directory in all storages used by member projects
+    repository_storages.each do |repository_storage|
       # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, path_was)
+      gitlab_shell.add_namespace(repository_storage, path_was)
 
-      unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
-        Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
+      unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
+        Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
 
         # if we cannot move namespace directory we should rollback
         # db changes in order to prevent out of sync between db and fs

db/migrate/20180326202229_create_ci_job_trace_chunks.rb → db/migrate/20180326202229_create_ci_build_trace_chunks.rb

-class CreateCiJobTraceChunks < ActiveRecord::Migration
+class CreateCiBuildTraceChunks < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
   DOWNTIME = false
 
   def change
-    create_table :ci_job_trace_chunks, id: :bigserial do |t|
-      t.integer :job_id, null: false
+    create_table :ci_build_trace_chunks, id: :bigserial do |t|
+      t.integer :build_id, null: false
       t.integer :chunk_index, null: false
       t.integer :data_store, null: false
       t.binary :raw_data
 
-      t.foreign_key :ci_builds, column: :job_id, on_delete: :cascade
-      t.index [:job_id, :chunk_index], unique: true
+      t.foreign_key :ci_builds, column: :build_id, on_delete: :cascade
+      t.index [:build_id, :chunk_index], unique: true
     end
   end
 end

db/migrate/20180406204716_add_limits_ci_job_trace_chunks_raw_data_for_mysql.rb → db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb

 # See http://doc.gitlab.com/ce/development/migration_style_guide.html
 # for more information on how to write migrations for GitLab.
-require Rails.root.join('db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql')
+require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
 
-class AddLimitsCiJobTraceChunksRawDataForMysql < ActiveRecord::Migration
+class AddLimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
   DOWNTIME = false
 
   def up
-    LimitsCiJobTraceChunksRawDataForMysql.new.up
+    LimitsCiBuildTraceChunksRawDataForMysql.new.up
   end
 end

db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb

+class AssureCommitsCountForMergeRequestDiff < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  def up
+    Gitlab::BackgroundMigration.steal('AddMergeRequestDiffCommitsCount')
+
+    MergeRequestDiff.where(commits_count: nil).each_batch(of: 50) do |batch|
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+
+      Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount.new.perform(*range)
+    end
+  end
+
+  def down
+    # noop
+  end
+end

db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql.rb → db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb

-class LimitsCiJobTraceChunksRawDataForMysql < ActiveRecord::Migration
+class LimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration
   def up
     return unless Gitlab::Database.mysql?
 
     # Mysql needs MEDIUMTEXT type (up to 16MB) rather than TEXT (up to 64KB)
-    # Because 'raw_data' is always capped by Ci::JobTraceChunk::CHUNK_SIZE, which is 128KB
-    change_column :ci_job_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT
+    # Because 'raw_data' is always capped by Ci::BuildTraceChunk::CHUNK_SIZE, which is 128KB
+    change_column :ci_build_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT
   end
 end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180418053107) do
+ActiveRecord::Schema.define(version: 20180425205249) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -246,6 +246,15 @@ ActiveRecord::Schema.define(version: 20180418053107) do
 
   add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree
 
+  create_table "ci_build_trace_chunks", id: :bigserial, force: :cascade do |t|
+    t.integer "build_id", null: false
+    t.integer "chunk_index", null: false
+    t.integer "data_store", null: false
+    t.binary "raw_data"
+  end
+
+  add_index "ci_build_trace_chunks", ["build_id", "chunk_index"], name: "index_ci_build_trace_chunks_on_build_id_and_chunk_index", unique: true, using: :btree
+
   create_table "ci_build_trace_section_names", force: :cascade do |t|
     t.integer "project_id", null: false
     t.string "name", null: false
@@ -371,15 +380,6 @@ ActiveRecord::Schema.define(version: 20180418053107) do
   add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree
   add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree
 
-  create_table "ci_job_trace_chunks", id: :bigserial, force: :cascade do |t|
-    t.integer "job_id", null: false
-    t.integer "chunk_index", null: false
-    t.integer "data_store", null: false
-    t.binary "raw_data"
-  end
-
-  add_index "ci_job_trace_chunks", ["job_id", "chunk_index"], name: "index_ci_job_trace_chunks_on_job_id_and_chunk_index", unique: true, using: :btree
-
   create_table "ci_pipeline_schedule_variables", force: :cascade do |t|
     t.string "key", null: false
     t.text "value"
@@ -2075,6 +2075,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do
   add_foreign_key "boards", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade
   add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
+  add_foreign_key "ci_build_trace_chunks", "ci_builds", column: "build_id", on_delete: :cascade
   add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade
   add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade
   add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade
@@ -2087,7 +2088,6 @@ ActiveRecord::Schema.define(version: 20180418053107) do
   add_foreign_key "ci_group_variables", "namespaces", column: "group_id", name: "fk_33ae4d58d8", on_delete: :cascade
   add_foreign_key "ci_job_artifacts", "ci_builds", column: "job_id", on_delete: :cascade
   add_foreign_key "ci_job_artifacts", "projects", on_delete: :cascade
-  add_foreign_key "ci_job_trace_chunks", "ci_builds", column: "job_id", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedule_variables", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_41c35fda51", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify

doc/api/README.md

@@ -13,6 +13,7 @@ following locations:
 - [Broadcast Messages](broadcast_messages.md)
 - [Project-level Variables](project_level_variables.md)
 - [Group-level Variables](group_level_variables.md)
+- [Code Snippets](snippets.md)
 - [Commits](commits.md)
 - [Custom Attributes](custom_attributes.md)
 - [Deployments](deployments.md)

doc/ci/runners/README.md

@@ -298,6 +298,28 @@ Mentioned briefly earlier, but the following things of Runners can be exploited.
 We're always looking for contributions that can mitigate these
 [Security Considerations](https://docs.gitlab.com/runner/security/).
 
+### Resetting the registration token for a Project
+
+If you think that registration token for a Project was revealed, you should
+reset them. It's recommended because such token can be used to register another
+Runner to thi Project. It may be next used to obtain the values of secret
+variables or clone the project code, that normally may be unavailable for the
+attacker.
+
+To reset the token:
+
+1. Go to **Settings > CI/CD** for a specified Project
+1. Expand the **General pipelines settings** section
+1. Find the **Runner token** form field and click the **Reveal value** button
+1. Delete the value and save the form
+1. After the page is refreshed, expand the **Runners settings** section
+    and check the registration token - it should be changed
+
+From now on the old token is not valid anymore and will not allow to register
+a new Runner to the project. If you are using any tools to provision and
+register new Runners, you should now update the token that is used to the
+new value.
+
 ## Determining the IP address of a Runner
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/17286) in GitLab 10.6.

doc/ci/variables/README.md

@@ -61,7 +61,7 @@ future GitLab releases.**
 | **CI_RUNNER_EXECUTABLE_ARCH**   | all    | 10.6   | The OS/architecture of the GitLab Runner executable (note that this is not necessarily the same as the environment of the executor) |
 | **CI_PIPELINE_ID**              | 8.10   | 0.5    | The unique id of the current pipeline that GitLab CI uses internally |
 | **CI_PIPELINE_TRIGGERED**       | all    | all    | The flag to indicate that job was [triggered] |
-| **CI_PIPELINE_SOURCE**          | 10.0   | all    | The source for this pipeline, one of: push, web, trigger, schedule, api, external. Pipelines created before 9.5 will have unknown as source |
+| **CI_PIPELINE_SOURCE**          | 10.0   | all    | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, and `pipeline`. For pipelines created before GitLab 9.5, this will show as `unknown` |
 | **CI_PROJECT_DIR**              | all    | all    | The full path where the repository is cloned and where the job is run |
 | **CI_PROJECT_ID**               | all    | all    | The unique id of the current project that GitLab CI uses internally |
 | **CI_PROJECT_NAME**             | 8.10   | 0.5    | The project name that is currently being built (actually it is project folder name) |

doc/install/kubernetes/gitlab_chart.md

 # GitLab Helm Chart
-> **Note:**
-* This chart has been tested on Google Kubernetes Engine and Azure Container Service.
+> **Note:** This chart is currently in alpha.
 
-**This chart is deprecated.** For small installations on Kubernetes today, we recommend the beta [`gitlab-omnibus` Helm chart](gitlab_omnibus.md).
+The cloud native `gitlab` chart is the next generation Helm chart, currently in alpha, and will replace the [`gitlab-omnibus`](gitlab_omnibus.md) chart. It will support large deployments with horizontal scaling of individual GitLab components.
 
-A new [cloud native GitLab chart](index.md#cloud-native-gitlab-chart) is in development with increased scalability and resilience, among other benefits. The cloud native chart will replace both the `gitlab` and `gitlab-omnibus` charts when available later this year.
-
-Due to the significant architectural changes, migrating will require backing up data out of this instance and restoring it into the new deployment. For more information on available GitLab Helm Charts, please see our [overview](index.md#chart-overview).
-
-## Introduction
-
-The `gitlab` Helm chart deploys just GitLab into your Kubernetes cluster, and offers extensive configuration options. This chart requires advanced knowledge of Kubernetes to successfully use. We **strongly recommend** the [gitlab-omnibus](gitlab_omnibus.md) chart.
-
-This chart includes the following:
-
-- Deployment using the [gitlab-ce](https://hub.docker.com/r/gitlab/gitlab-ce) or [gitlab-ee](https://hub.docker.com/r/gitlab/gitlab-ee) container image
-- ConfigMap containing the `gitlab.rb` contents that configure [Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options)
-- Persistent Volume Claims for Data, Config, Logs, and Registry Storage
-- A Kubernetes service
-- Optional Redis deployment using the [Redis Chart](https://github.com/kubernetes/charts/tree/master/stable/redis) (defaults to enabled)
-- Optional PostgreSQL deployment using the [PostgreSQL Chart](https://github.com/kubernetes/charts/tree/master/stable/postgresql) (defaults to enabled)
-- Optional Ingress (defaults to disabled)
-
-## Prerequisites
-
-- _At least_ 3 GB of RAM available on your cluster. 41GB of storage and 2 CPU are also required.
-- Kubernetes 1.4+ with Beta APIs enabled
-- [Persistent Volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) provisioner support in the underlying infrastructure
-- The ability to point a DNS entry or URL at your GitLab install
-- The `kubectl` CLI installed locally and authenticated for the cluster
-- The [Helm client](https://github.com/kubernetes/helm/blob/master/docs/quickstart.md) installed locally on your machine
-
-## Configuring GitLab
-
-Create a `values.yaml` file for your GitLab configuration. See the
-[Helm docs](https://github.com/kubernetes/helm/blob/master/docs/chart_template_guide/values_files.md)
-for information on how your values file will override the defaults.
-
-The default configuration can always be [found in the `values.yaml`](https://gitlab.com/charts/charts.gitlab.io/blob/master/charts/gitlab/values.yaml), in the chart repository.
-
-### Required configuration
-
-In order for GitLab to function, your config file **must** specify the following:
-
-- An `externalUrl` that GitLab will be reachable at.
-
-### Choosing GitLab Edition
-
-The Helm chart defaults to installing GitLab CE. This can be controlled by setting the `edition` variable in your values.
-
-Setting `edition` to GitLab Enterprise Edition (EE) in your `values.yaml`
-
-```yaml
-edition: EE
-
-externalUrl: 'http://gitlab.example.com'
-```
-
-### Choosing a different GitLab release version
-
-The version of GitLab installed is based on the `edition` setting (see [section](#choosing-gitlab-edition) above), and
-the value of the corresponding helm setting: `ceImage` or `eeImage`.
-
-```yaml
-## GitLab Edition
-## ref: https://about.gitlab.com/products/
-## - CE - Community Edition
-## - EE - Enterprise Edition - (requires license issued by GitLab Inc)
-##
-edition: CE
-
-## GitLab CE image
-## ref: https://hub.docker.com/r/gitlab/gitlab-ce/tags/
-##
-ceImage: gitlab/gitlab-ce:9.1.2-ce.0
-
-## GitLab EE image
-## ref: https://hub.docker.com/r/gitlab/gitlab-ee/tags/
-##
-eeImage: gitlab/gitlab-ee:9.1.2-ee.0
-```
-
-The different images can be found in the [gitlab-ce](https://hub.docker.com/r/gitlab/gitlab-ce/tags/) and [gitlab-ee](https://hub.docker.com/r/gitlab/gitlab-ee/tags/)
-repositories on Docker Hub
-
-> **Note:**
-There is no guarantee that other release versions of GitLab, other than what are
-used by default in the chart, will be supported by a chart install.
-
-
-### Custom Omnibus GitLab configuration
-
-In addition to the configuration options provided for GitLab in the Helm Chart, you can also pass any custom configuration
-that is valid for the [Omnibus GitLab Configuration](https://docs.gitlab.com/omnibus/settings/configuration.html).
-
-The setting to pass these values in is `omnibusConfigRuby`. It accepts any valid
-Ruby code that could used in the Omnibus `/etc/gitlab/gitlab.rb` file. In
-Kubernetes, the contents will be stored in a ConfigMap.
-
-Example setting:
-
-```yaml
-omnibusConfigRuby: |
-  unicorn['worker_processes'] = 2;
-  gitlab_rails['trusted_proxies'] = ["10.0.0.0/8","172.16.0.0/12","192.168.0.0/16"];
-```
-
-### Persistent storage
-
-By default, persistent storage is enabled for GitLab and the charts it depends
-on (Redis and PostgreSQL).
-
-Components can have their claim size set from your `values.yaml`, and each
-component allows you to optionally configure the `storageClass` variable so you
-can take advantage of faster drives on your cloud provider.
-
-Basic configuration:
-
-```yaml
-## Enable persistence using Persistent Volume Claims
-## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
-## ref: https://docs.gitlab.com/ce/install/requirements.html#storage
-##
-persistence:
-  ## This volume persists generated configuration files, keys, and certs.
-  ##
-  gitlabEtc:
-    enabled: true
-    size: 1Gi
-    ## If defined, volume.beta.kubernetes.io/storage-class: <storageClass>
-    ## Default: volume.alpha.kubernetes.io/storage-class: default
-    ##
-    # storageClass:
-    accessMode: ReadWriteOnce
-  ## This volume is used to store git data and other project files.
-  ## ref: https://docs.gitlab.com/omnibus/settings/configuration.html#storing-git-data-in-an-alternative-directory
-  ##
-  gitlabData:
-    enabled: true
-    size: 10Gi
-    ## If defined, volume.beta.kubernetes.io/storage-class: <storageClass>
-    ## Default: volume.alpha.kubernetes.io/storage-class: default
-    ##
-    # storageClass:
-    accessMode: ReadWriteOnce
-  gitlabRegistry:
-    enabled: true
-    size: 10Gi
-    ## If defined, volume.beta.kubernetes.io/storage-class: <storageClass>
-    ## Default: volume.alpha.kubernetes.io/storage-class: default
-    ##
-    # storageClass:
-
-  postgresql:
-    persistence:
-      # storageClass:
-      size: 10Gi
-  ## Configuration values for the Redis dependency.
-  ## ref: https://github.com/kubernetes/charts/blob/master/stable/redis/README.md
-  ##
-  redis:
-    persistence:
-      # storageClass:
-      size: 10Gi
-```
-
->**Note:**
-You can make use of faster SSD drives by adding a [StorageClass] to your cluster
-and using the `storageClass` setting in the above config to the name of
-your new storage class.
-
-### Routing
-
-By default, the GitLab chart uses a service type of `LoadBalancer` which will
-result in the GitLab service being exposed externally using your cloud provider's
-load balancer.
-
-This field is configurable in your `values.yml` by setting the top-level
-`serviceType` field. See the [Service documentation][kube-srv] for more
-information on the possible values.
-
-#### Ingress routing
-
-Optionally, you can enable the Chart's ingress for use by an ingress controller
-deployed in your cluster.
-
-To enable the ingress, edit its section in your `values.yaml`:
-
-```yaml
-ingress:
-  ## If true, gitlab Ingress will be created
-  ##
-  enabled: true
-
-  ## gitlab Ingress hostnames
-  ## Must be provided if Ingress is enabled
-  ##
-  hosts:
-    - gitlab.example.com
-
-  ## gitlab Ingress annotations
-  ##
-  annotations:
-    kubernetes.io/ingress.class: nginx
-```
-
-You must also provide the list of hosts that the ingress will use. In order for
-you ingress controller to work with the GitLab Ingress, you will need to specify
-its class in an annotation.
-
->**Note:**
-The Ingress alone doesn't expose GitLab externally. You need to have a Ingress controller setup to do that.
-Setting up an Ingress controller can be done by installing the `nginx-ingress` helm chart. But be sure
-to read the [documentation](https://github.com/kubernetes/charts/blob/master/stable/nginx-ingress/README.md).
->**Note:**
-If you would like to use the Registry, you will also need to ensure your Ingress supports a [sufficiently large request size](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size).
-
-#### Preserving Source IPs
-
-If you are using the `LoadBalancer` serviceType you may run into issues where user IP addresses in the GitLab
-logs, and used in abuse throttling are not accurate. This is due to how Kubernetes uses source NATing on cluster nodes without endpoints.
-
-See the [Kubernetes documentation](https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer) for more information.
-
-To fix this you can add the following service annotation to your `values.yaml`
-
-```yaml
-## For minikube, set this to NodePort, elsewhere use LoadBalancer
-## ref: http://kubernetes.io/docs/user-guide/services/#publishing-services---service-types
-##
-serviceType: LoadBalancer
-
-## Optional annotations for gitlab service.
-serviceAnnotations:
-  service.beta.kubernetes.io/external-traffic: "OnlyLocal"
-```
-
->**Note:**
-If you are using the ingress routing, you will likely also need to specify the annotation on the service for the ingress
-controller. For `nginx-ingress` you can check the
-[configuration documentation](https://github.com/kubernetes/charts/blob/master/stable/nginx-ingress/README.md#configuration)
-on how to add the annotation to the  `controller.service.annotations` array.
-
->**Note:**
-When using the `nginx-ingress` controller on Google Kubernetes Engine (GKE), and using the `external-traffic` annotation,
-you will need to additionally set the `controller.kind` to be DaemonSet. Otherwise only pods running on the same node
-as the nginx controller will be able to reach GitLab. This may result in pods within your cluster not being able to reach GitLab.
-See the [Kubernetes documentation](https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer) and
-[nginx-ingress configuration documentation](https://github.com/kubernetes/charts/blob/master/stable/nginx-ingress/README.md#configuration)
-for more information.
-
-### External database
-
-You can configure the GitLab Helm chart to connect to an external PostgreSQL
-database.
-
->**Note:**
-This is currently our recommended approach for a Production setup.
-
-To use an external database, in your `values.yaml`, disable the included
-PostgreSQL dependency, then configure access to your database:
-
-```yaml
-dbHost: "<reachable postgres hostname>"
-dbPassword: "<password for the user with access to the db>"
-dbUsername: "<user with read/write access to the database>"
-dbDatabase: "<database name on postgres to connect to for GitLab>"
-
-postgresql:
-  # Sets whether the PostgreSQL helm chart is used as a dependency
-  enabled: false
-```
-
-Be sure to check the GitLab documentation on how to
-[configure the external database](../requirements.md#postgresql-requirements)
-
-You can also configure the chart to use an external Redis server, but this is
-not required for basic production use:
-
-```yaml
-dbHost: "<reachable redis hostname>"
-dbPassword: "<password>"
-
-redis:
-  # Sets whether the Redis helm chart is used as a dependency
-  enabled: false
-```
-
-### Sending email
-
-By default, the GitLab container will not be able to send email from your cluster.
-In order to send email, you should configure SMTP settings in the
-`omnibusConfigRuby` section, as per the [GitLab Omnibus documentation](https://docs.gitlab.com/omnibus/settings/smtp.html).
-
->**Note:**
-Some cloud providers restrict emails being sent out on SMTP, so you will have
-to use a SMTP service that is supported by your provider. See this
-[Google Cloud Platform page](https://cloud.google.com/compute/docs/tutorials/sending-mail/)
-as and example.
-
-Here is an example configuration for Mailgun SMTP support:
-
-```yaml
-omnibusConfigRuby: |
-  # This is example config of what you may already have in your omnibusConfigRuby object
-  unicorn['worker_processes'] = 2;
-  gitlab_rails['trusted_proxies'] = ["10.0.0.0/8","172.16.0.0/12","192.168.0.0/16"];
-
-  # SMTP settings
-  gitlab_rails['smtp_enable'] = true
-  gitlab_rails['smtp_address'] = "smtp.mailgun.org"
-  gitlab_rails['smtp_port'] = 2525 # High port needed for Google Cloud
-  gitlab_rails['smtp_authentication'] = "plain"
-  gitlab_rails['smtp_enable_starttls_auto'] = false
-  gitlab_rails['smtp_user_name'] = "postmaster@mg.your-mail-domain"
-  gitlab_rails['smtp_password'] = "you-password"
-  gitlab_rails['smtp_domain'] = "mg.your-mail-domain"
-```
-
-### HTTPS configuration
-
-To setup HTTPS access to your GitLab server, first you need to configure the
-chart to use the [ingress](#ingress-routing).
-
-GitLab's config should be updated to support [proxied SSL](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl).
-
-In addition to having a Ingress Controller deployed and the basic ingress
-settings configured, you will also need to specify in the ingress settings
-which hosts to use HTTPS for.
-
-Make sure `externalUrl` now includes `https://` instead of `http://` in its
-value, and update the `omnibusConfigRuby` section:
-
-```yaml
-externalUrl: 'https://gitlab.example.com'
-
-omnibusConfigRuby: |
-  # This is example config of what you may already have in your omnibusConfigRuby object
-  unicorn['worker_processes'] = 2;
-  gitlab_rails['trusted_proxies'] = ["10.0.0.0/8","172.16.0.0/12","192.168.0.0/16"];
-
-  # These are the settings needed to support proxied SSL
-  nginx['listen_port'] = 80
-  nginx['listen_https'] = false
-  nginx['proxy_set_headers'] = {
-    "X-Forwarded-Proto" => "https",
-    "X-Forwarded-Ssl" => "on"
-  }
-
-ingress:
-  enabled: true
-  annotations:
-   kubernetes.io/ingress.class: nginx
-   # kubernetes.io/tls-acme: 'true' Annotation used for letsencrypt support
-
-  hosts:
-    - gitlab.example.com
-
-    ## gitlab Ingress TLS configuration
-    ## Secrets must be created in the namespace, and is not done for you in this chart
-    ##
-    tls:
-      - secretName: gitlab-tls
-        hosts:
-          - gitlab.example.com
-```
-
-You will need to create the named secret in your cluster, specifying the private
-and public certificate pair using the format outlined in the
-[ingress documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls).
-
-Alternatively, you can use the `kubernetes.io/tls-acme` annotation, and install
-the `kube-lego` chart to your cluster to have Let's Encrypt issue your
-certificate. See the [kube-lego documentation](https://github.com/kubernetes/charts/blob/master/stable/kube-lego/README.md)
-for more information.
-
-### Enabling the GitLab Container Registry
-
-The GitLab Registry is disabled by default but can be enabled by providing an
-external URL for it in the configuration. In order for the Registry to be easily
-used by GitLab CI and your Kubernetes cluster, you will need to set it up with
-a TLS certificate, so these examples will include the ingress settings for that
-as well. See the [HTTPS Configuration section](#https-configuration)
-for more explanation on some of these settings.
-
-Example config:
-
-```yaml
-externalUrl: 'https://gitlab.example.com'
-
-omnibusConfigRuby: |
-  # This is example config of what you may already have in your omnibusConfigRuby object
-  unicorn['worker_processes'] = 2;
-  gitlab_rails['trusted_proxies'] = ["10.0.0.0/8","172.16.0.0/12","192.168.0.0/16"];
-
-  registry_external_url 'https://registry.example.com';
-
-  # These are the settings needed to support proxied SSL
-  nginx['listen_port'] = 80
-  nginx['listen_https'] = false
-  nginx['proxy_set_headers'] = {
-    "X-Forwarded-Proto" => "https",
-    "X-Forwarded-Ssl" => "on"
-  }
-  registry_nginx['listen_port'] = 80
-  registry_nginx['listen_https'] = false
-  registry_nginx['proxy_set_headers'] = {
-    "X-Forwarded-Proto" => "https",
-    "X-Forwarded-Ssl" => "on"
-  }
-
-ingress:
-  enabled: true
-  annotations:
-   kubernetes.io/ingress.class: nginx
-   # kubernetes.io/tls-acme: 'true' Annotation used for letsencrypt support
-
-  hosts:
-    - gitlab.example.com
-    - registry.example.com
-
-    ## gitlab Ingress TLS configuration
-    ## Secrets must be created in the namespace, and is not done for you in this chart
-    ##
-    tls:
-      - secretName: gitlab-tls
-        hosts:
-          - gitlab.example.com
-          - registry.example.com
-```
-
-## Installing GitLab using the Helm Chart
-> You may see a temporary error message `SchedulerPredicates failed due to PersistentVolumeClaim is not bound` while storage provisions. Once the storage provisions, the pods will automatically restart. This may take a couple minutes depending on your cloud provider. If the error persists, please review the [prerequisites](#prerequisites) to ensure you have enough RAM, CPU, and storage.
-
-Add the GitLab Helm repository and initialize Helm:
-
-```bash
-helm repo add gitlab https://charts.gitlab.io
-helm init
-```
-
-Once you [have configured](#configuration) GitLab in your `values.yml` file,
-run the following:
-
-```bash
-helm install --namespace <NAMESPACE> --name gitlab -f <CONFIG_VALUES_FILE> gitlab/gitlab
-```
-
-where:
-
-- `<NAMESPACE>` is the Kubernetes namespace where you want to install GitLab.
-- `<CONFIG_VALUES_FILE>` is the path to values file containing your custom
-  configuration. See the [Configuration](#configuration) section to create it.
-
-## Updating GitLab using the Helm Chart
-
-Once your GitLab Chart is installed, configuration changes and chart updates
-should we done using `helm upgrade`
-
-```bash
-helm upgrade --namespace <NAMESPACE> -f <CONFIG_VALUES_FILE> <RELEASE-NAME> gitlab/gitlab
-```
-
-where:
-
-- `<NAMESPACE>` is the Kubernetes namespace where GitLab is installed.
-- `<CONFIG_VALUES_FILE>` is the path to values file containing your custom
-  [configuration] (#configuration).
-- `<RELEASE-NAME>` is the name you gave the chart when installing it.
-  In the [Install section](#installing) we called it `gitlab`.
-
-## Uninstalling GitLab using the Helm Chart
-
-To uninstall the GitLab Chart, run the following:
-
-```bash
-helm delete --namespace <NAMESPACE> <RELEASE-NAME>
-```
-
-where:
-
-- `<NAMESPACE>` is the Kubernetes namespace where GitLab is installed.
-- `<RELEASE-NAME>` is the name you gave the chart when installing it.
-  In the [Install section](#installing) we called it `gitlab`.
-
-[kube-srv]: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services---service-types
-[storageclass]: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#storageclasses
+Installation instructions and known issues during alpha are available at the [project page](https://gitlab.com/charts/gitlab/).
\ No newline at end of file

doc/install/kubernetes/index.md

@@ -10,10 +10,9 @@ should be deployed, upgraded, and configured.
 ## Chart Overview
 
 * **[GitLab-Omnibus](gitlab_omnibus.md)**: The best way to run GitLab on Kubernetes today, suited for small deployments. The chart is in beta and will be deprecated by the [cloud native GitLab chart](#cloud-native-gitlab-chart).
-* **[Cloud Native GitLab Chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md)**: The next generation GitLab chart, currently in alpha. Will support large deployments with horizontal scaling of individual GitLab components.
+* **[Cloud Native GitLab Chart](https://gitlab.com/charts/gitlab/blob/master/README.md)**: The next generation GitLab chart, currently in alpha. Will support large deployments with horizontal scaling of individual GitLab components.
 * Other Charts
   * [GitLab Runner Chart](gitlab_runner_chart.md): For deploying just the GitLab Runner.
-  * [Advanced GitLab Installation](gitlab_chart.md): Deprecated, being replaced by the [cloud native GitLab chart](#cloud-native-gitlab-chart). Provides additional deployment options, but provides less functionality out-of-the-box.
   * [Community Contributed Charts](#community-contributed-charts): Community contributed charts, deprecated by the official GitLab chart.
 
 ## GitLab-Omnibus Chart (Recommended)
@@ -27,7 +26,7 @@ Learn more about the [gitlab-omnibus chart](gitlab_omnibus.md).
 
 ## Cloud Native GitLab Chart
 
-GitLab is working towards building a [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). A key part of this effort is to isolate each service into its [own Docker container and Helm chart](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2420), rather than utilizing the all-in-one container image of the [current chart](#gitlab-omnibus-chart-recommended).
+GitLab is working towards building a [cloud native GitLab chart](https://gitlab.com/charts/gitlab/blob/master/README.md). A key part of this effort is to isolate each service into its [own Docker container and Helm chart](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2420), rather than utilizing the all-in-one container image of the [current chart](#gitlab-omnibus-chart-recommended).
 
 By offering individual containers and charts, we will be able to provide a number of benefits:
 * Easier horizontal scaling of each service,
@@ -37,7 +36,7 @@ By offering individual containers and charts, we will be able to provide a numbe
 
 Presently this chart is available in alpha for testing, and not recommended for production use. 
 
-Learn more about the [cloud native GitLab chart here ](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md) and [here [Video]](https://youtu.be/Z6jWR8Z8dv8).
+Learn more about the [cloud native GitLab chart here ](https://gitlab.com/charts/gitlab/blob/master/README.md) and [here [Video]](https://youtu.be/Z6jWR8Z8dv8).
 
 ## Other Charts
 

doc/integration/github.md

@@ -69,7 +69,7 @@ GitHub will generate an application ID and secret key for you to use.
           "name" => "github",
           "app_id" => "YOUR_APP_ID",
           "app_secret" => "YOUR_APP_SECRET",
-          "url" => "https://github.com/",
+          "url" => "https://github.example.com/",
           "args" => { "scope" => "user:email" }
         }
       ]
@@ -125,7 +125,7 @@ For omnibus package:
       "name" => "github",
       "app_id" => "YOUR_APP_ID",
       "app_secret" => "YOUR_APP_SECRET",
-      "url" => "https://github.com/",
+      "url" => "https://github.example.com/",
       "verify_ssl" => false,
       "args" => { "scope" => "user:email" }
     }

doc/topics/autodevops/index.md

@@ -10,8 +10,30 @@ applications.
 ## Overview
 
 With Auto DevOps, the software development process becomes easier to set up
-as every project can have a complete workflow from build to deploy and monitoring,
-with minimal to zero configuration.
+as every project can have a complete workflow from verification to monitoring
+without needing to configure anything. Just push your code and GitLab takes
+care of everything else. This makes it easier to start new projects and brings
+consistency to how applications are set up throughout a company.
+
+## Comparison to application platforms and PaaS
+
+Auto DevOps provides functionality described by others as an application
+platform or as a Platform as a Service (PaaS). It takes inspiration from the
+innovative work done by [Heroku](https://www.heroku.com/) and goes beyond it
+in a couple of ways:
+
+1. Auto DevOps works with any Kubernetes cluster, you're not limited to running
+   on GitLab's infrastructure (note that many features also work without Kubernetes).
+1. There is no additional cost (no markup on the infrastructure costs), and you
+   can use a self-hosted Kubernetes cluster or Containers as a Service on any
+   public cloud (for example [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/)).
+1. Auto DevOps has more features including security testing, performance testing,
+   and code quality testing.
+1. It offers an incremental graduation path. If you need advanced customizations
+   you can start modifying the templates without having to start over on a
+   completely different platform.
+
+## Features
 
 Comprised of a set of stages, Auto DevOps brings these best practices to your
 project in an easy and automatic way:

doc/user/project/settings/import_export.md

@@ -31,7 +31,8 @@ with all their related data and be moved into a new GitLab instance.
 
 | GitLab version   | Import/Export version |
 | ---------------- | --------------------- |
-| 10.4 to current  | 0.2.2                 |
+| 10.8 to current  | 0.2.3                 |
+| 10.4             | 0.2.2                 |
 | 10.3             | 0.2.1                 |
 | 10.0             | 0.2.0                 |
 | 9.4.0            | 0.1.8                 |

lib/gitlab.rb

-require_dependency 'settings'
 require_dependency 'gitlab/popen'
 
 module Gitlab
@@ -30,6 +29,6 @@ module Gitlab
   end
 
   def self.dev_env_or_com?
-    Rails.env.test? || Rails.env.development? || org? || com?
+    Rails.env.development? || org? || com?
   end
 end

lib/gitlab/bare_repository_import/importer.rb

@@ -75,10 +75,11 @@ module Gitlab
       end
 
       def mv_repo(project)
-        FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git'))
+        storage_path = storage_path_for_shard(project.repository_storage)
+        FileUtils.mv(repo_path, project.repository.path_to_repo)
 
         if bare_repo.wiki_exists?
-          FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git'))
+          FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git'))
         end
 
         true
@@ -88,6 +89,10 @@ module Gitlab
         false
       end
 
+      def storage_path_for_shard(shard)
+        Gitlab.config.repositories.storages[shard].legacy_disk_path
+      end
+
       def find_or_create_groups
         return nil unless group_path.present?
 

lib/gitlab/ci/trace.rb

@@ -54,14 +54,14 @@ module Gitlab
       end
 
       def exist?
-        trace_artifact&.exists? || job.chunks.any? || current_path.present? || old_trace.present?
+        trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present?
       end
 
       def read
         stream = Gitlab::Ci::Trace::Stream.new do
           if trace_artifact
             trace_artifact.open
-          elsif job.chunks.any?
+          elsif job.trace_chunks.any?
             Gitlab::Ci::Trace::ChunkedIO.new(job)
           elsif current_path
             File.open(current_path, "rb")
@@ -100,7 +100,7 @@ module Gitlab
           FileUtils.rm(trace_path, force: true)
         end
 
-        job.chunks.fast_destroy_all
+        job.trace_chunks.fast_destroy_all
         job.erase_old_trace!
       end
 
@@ -108,7 +108,7 @@ module Gitlab
         raise ArchiveError, 'Already archived' if trace_artifact
         raise ArchiveError, 'Job is not finished yet' unless job.complete?
 
-        if job.chunks.any?
+        if job.trace_chunks.any?
           Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
             archive_stream!(stream)
             stream.delete!
@@ -130,7 +130,7 @@ module Gitlab
 
       def archive_stream!(stream)
         clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
-          create_job_trace!(job, clone_path)
+          create_build_trace!(job, clone_path)
         end
       end
 
@@ -146,7 +146,7 @@ module Gitlab
         end
       end
 
-      def create_job_trace!(job, path)
+      def create_build_trace!(job, path)
         File.open(path) do |stream|
           job.create_job_artifacts_trace!(
             project: job.project,

lib/gitlab/ci/trace/chunked_io.rb

@@ -5,18 +5,18 @@ module Gitlab
   module Ci
     class Trace
       class ChunkedIO
-        CHUNK_SIZE = ::Ci::JobTraceChunk::CHUNK_SIZE
+        CHUNK_SIZE = ::Ci::BuildTraceChunk::CHUNK_SIZE
 
         FailedToGetChunkError = Class.new(StandardError)
 
-        attr_reader :job
+        attr_reader :build
         attr_reader :tell, :size
         attr_reader :chunk, :chunk_range
 
         alias_method :pos, :tell
 
-        def initialize(job, &block)
-          @job = job
+        def initialize(build, &block)
+          @build = build
           @chunks_cache = []
           @tell = 0
           @size = calculate_size
@@ -140,10 +140,10 @@ module Gitlab
           @size = offset
 
           # remove all next chunks
-          job_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all
+          trace_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all
 
           # truncate current chunk
-          current_chunk.truncate(chunk_offset) if chunk_offset != 0
+          current_chunk.truncate(chunk_offset)
         ensure
           invalidate_chunk_cache
         end
@@ -157,7 +157,7 @@ module Gitlab
         end
 
         def destroy!
-          job_chunks.fast_destroy_all
+          trace_chunks.fast_destroy_all
           @tell = @size = 0
         ensure
           invalidate_chunk_cache
@@ -206,23 +206,23 @@ module Gitlab
         end
 
         def current_chunk
-          @chunks_cache[chunk_index] ||= job_chunks.find_by(chunk_index: chunk_index)
+          @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index)
         end
 
         def build_chunk
-          @chunks_cache[chunk_index] = ::Ci::JobTraceChunk.new(job: job, chunk_index: chunk_index)
+          @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index)
         end
 
         def ensure_chunk
           current_chunk || build_chunk
         end
 
-        def job_chunks
-          ::Ci::JobTraceChunk.where(job: job)
+        def trace_chunks
+          ::Ci::BuildTraceChunk.where(build: build)
         end
 
         def calculate_size
-          job_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
+          trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
         end
       end
     end

lib/gitlab/ci/trace/stream.rb

@@ -52,7 +52,6 @@ module Gitlab
 
           stream.seek(0, IO::SEEK_SET)
           stream.write(data)
-          stream.truncate(data.bytesize)
           stream.flush()
         end
 

lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb

@@ -62,21 +62,20 @@ module Gitlab
           end
 
           def move_repositories(namespace, old_full_path, new_full_path)
-            repo_paths_for_namespace(namespace).each do |repository_storage_path|
+            repo_shards_for_namespace(namespace).each do |repository_storage|
               # Ensure old directory exists before moving it
-              gitlab_shell.add_namespace(repository_storage_path, old_full_path)
+              gitlab_shell.add_namespace(repository_storage, old_full_path)
 
-              unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
-                message = "Exception moving path #{repository_storage_path} \
-                           from #{old_full_path} to #{new_full_path}"
+              unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path)
+                message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}"
                 Rails.logger.error message
               end
             end
           end
 
-          def repo_paths_for_namespace(namespace)
+          def repo_shards_for_namespace(namespace)
             projects_for_namespace(namespace).distinct.select(:repository_storage)
-              .map(&:repository_storage_path)
+              .map(&:repository_storage)
           end
 
           def projects_for_namespace(namespace)

lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb

@@ -51,7 +51,7 @@ module Gitlab
           end
 
           def move_repository(project, old_path, new_path)
-            unless gitlab_shell.mv_repository(project.repository_storage_path,
+            unless gitlab_shell.mv_repository(project.repository_storage,
                                               old_path,
                                               new_path)
               Rails.logger.error "Error moving #{old_path} to #{new_path}"

lib/gitlab/import_export.rb

@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.2.2'.freeze
+    VERSION = '0.2.3'.freeze
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)

lib/gitlab/shell.rb

@@ -65,11 +65,11 @@ module Gitlab
 
     # Init new repository
     #
-    # storage - project's storage name
+    # storage - the shard key
     # name - project disk path
     #
     # Ex.
-    #   create_repository("/path/to/storage", "gitlab/gitlab-ci")
+    #   create_repository("default", "gitlab/gitlab-ci")
     #
     def create_repository(storage, name)
       relative_path = name.dup
@@ -291,13 +291,13 @@ module Gitlab
     # Add empty directory for storing repositories
     #
     # Ex.
-    #   add_namespace("/path/to/storage", "gitlab")
+    #   add_namespace("default", "gitlab")
     #
     def add_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:add_namespace,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).add(name)
+          Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
         else
           path = full_path(storage, name)
           FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
@@ -313,13 +313,13 @@ module Gitlab
     # Every repository inside this directory will be removed too
     #
     # Ex.
-    #   rm_namespace("/path/to/storage", "gitlab")
+    #   rm_namespace("default", "gitlab")
     #
     def rm_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:remove_namespace,
                                status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).remove(name)
+          Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
         else
           FileUtils.rm_r(full_path(storage, name), force: true)
         end
@@ -338,7 +338,8 @@ module Gitlab
       Gitlab::GitalyClient.migrate(:rename_namespace,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).rename(old_name, new_name)
+          Gitlab::GitalyClient::NamespaceService.new(storage)
+            .rename(old_name, new_name)
         else
           break false if exists?(storage, new_name) || !exists?(storage, old_name)
 
@@ -374,7 +375,8 @@ module Gitlab
       Gitlab::GitalyClient.migrate(:namespace_exists,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).exists?(dir_name)
+          Gitlab::GitalyClient::NamespaceService.new(storage)
+            .exists?(dir_name)
         else
           File.exist?(full_path(storage, dir_name))
         end
@@ -398,7 +400,7 @@ module Gitlab
     def full_path(storage, dir_name)
       raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
 
-      File.join(storage, dir_name)
+      File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name)
     end
 
     def gitlab_shell_projects_path
@@ -475,14 +477,6 @@ module Gitlab
       Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
     end
 
-    def gitaly_namespace_client(storage_path)
-      storage, _value = Gitlab.config.repositories.storages.find do |storage, value|
-        value.legacy_disk_path == storage_path
-      end
-
-      Gitlab::GitalyClient::NamespaceService.new(storage)
-    end
-
     def git_timeout
       Gitlab.config.gitlab_shell.git_timeout
     end

lib/tasks/gitlab/check.rake

@@ -427,10 +427,7 @@ namespace :gitlab do
       user = User.find_by(username: username)
       if user
         repo_dirs = user.authorized_projects.map do |p|
-          File.join(
-            p.repository_storage_path,
-            "#{p.disk_path}.git"
-          )
+          p.repository.path_to_repo
         end
 
         repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }

lib/tasks/gitlab/list_repos.rake

@@ -10,9 +10,8 @@ namespace :gitlab do
     end
 
     scope.find_each do |project|
-      base = File.join(project.repository_storage_path, project.disk_path)
-      puts base + '.git'
-      puts base + '.wiki.git'
+      puts project.repository.path_to_repo
+      puts project.wiki.repository.path_to_repo
     end
   end
 end

lib/tasks/migrate/add_limits_mysql.rake

 require Rails.root.join('db/migrate/limits_to_mysql')
 require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
 require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
-require Rails.root.join('db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql')
+require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
 
 desc "GitLab | Add limits to strings in mysql database"
 task add_limits_mysql: :environment do
@@ -9,5 +9,5 @@ task add_limits_mysql: :environment do
   LimitsToMysql.new.up
   MarkdownCacheLimitsToMysql.new.up
   MergeRequestDiffFileLimitsToMysql.new.up
-  LimitsCiJobTraceChunksRawDataForMysql.new.up
+  LimitsCiBuildTraceChunksRawDataForMysql.new.up
 end

locale/gitlab.pot

@@ -8,8 +8,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-04-17 11:44+0200\n"
-"PO-Revision-Date: 2018-04-17 11:44+0200\n"
+"POT-Creation-Date: 2018-04-24 13:19+0000\n"
+"PO-Revision-Date: 2018-04-24 13:19+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -3136,6 +3136,9 @@ msgstr ""
 msgid "Select Archive Format"
 msgstr ""
 
+msgid "Select a namespace to fork the project"
+msgstr ""
+
 msgid "Select a timezone"
 msgstr ""
 

spec/controllers/profiles_controller_spec.rb

@@ -125,7 +125,7 @@ describe ProfilesController, :request_store do
         user.reload
 
         expect(response.status).to eq(302)
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{new_username}/#{project.path}.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, "#{new_username}/#{project.path}.git")).to be_truthy
       end
     end
 
@@ -143,7 +143,7 @@ describe ProfilesController, :request_store do
         user.reload
 
         expect(response.status).to eq(302)
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be_truthy
         expect(before_disk_path).to eq(project.disk_path)
       end
     end

spec/factories/ci/job_trace_chunks.rb → spec/factories/ci/build_trace_chunks.rb

 FactoryBot.define do
-  factory :ci_job_trace_chunk, class: Ci::JobTraceChunk do
+  factory :ci_build_trace_chunk, class: Ci::BuildTraceChunk do
     job factory: :ci_build
     chunk_index 0
     data_store :redis

spec/factories/projects.rb

@@ -147,7 +147,8 @@ FactoryBot.define do
 
         # We delete hooks so that gitlab-shell will not try to authenticate with
         # an API that isn't running
-        FileUtils.rm_r(File.join(project.repository_storage_path, "#{project.disk_path}.git", 'hooks'))
+        project.gitlab_shell.rm_directory(project.repository_storage,
+                                          File.join("#{project.disk_path}.git", 'hooks'))
       end
     end
 
@@ -172,7 +173,8 @@ FactoryBot.define do
       after(:create) do |project|
         raise "Failed to create repository!" unless project.create_repository
 
-        FileUtils.rm_r(File.join(project.repository_storage_path, "#{project.disk_path}.git", 'refs'))
+        project.gitlab_shell.rm_directory(project.repository_storage,
+                                          File.join("#{project.disk_path}.git", 'refs'))
       end
     end
 

spec/fast_spec_helper.rb

@@ -9,7 +9,8 @@ unless Object.respond_to?(:require_dependency)
   end
 end
 
-# Defines Gitlab and Gitlab.config which are at the center of the app
+# Defines Settings and Gitlab.config which are at the center of the app
+require_relative '../config/settings'
 require_relative '../lib/gitlab' unless defined?(Gitlab.config)
 
 require_relative 'support/rspec'

spec/helpers/projects_helper_spec.rb

@@ -274,16 +274,16 @@ describe ProjectsHelper do
     end
   end
 
-  describe '#sanitized_import_error' do
+  describe '#sanitizerepo_repo_path' do
     let(:project) { create(:project, :repository) }
+    let(:storage_path) { Gitlab.config.repositories.storages.default.legacy_disk_path }
 
     before do
-      allow(project).to receive(:repository_storage_path).and_return('/base/repo/path')
       allow(Settings.shared).to receive(:[]).with('path').and_return('/base/repo/export/path')
     end
 
     it 'removes the repo path' do
-      repo = '/base/repo/path/namespace/test.git'
+      repo = "#{storage_path}/namespace/test.git"
       import_error = "Could not clone #{repo}\n"
 
       expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')

spec/lib/gitlab/bare_repository_import/importer_spec.rb

@@ -4,6 +4,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
   let!(:admin) { create(:admin) }
   let!(:base_dir) { Dir.mktmpdir + '/' }
   let(:bare_repository) { Gitlab::BareRepositoryImport::Repository.new(base_dir, File.join(base_dir, "#{project_path}.git")) }
+  let(:gitlab_shell) { Gitlab::Shell.new }
 
   subject(:importer) { described_class.new(admin, bare_repository) }
 
@@ -84,12 +85,14 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
         importer.create_project_if_needed
 
         project = Project.find_by_full_path(project_path)
-        repo_path = File.join(project.repository_storage_path, project.disk_path + '.git')
+        repo_path = "#{project.disk_path}.git"
         hook_path = File.join(repo_path, 'hooks')
 
-        expect(File).to exist(repo_path)
-        expect(File.symlink?(hook_path)).to be true
-        expect(File.readlink(hook_path)).to eq(Gitlab.config.gitlab_shell.hooks_path)
+        expect(gitlab_shell.exists?(project.repository_storage, repo_path)).to be(true)
+        expect(gitlab_shell.exists?(project.repository_storage, hook_path)).to be(true)
+
+        full_hook_path = File.join(project.repository.path_to_repo, 'hooks')
+        expect(File.readlink(full_hook_path)).to eq(Gitlab.config.gitlab_shell.hooks_path)
       end
 
       context 'hashed storage enabled' do
@@ -144,8 +147,8 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
 
       project = Project.find_by_full_path("#{admin.full_path}/#{project_path}")
 
-      expect(File).to exist(File.join(project.repository_storage_path, project.disk_path + '.git'))
-      expect(File).to exist(File.join(project.repository_storage_path, project.disk_path + '.wiki.git'))
+      expect(gitlab_shell.exists?(project.repository_storage, project.disk_path + '.git')).to be(true)
+      expect(gitlab_shell.exists?(project.repository_storage, project.disk_path + '.wiki.git')).to be(true)
     end
 
     it 'moves an existing project to the correct path' do
@@ -155,7 +158,9 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
       project = build(:project, :legacy_storage, :repository)
       original_commit_count = project.repository.commit_count
 
-      bare_repo = Gitlab::BareRepositoryImport::Repository.new(project.repository_storage_path, project.repository.path)
+      legacy_path = Gitlab.config.repositories.storages[project.repository_storage].legacy_disk_path
+
+      bare_repo = Gitlab::BareRepositoryImport::Repository.new(legacy_path, project.repository.path)
       gitlab_importer = described_class.new(admin, bare_repo)
 
       expect(gitlab_importer).to receive(:create_project).and_call_original
@@ -183,7 +188,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
 
       project = Project.find_by_full_path(project_path)
 
-      expect(File).to exist(File.join(project.repository_storage_path, project.disk_path + '.wiki.git'))
+      expect(gitlab_shell.exists?(project.repository_storage, project.disk_path + '.wiki.git')).to be(true)
     end
   end
 

spec/lib/gitlab/bare_repository_import/repository_spec.rb

@@ -67,7 +67,7 @@ describe ::Gitlab::BareRepositoryImport::Repository do
     end
 
     after do
-      gitlab_shell.remove_repository(root_path, hashed_path)
+      gitlab_shell.remove_repository(repository_storage, hashed_path)
     end
 
     subject { described_class.new(root_path, repo_path) }

spec/lib/gitlab/ci/trace/chunked_io_spec.rb

@@ -3,8 +3,8 @@ require 'spec_helper'
 describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
   include ChunkedIOHelpers
 
-  set(:job) { create(:ci_build, :running) }
-  let(:chunked_io) { described_class.new(job) }
+  set(:build) { create(:ci_build, :running) }
+  let(:chunked_io) { described_class.new(build) }
 
   before do
     stub_feature_flags(ci_enable_live_trace: true)
@@ -13,7 +13,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
   context "#initialize" do
     context 'when a chunk exists' do
       before do
-        job.trace.set('ABC')
+        build.trace.set('ABC')
       end
 
       it { expect(chunked_io.size).to eq(3) }
@@ -22,7 +22,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     context 'when two chunks exist' do
       before do
         stub_buffer_size(4)
-        job.trace.set('ABCDEF')
+        build.trace.set('ABCDEF')
       end
 
       it { expect(chunked_io.size).to eq(6) }
@@ -37,7 +37,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     subject { chunked_io.seek(pos, where) }
 
     before do
-      job.trace.set(sample_trace_raw)
+      build.trace.set(sample_trace_raw)
     end
 
     context 'when moves pos to end of the file' do
@@ -68,7 +68,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     subject { chunked_io.eof? }
 
     before do
-      job.trace.set(sample_trace_raw)
+      build.trace.set(sample_trace_raw)
     end
 
     context 'when current pos is at end of the file' do
@@ -94,7 +94,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     context 'when buffer size is smaller than file size' do
       before do
         stub_buffer_size(sample_trace_raw.bytesize / 2)
-        job.trace.set(sample_trace_raw)
+        build.trace.set(sample_trace_raw)
       end
 
       it 'yields lines' do
@@ -106,7 +106,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     context 'when buffer size is larger than file size' do
       before do
         stub_buffer_size(sample_trace_raw.bytesize * 2)
-        job.trace.set(sample_trace_raw)
+        build.trace.set(sample_trace_raw)
       end
 
       it 'calls get_chunk only once' do
@@ -127,7 +127,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is smaller than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize / 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it { is_expected.to eq(sample_trace_raw) }
@@ -136,7 +136,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is larger than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize * 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it { is_expected.to eq(sample_trace_raw) }
@@ -149,7 +149,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is smaller than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize / 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it 'reads a trace' do
@@ -160,7 +160,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is larger than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize * 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it 'reads a trace' do
@@ -175,7 +175,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is smaller than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize / 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it 'reads a trace' do
@@ -186,7 +186,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is larger than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize * 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it 'reads a trace' do
@@ -201,7 +201,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is smaller than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize / 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it 'reads a trace' do
@@ -212,7 +212,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is larger than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize * 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it 'reads a trace' do
@@ -238,7 +238,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     context 'when buffer size is smaller than file size' do
       before do
         stub_buffer_size(sample_trace_raw.bytesize / 2)
-        job.trace.set(sample_trace_raw)
+        build.trace.set(sample_trace_raw)
       end
 
       it_behaves_like 'all line matching'
@@ -247,7 +247,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     context 'when buffer size is larger than file size' do
       before do
         stub_buffer_size(sample_trace_raw.bytesize * 2)
-        job.trace.set(sample_trace_raw)
+        build.trace.set(sample_trace_raw)
       end
 
       it_behaves_like 'all line matching'
@@ -256,7 +256,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     context 'when pos is at middle of the file' do
       before do
         stub_buffer_size(sample_trace_raw.bytesize / 2)
-        job.trace.set(sample_trace_raw)
+        build.trace.set(sample_trace_raw)
 
         chunked_io.seek(chunked_io.size / 2)
         string_io.seek(string_io.size / 2)
@@ -316,7 +316,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is smaller than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize / 2)
-          job.trace.set(exist_data)
+          build.trace.set(exist_data)
         end
 
         it_behaves_like 'appends a trace'
@@ -325,7 +325,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is larger than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize * 2)
-          job.trace.set(exist_data)
+          build.trace.set(exist_data)
         end
 
         it_behaves_like 'appends a trace'
@@ -349,7 +349,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is smaller than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize / 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it_behaves_like 'truncates a trace'
@@ -358,7 +358,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
       context 'when buffer size is larger than file size' do
         before do
           stub_buffer_size(sample_trace_raw.bytesize * 2)
-          job.trace.set(sample_trace_raw)
+          build.trace.set(sample_trace_raw)
         end
 
         it_behaves_like 'truncates a trace'
@@ -370,14 +370,14 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
     subject { chunked_io.destroy! }
 
     before do
-      job.trace.set(sample_trace_raw)
+      build.trace.set(sample_trace_raw)
     end
 
     it 'deletes' do
       expect { subject }.to change { chunked_io.size }
         .from(sample_trace_raw.bytesize).to(0)
 
-      expect(Ci::JobTraceChunk.where(job: job).count).to eq(0)
+      expect(Ci::BuildTraceChunk.where(build: build).count).to eq(0)
     end
   end
 end

spec/lib/gitlab/ci/trace/stream_spec.rb

 require 'spec_helper'
 
 describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
-  set(:job) { create(:ci_build, :running) }
+  set(:build) { create(:ci_build, :running) }
 
   before do
     stub_feature_flags(ci_enable_live_trace: true)
@@ -83,7 +83,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
     context 'when stream is ChunkedIO' do
       let(:stream) do
         described_class.new do
-          Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
             chunked_io.write((1..8).to_a.join("\n"))
             chunked_io.seek(0, IO::SEEK_SET)
           end
@@ -137,7 +137,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
     context 'when stream is ChunkedIO' do
       let(:stream) do
         described_class.new do
-          Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
             chunked_io.write('12345678')
             chunked_io.seek(0, IO::SEEK_SET)
           end
@@ -175,7 +175,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
     context 'when stream is ChunkedIO' do
       let(:stream) do
         described_class.new do
-          Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
             chunked_io.write('12345678')
             chunked_io.seek(0, IO::SEEK_SET)
           end
@@ -234,7 +234,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
     context 'when stream is ChunkedIO' do
       let(:stream) do
         described_class.new do
-          Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
             chunked_io.write(File.binread(path))
             chunked_io.seek(0, IO::SEEK_SET)
           end
@@ -283,7 +283,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
     context 'when stream is ChunkedIO' do
       let(:stream) do
         described_class.new do
-          Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
             chunked_io.write("1234")
             chunked_io.seek(0, IO::SEEK_SET)
           end
@@ -318,7 +318,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
     context 'when stream is ChunkedIO' do
       let(:stream) do
         described_class.new do
-          Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
             chunked_io.write("12\n34\n56")
             chunked_io.seek(0, IO::SEEK_SET)
           end
@@ -473,7 +473,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
     context 'when stream is ChunkedIO' do
       let(:stream) do
         described_class.new do
-          Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io|
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
             chunked_io.write(data)
             chunked_io.seek(0, IO::SEEK_SET)
           end

spec/lib/gitlab/ci/trace_spec.rb

@@ -436,7 +436,7 @@ describe Gitlab::Ci::Trace, :clean_gitlab_redis_cache do
       it "can be erased" do
         trace.erase!
         expect(trace.exist?).to be(false)
-        expect(Ci::JobTraceChunk.where(job: build)).not_to be_exist
+        expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist
       end
 
       it "returns live trace data" do
@@ -512,7 +512,7 @@ describe Gitlab::Ci::Trace, :clean_gitlab_redis_cache do
         expect(build.trace.exist?).to be_truthy
         expect(build.job_artifacts_trace.file.exists?).to be_truthy
         expect(build.job_artifacts_trace.file.filename).to eq('job.log')
-        expect(Ci::JobTraceChunk.where(job: build)).not_to be_exist
+        expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist
         expect(src_checksum)
           .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
         expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)

spec/lib/gitlab/git/repository_spec.rb

@@ -689,7 +689,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
 
     after do
-      Gitlab::Shell.new.remove_repository(storage_path, 'my_project')
+      Gitlab::Shell.new.remove_repository('default', 'my_project')
     end
 
     shared_examples 'repository mirror fecthing' do

spec/lib/gitlab/import_export/wiki_restorer_spec.rb

@@ -24,8 +24,8 @@ describe Gitlab::ImportExport::WikiRestorer do
 
     after do
       FileUtils.rm_rf(export_path)
-      Gitlab::Shell.new.remove_repository(project_with_wiki.wiki.repository_storage_path, project_with_wiki.wiki.disk_path)
-      Gitlab::Shell.new.remove_repository(project.wiki.repository_storage_path, project.wiki.disk_path)
+      Gitlab::Shell.new.remove_repository(project_with_wiki.wiki.repository_storage, project_with_wiki.wiki.disk_path)
+      Gitlab::Shell.new.remove_repository(project.wiki.repository_storage, project.wiki.disk_path)
     end
 
     it 'restores the wiki repo successfully' do

spec/lib/gitlab/shell_spec.rb

@@ -447,18 +447,18 @@ describe Gitlab::Shell do
       let(:disk_path) { "#{project.disk_path}.git" }
 
       it 'returns true when the command succeeds' do
-        expect(gitlab_shell.exists?(project.repository_storage_path, disk_path)).to be(true)
+        expect(gitlab_shell.exists?(project.repository_storage, disk_path)).to be(true)
 
-        expect(gitlab_shell.remove_repository(project.repository_storage_path, project.disk_path)).to be(true)
+        expect(gitlab_shell.remove_repository(project.repository_storage, project.disk_path)).to be(true)
 
-        expect(gitlab_shell.exists?(project.repository_storage_path, disk_path)).to be(false)
+        expect(gitlab_shell.exists?(project.repository_storage, disk_path)).to be(false)
       end
 
       it 'keeps the namespace directory' do
-        gitlab_shell.remove_repository(project.repository_storage_path, project.disk_path)
+        gitlab_shell.remove_repository(project.repository_storage, project.disk_path)
 
-        expect(gitlab_shell.exists?(project.repository_storage_path, disk_path)).to be(false)
-        expect(gitlab_shell.exists?(project.repository_storage_path, project.disk_path.gsub(project.name, ''))).to be(true)
+        expect(gitlab_shell.exists?(project.repository_storage, disk_path)).to be(false)
+        expect(gitlab_shell.exists?(project.repository_storage, project.disk_path.gsub(project.name, ''))).to be(true)
       end
     end
 
@@ -469,18 +469,18 @@ describe Gitlab::Shell do
         old_path = project2.disk_path
         new_path = "project/new_path"
 
-        expect(gitlab_shell.exists?(project2.repository_storage_path, "#{old_path}.git")).to be(true)
-        expect(gitlab_shell.exists?(project2.repository_storage_path, "#{new_path}.git")).to be(false)
+        expect(gitlab_shell.exists?(project2.repository_storage, "#{old_path}.git")).to be(true)
+        expect(gitlab_shell.exists?(project2.repository_storage, "#{new_path}.git")).to be(false)
 
-        expect(gitlab_shell.mv_repository(project2.repository_storage_path, old_path, new_path)).to be_truthy
+        expect(gitlab_shell.mv_repository(project2.repository_storage, old_path, new_path)).to be_truthy
 
-        expect(gitlab_shell.exists?(project2.repository_storage_path, "#{old_path}.git")).to be(false)
-        expect(gitlab_shell.exists?(project2.repository_storage_path, "#{new_path}.git")).to be(true)
+        expect(gitlab_shell.exists?(project2.repository_storage, "#{old_path}.git")).to be(false)
+        expect(gitlab_shell.exists?(project2.repository_storage, "#{new_path}.git")).to be(true)
       end
 
       it 'returns false when the command fails' do
-        expect(gitlab_shell.mv_repository(project2.repository_storage_path, project2.disk_path, '')).to be_falsy
-        expect(gitlab_shell.exists?(project2.repository_storage_path, "#{project2.disk_path}.git")).to be(true)
+        expect(gitlab_shell.mv_repository(project2.repository_storage, project2.disk_path, '')).to be_falsy
+        expect(gitlab_shell.exists?(project2.repository_storage, "#{project2.disk_path}.git")).to be(true)
       end
     end
 
@@ -679,48 +679,48 @@ describe Gitlab::Shell do
 
   describe 'namespace actions' do
     subject { described_class.new }
-    let(:storage_path) { Gitlab.config.repositories.storages.default.legacy_disk_path }
+    let(:storage) { Gitlab.config.repositories.storages.keys.first }
 
     describe '#add_namespace' do
       it 'creates a namespace' do
-        subject.add_namespace(storage_path, "mepmep")
+        subject.add_namespace(storage, "mepmep")
 
-        expect(subject.exists?(storage_path, "mepmep")).to be(true)
+        expect(subject.exists?(storage, "mepmep")).to be(true)
       end
     end
 
     describe '#exists?' do
       context 'when the namespace does not exist' do
         it 'returns false' do
-          expect(subject.exists?(storage_path, "non-existing")).to be(false)
+          expect(subject.exists?(storage, "non-existing")).to be(false)
         end
       end
 
       context 'when the namespace exists' do
         it 'returns true' do
-          subject.add_namespace(storage_path, "mepmep")
+          subject.add_namespace(storage, "mepmep")
 
-          expect(subject.exists?(storage_path, "mepmep")).to be(true)
+          expect(subject.exists?(storage, "mepmep")).to be(true)
         end
       end
     end
 
     describe '#remove' do
       it 'removes the namespace' do
-        subject.add_namespace(storage_path, "mepmep")
-        subject.rm_namespace(storage_path, "mepmep")
+        subject.add_namespace(storage, "mepmep")
+        subject.rm_namespace(storage, "mepmep")
 
-        expect(subject.exists?(storage_path, "mepmep")).to be(false)
+        expect(subject.exists?(storage, "mepmep")).to be(false)
       end
     end
 
     describe '#mv_namespace' do
       it 'renames the namespace' do
-        subject.add_namespace(storage_path, "mepmep")
-        subject.mv_namespace(storage_path, "mepmep", "2mep")
+        subject.add_namespace(storage, "mepmep")
+        subject.mv_namespace(storage, "mepmep", "2mep")
 
-        expect(subject.exists?(storage_path, "mepmep")).to be(false)
-        expect(subject.exists?(storage_path, "2mep")).to be(true)
+        expect(subject.exists?(storage, "mepmep")).to be(false)
+        expect(subject.exists?(storage, "2mep")).to be(true)
       end
     end
   end

spec/migrations/assure_commits_count_for_merge_request_diff_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20180425131009_assure_commits_count_for_merge_request_diff.rb')
+
+describe AssureCommitsCountForMergeRequestDiff, :migration, :sidekiq, :redis do
+  let(:migration) { spy('migration') }
+
+  before do
+    allow(Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount)
+      .to receive(:new).and_return(migration)
+  end
+
+  context 'when there are still unmigrated commit_counts afterwards' do
+    let(:namespaces) { table('namespaces') }
+    let(:projects) { table('projects') }
+    let(:merge_requests) { table('merge_requests') }
+    let(:diffs) { table('merge_request_diffs') }
+
+    before do
+      namespace = namespaces.create(name: 'foo', path: 'foo')
+      project = projects.create!(namespace_id: namespace.id)
+      merge_request = merge_requests.create!(source_branch: 'x', target_branch: 'y', target_project_id: project.id)
+      diffs.create!(commits_count: nil, merge_request_id: merge_request.id)
+      diffs.create!(commits_count: nil, merge_request_id: merge_request.id)
+    end
+
+    it 'migrates commit_counts sequentially in batches' do
+      migrate!
+
+      expect(migration).to have_received(:perform).once
+    end
+  end
+end

spec/models/ci/job_trace_chunk_spec.rb → spec/models/ci/build_trace_chunk_spec.rb

 require 'spec_helper'
 
-describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
-  set(:job) { create(:ci_build, :running) }
+describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do
+  set(:build) { create(:ci_build, :running) }
   let(:chunk_index) { 0 }
   let(:data_store) { :redis }
   let(:raw_data) { nil }
-
-  let(:job_trace_chunk) do
-    described_class.new(job: job, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data)
+  
+  let(:build_trace_chunk) do
+    described_class.new(build: build, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data)
   end
 
   describe 'CHUNK_SIZE' do
@@ -17,13 +17,13 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
   end
 
   describe '#data' do
-    subject { job_trace_chunk.data }
+    subject { build_trace_chunk.data }
 
     context 'when data_store is redis' do
       let(:data_store) { :redis }
 
       before do
-        job_trace_chunk.send(:redis_set_data, 'Sample data in redis')
+        build_trace_chunk.send(:redis_set_data, 'Sample data in redis')
       end
 
       it { is_expected.to eq('Sample data in redis') }
@@ -38,7 +38,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
 
     context 'when data_store is others' do
       before do
-        job_trace_chunk.send(:write_attribute, :data_store, -1)
+        build_trace_chunk.send(:write_attribute, :data_store, -1)
       end
 
       it { expect { subject }.to raise_error('Unsupported data store') }
@@ -46,7 +46,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
   end
 
   describe '#set_data' do
-    subject { job_trace_chunk.set_data(value) }
+    subject { build_trace_chunk.set_data(value) }
 
     let(:value) { 'Sample data' }
 
@@ -60,11 +60,11 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
       let(:data_store) { :redis }
 
       it do
-        expect(job_trace_chunk.send(:redis_data)).to be_nil
+        expect(build_trace_chunk.send(:redis_data)).to be_nil
 
         subject
 
-        expect(job_trace_chunk.send(:redis_data)).to eq(value)
+        expect(build_trace_chunk.send(:redis_data)).to eq(value)
       end
 
       context 'when fullfilled chunk size' do
@@ -82,26 +82,26 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
       let(:data_store) { :db }
 
       it 'sets data' do
-        expect(job_trace_chunk.raw_data).to be_nil
+        expect(build_trace_chunk.raw_data).to be_nil
 
         subject
 
-        expect(job_trace_chunk.raw_data).to eq(value)
-        expect(job_trace_chunk.persisted?).to be_truthy
+        expect(build_trace_chunk.raw_data).to eq(value)
+        expect(build_trace_chunk.persisted?).to be_truthy
       end
 
       context 'when raw_data is not changed' do
         it 'does not execute UPDATE' do
-          expect(job_trace_chunk.raw_data).to be_nil
-          job_trace_chunk.save!
+          expect(build_trace_chunk.raw_data).to be_nil
+          build_trace_chunk.save!
 
           # First set
           expect(ActiveRecord::QueryRecorder.new { subject }.count).to be > 0
-          expect(job_trace_chunk.raw_data).to eq(value)
-          expect(job_trace_chunk.persisted?).to be_truthy
+          expect(build_trace_chunk.raw_data).to eq(value)
+          expect(build_trace_chunk.persisted?).to be_truthy
 
           # Second set
-          job_trace_chunk.reload
+          build_trace_chunk.reload
           expect(ActiveRecord::QueryRecorder.new { subject }.count).to be(0)
         end
       end
@@ -117,7 +117,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
 
     context 'when data_store is others' do
       before do
-        job_trace_chunk.send(:write_attribute, :data_store, -1)
+        build_trace_chunk.send(:write_attribute, :data_store, -1)
       end
 
       it { expect { subject }.to raise_error('Unsupported data store') }
@@ -125,7 +125,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
   end
 
   describe '#truncate' do
-    subject { job_trace_chunk.truncate(offset) }
+    subject { build_trace_chunk.truncate(offset) }
 
     shared_examples_for 'truncates' do
       context 'when offset is negative' do
@@ -146,7 +146,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
         it 'truncates' do
           subject
 
-          expect(job_trace_chunk.data).to eq(data.byteslice(0, offset))
+          expect(build_trace_chunk.data).to eq(data.byteslice(0, offset))
         end
       end
     end
@@ -156,7 +156,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
       let(:data) { 'Sample data in redis' }
 
       before do
-        job_trace_chunk.send(:redis_set_data, data)
+        build_trace_chunk.send(:redis_set_data, data)
       end
 
       it_behaves_like 'truncates'
@@ -172,7 +172,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
   end
 
   describe '#append' do
-    subject { job_trace_chunk.append(new_data, offset) }
+    subject { build_trace_chunk.append(new_data, offset) }
 
     let(:new_data) { 'Sample new data' }
     let(:offset) { 0 }
@@ -203,7 +203,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
         it 'appends' do
           subject
 
-          expect(job_trace_chunk.data).to eq(total_data)
+          expect(build_trace_chunk.data).to eq(total_data)
         end
       end
 
@@ -213,7 +213,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
         it 'appends' do
           subject
 
-          expect(job_trace_chunk.data).to eq(data.byteslice(0, offset) + new_data)
+          expect(build_trace_chunk.data).to eq(data.byteslice(0, offset) + new_data)
         end
       end
     end
@@ -223,7 +223,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
       let(:data) { 'Sample data in redis' }
 
       before do
-        job_trace_chunk.send(:redis_set_data, data)
+        build_trace_chunk.send(:redis_set_data, data)
       end
 
       it_behaves_like 'appends'
@@ -239,7 +239,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
   end
 
   describe '#size' do
-    subject { job_trace_chunk.size }
+    subject { build_trace_chunk.size }
 
     context 'when data_store is redis' do
       let(:data_store) { :redis }
@@ -248,7 +248,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
         let(:data) { 'Sample data in redis' }
 
         before do
-          job_trace_chunk.send(:redis_set_data, data)
+          build_trace_chunk.send(:redis_set_data, data)
         end
 
         it { is_expected.to eq(data.bytesize) }
@@ -276,7 +276,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
   end
 
   describe '#use_database!' do
-    subject { job_trace_chunk.use_database! }
+    subject { build_trace_chunk.use_database! }
 
     context 'when data_store is redis' do
       let(:data_store) { :redis }
@@ -285,19 +285,19 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
         let(:data) { 'Sample data in redis' }
 
         before do
-          job_trace_chunk.send(:redis_set_data, data)
+          build_trace_chunk.send(:redis_set_data, data)
         end
 
         it 'stashes the data' do
-          expect(job_trace_chunk.data_store).to eq('redis')
-          expect(job_trace_chunk.send(:redis_data)).to eq(data)
-          expect(job_trace_chunk.raw_data).to be_nil
+          expect(build_trace_chunk.data_store).to eq('redis')
+          expect(build_trace_chunk.send(:redis_data)).to eq(data)
+          expect(build_trace_chunk.raw_data).to be_nil
 
           subject
 
-          expect(job_trace_chunk.data_store).to eq('db')
-          expect(job_trace_chunk.send(:redis_data)).to be_nil
-          expect(job_trace_chunk.raw_data).to eq(data)
+          expect(build_trace_chunk.data_store).to eq('db')
+          expect(build_trace_chunk.send(:redis_data)).to be_nil
+          expect(build_trace_chunk.raw_data).to eq(data)
         end
       end
 
@@ -320,11 +320,11 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
   describe 'ExclusiveLock' do
     before do
       allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
-      stub_const('Ci::JobTraceChunk::LOCK_RETRY', 1)
+      stub_const('Ci::BuildTraceChunk::LOCK_RETRY', 1)
     end
 
     it 'raise an error' do
-      expect { job_trace_chunk.append('ABC', 0) }.to raise_error('Failed to obtain write lock')
+      expect { build_trace_chunk.append('ABC', 0) }.to raise_error('Failed to obtain write lock')
     end
   end
 
@@ -338,7 +338,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
       create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project)
     end
 
-    shared_examples_for 'deletes all job_trace_chunk and data in redis' do
+    shared_examples_for 'deletes all build_trace_chunk and data in redis' do
       it do
         project.builds.each do |build|
           Gitlab::Redis::SharedState.with do |redis|
@@ -364,20 +364,20 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
       end
     end
 
-    context 'when job_trace_chunk is destroyed' do
+    context 'when build_trace_chunk is destroyed' do
       let(:subject) do
         project.builds.each { |build| build.chunks.destroy_all }
       end
 
-      it_behaves_like 'deletes all job_trace_chunk and data in redis'
+      it_behaves_like 'deletes all build_trace_chunk and data in redis'
     end
 
-    context 'when job is destroyed' do
+    context 'when build is destroyed' do
       let(:subject) do
         project.builds.destroy_all
       end
 
-      it_behaves_like 'deletes all job_trace_chunk and data in redis'
+      it_behaves_like 'deletes all build_trace_chunk and data in redis'
     end
 
     context 'when project is destroyed' do
@@ -385,7 +385,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do
         project.destroy!
       end
 
-      it_behaves_like 'deletes all job_trace_chunk and data in redis'
+      it_behaves_like 'deletes all build_trace_chunk and data in redis'
     end
   end
 end

spec/models/namespace_spec.rb

@@ -5,6 +5,7 @@ describe Namespace do
 
   let!(:namespace) { create(:namespace) }
   let(:gitlab_shell) { Gitlab::Shell.new }
+  let(:repository_storage) { 'default' }
 
   describe 'associations' do
     it { is_expected.to have_many :projects }
@@ -201,7 +202,7 @@ describe Namespace do
       it "moves dir if path changed" do
         namespace.update_attributes(path: namespace.full_path + '_new')
 
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{namespace.path}/#{project.path}.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, "#{namespace.path}/#{project.path}.git")).to be_truthy
       end
 
       context 'with subgroups', :nested_groups do
@@ -281,7 +282,7 @@ describe Namespace do
         namespace.update_attributes(path: namespace.full_path + '_new')
 
         expect(before_disk_path).to eq(project.disk_path)
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be_truthy
       end
     end
 
@@ -322,7 +323,7 @@ describe Namespace do
       end
 
       it 'schedules the namespace for deletion' do
-        expect(GitlabShellWorker).to receive(:perform_in).with(5.minutes, :rm_namespace, repository_storage_path, deleted_path)
+        expect(GitlabShellWorker).to receive(:perform_in).with(5.minutes, :rm_namespace, repository_storage, deleted_path)
 
         namespace.destroy
       end
@@ -344,7 +345,7 @@ describe Namespace do
         end
 
         it 'schedules the namespace for deletion' do
-          expect(GitlabShellWorker).to receive(:perform_in).with(5.minutes, :rm_namespace, repository_storage_path, deleted_path)
+          expect(GitlabShellWorker).to receive(:perform_in).with(5.minutes, :rm_namespace, repository_storage, deleted_path)
 
           child.destroy
         end

spec/models/project_spec.rb

@@ -449,14 +449,6 @@ describe Project do
     end
   end
 
-  describe '#repository_storage_path' do
-    let(:project) { create(:project) }
-
-    it 'returns the repository storage path' do
-      expect(Dir.exist?(project.repository_storage_path)).to be(true)
-    end
-  end
-
   it 'returns valid url to repo' do
     project = described_class.new(path: 'somewhere')
     expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git')
@@ -1108,7 +1100,7 @@ describe Project do
   end
 
   context 'repository storage by default' do
-    let(:project) { create(:project) }
+    let(:project) { build(:project) }
 
     before do
       storages = {
@@ -1461,7 +1453,7 @@ describe Project do
         .and_return(false)
 
       allow(shell).to receive(:create_repository)
-        .with(project.repository_storage_path, project.disk_path)
+        .with(project.repository_storage, project.disk_path)
         .and_return(true)
 
       expect(project).to receive(:create_repository).with(force: true)
@@ -2636,7 +2628,7 @@ describe Project do
 
     describe '#ensure_storage_path_exists' do
       it 'delegates to gitlab_shell to ensure namespace is created' do
-        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, project.base_dir)
+        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage, project.base_dir)
 
         project.ensure_storage_path_exists
       end
@@ -2675,12 +2667,12 @@ describe Project do
 
         expect(gitlab_shell).to receive(:mv_repository)
           .ordered
-          .with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
+          .with(project.repository_storage, "#{project.namespace.full_path}/foo", "#{project.full_path}")
           .and_return(true)
 
         expect(gitlab_shell).to receive(:mv_repository)
           .ordered
-          .with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
+          .with(project.repository_storage, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
           .and_return(true)
 
         expect_any_instance_of(SystemHooksService)
@@ -2829,7 +2821,7 @@ describe Project do
       it 'delegates to gitlab_shell to ensure namespace is created' do
         allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
 
-        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, hashed_prefix)
+        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage, hashed_prefix)
 
         project.ensure_storage_path_exists
       end

spec/models/project_wiki_spec.rb

@@ -11,7 +11,7 @@ describe ProjectWiki do
   subject { project_wiki }
 
   it { is_expected.to delegate_method(:empty?).to :pages }
-  it { is_expected.to delegate_method(:repository_storage_path).to :project }
+  it { is_expected.to delegate_method(:repository_storage).to :project }
   it { is_expected.to delegate_method(:hashed_storage?).to :project }
 
   describe "#full_path" do

spec/services/groups/destroy_service_spec.rb

@@ -53,8 +53,8 @@ describe Groups::DestroyService do
         end
 
         it 'verifies that paths have been deleted' do
-          expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey
-          expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey
+          expect(gitlab_shell.exists?(project.repository_storage, group.path)).to be_falsey
+          expect(gitlab_shell.exists?(project.repository_storage, remove_path)).to be_falsey
         end
       end
     end
@@ -71,13 +71,13 @@ describe Groups::DestroyService do
 
       after do
         # Clean up stale directories
-        gitlab_shell.rm_namespace(project.repository_storage_path, group.path)
-        gitlab_shell.rm_namespace(project.repository_storage_path, remove_path)
+        gitlab_shell.rm_namespace(project.repository_storage, group.path)
+        gitlab_shell.rm_namespace(project.repository_storage, remove_path)
       end
 
       it 'verifies original paths and projects still exist' do
-        expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_truthy
-        expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey
+        expect(gitlab_shell.exists?(project.repository_storage, group.path)).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, remove_path)).to be_falsey
         expect(Project.unscoped.count).to eq(1)
         expect(Group.unscoped.count).to eq(2)
       end
@@ -144,7 +144,7 @@ describe Groups::DestroyService do
       let!(:project) { create(:project, :legacy_storage, :empty_repo, namespace: group) }
 
       it 'removes repository' do
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
+        expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be_falsey
       end
     end
 
@@ -152,7 +152,7 @@ describe Groups::DestroyService do
       let!(:project) { create(:project, :empty_repo, namespace: group) }
 
       it 'removes repository' do
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
+        expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be_falsey
       end
     end
   end

spec/services/projects/create_service_spec.rb

@@ -171,7 +171,6 @@ describe Projects::CreateService, '#execute' do
 
     context 'when another repository already exists on disk' do
       let(:repository_storage) { 'default' }
-      let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage].legacy_disk_path }
 
       let(:opts) do
         {
@@ -186,7 +185,7 @@ describe Projects::CreateService, '#execute' do
         end
 
         after do
-          gitlab_shell.remove_repository(repository_storage_path, "#{user.namespace.full_path}/existing")
+          gitlab_shell.remove_repository(repository_storage, "#{user.namespace.full_path}/existing")
         end
 
         it 'does not allow to create a project when path matches existing repository on disk' do
@@ -222,7 +221,7 @@ describe Projects::CreateService, '#execute' do
         end
 
         after do
-          gitlab_shell.remove_repository(repository_storage_path, hashed_path)
+          gitlab_shell.remove_repository(repository_storage, hashed_path)
         end
 
         it 'does not allow to create a project when path matches existing repository on disk' do

spec/services/projects/destroy_service_spec.rb

@@ -18,8 +18,8 @@ describe Projects::DestroyService do
     it 'deletes the project' do
       expect(Project.unscoped.all).not_to include(project)
 
-      expect(project.gitlab_shell.exists?(project.repository_storage_path, path + '.git')).to be_falsey
-      expect(project.gitlab_shell.exists?(project.repository_storage_path, remove_path + '.git')).to be_falsey
+      expect(project.gitlab_shell.exists?(project.repository_storage, path + '.git')).to be_falsey
+      expect(project.gitlab_shell.exists?(project.repository_storage, remove_path + '.git')).to be_falsey
     end
   end
 
@@ -252,21 +252,21 @@ describe Projects::DestroyService do
     let(:path) { project.disk_path + '.git' }
 
     before do
-      expect(project.gitlab_shell.exists?(project.repository_storage_path, path)).to be_truthy
-      expect(project.gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey
+      expect(project.gitlab_shell.exists?(project.repository_storage, path)).to be_truthy
+      expect(project.gitlab_shell.exists?(project.repository_storage, remove_path)).to be_falsey
 
       # Dont run sidekiq to check if renamed repository exists
       Sidekiq::Testing.fake! { destroy_project(project, user, {}) }
 
-      expect(project.gitlab_shell.exists?(project.repository_storage_path, path)).to be_falsey
-      expect(project.gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_truthy
+      expect(project.gitlab_shell.exists?(project.repository_storage, path)).to be_falsey
+      expect(project.gitlab_shell.exists?(project.repository_storage, remove_path)).to be_truthy
     end
 
     it 'restores the repositories' do
       Sidekiq::Testing.fake! { described_class.new(project, user).attempt_repositories_rollback }
 
-      expect(project.gitlab_shell.exists?(project.repository_storage_path, path)).to be_truthy
-      expect(project.gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey
+      expect(project.gitlab_shell.exists?(project.repository_storage, path)).to be_truthy
+      expect(project.gitlab_shell.exists?(project.repository_storage, remove_path)).to be_falsey
     end
   end
 

spec/services/projects/fork_service_spec.rb

@@ -112,7 +112,7 @@ describe Projects::ForkService do
         end
 
         after do
-          gitlab_shell.remove_repository(repository_storage_path, "#{@to_user.namespace.full_path}/#{@from_project.path}")
+          gitlab_shell.remove_repository(repository_storage, "#{@to_user.namespace.full_path}/#{@from_project.path}")
         end
 
         it 'does not allow creation' do

spec/services/projects/hashed_storage/migrate_repository_service_spec.rb

@@ -16,8 +16,8 @@ describe Projects::HashedStorage::MigrateRepositoryService do
       it 'renames project and wiki repositories' do
         service.execute
 
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.git")).to be_truthy
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.wiki.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, "#{hashed_storage.disk_path}.git")).to be_truthy
+        expect(gitlab_shell.exists?(project.repository_storage, "#{hashed_storage.disk_path}.wiki.git")).to be_truthy
       end
 
       it 'updates project to be hashed and not read-only' do
@@ -52,8 +52,8 @@ describe Projects::HashedStorage::MigrateRepositoryService do
 
         service.execute
 
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.git")).to be_falsey
-        expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.wiki.git")).to be_falsey
+        expect(gitlab_shell.exists?(project.repository_storage, "#{hashed_storage.disk_path}.git")).to be_falsey
+        expect(gitlab_shell.exists?(project.repository_storage, "#{hashed_storage.disk_path}.wiki.git")).to be_falsey
         expect(project.repository_read_only?).to be_falsey
       end
 
@@ -63,11 +63,11 @@ describe Projects::HashedStorage::MigrateRepositoryService do
 
         before do
           hashed_storage.ensure_storage_path_exists
-          gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+          gitlab_shell.mv_repository(project.repository_storage, from_name, to_name)
         end
 
         it 'does not try to move nil repository over hashed' do
-          expect(gitlab_shell).not_to receive(:mv_repository).with(project.repository_storage_path, from_name, to_name)
+          expect(gitlab_shell).not_to receive(:mv_repository).with(project.repository_storage, from_name, to_name)
           expect_move_repository("#{project.disk_path}.wiki", "#{hashed_storage.disk_path}.wiki")
 
           service.execute
@@ -76,7 +76,7 @@ describe Projects::HashedStorage::MigrateRepositoryService do
     end
 
     def expect_move_repository(from_name, to_name)
-      expect(gitlab_shell).to receive(:mv_repository).with(project.repository_storage_path, from_name, to_name).and_call_original
+      expect(gitlab_shell).to receive(:mv_repository).with(project.repository_storage, from_name, to_name).and_call_original
     end
   end
 end

spec/services/projects/transfer_service_spec.rb

@@ -84,7 +84,7 @@ describe Projects::TransferService do
     end
 
     def project_path(project)
-      File.join(project.repository_storage_path, "#{project.disk_path}.git")
+      project.repository.path_to_repo
     end
 
     def current_path
@@ -94,7 +94,7 @@ describe Projects::TransferService do
     it 'rolls back repo location' do
       attempt_project_transfer
 
-      expect(Dir.exist?(original_path)).to be_truthy
+      expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be(true)
       expect(original_path).to eq current_path
     end
 
@@ -165,7 +165,7 @@ describe Projects::TransferService do
     end
 
     after do
-      gitlab_shell.remove_repository(repository_storage_path, "#{group.full_path}/#{project.path}")
+      gitlab_shell.remove_repository(repository_storage, "#{group.full_path}/#{project.path}")
     end
 
     it { expect(@result).to eq false }

spec/services/projects/update_service_spec.rb

@@ -200,7 +200,7 @@ describe Projects::UpdateService do
         end
 
         after do
-          gitlab_shell.remove_repository(repository_storage_path, "#{user.namespace.full_path}/existing")
+          gitlab_shell.remove_repository(repository_storage, "#{user.namespace.full_path}/existing")
         end
 
         it 'does not allow renaming when new path matches existing repository on disk' do

spec/services/users/destroy_service_spec.rb

@@ -176,7 +176,7 @@ describe Users::DestroyService do
         let!(:project) { create(:project, :empty_repo, :legacy_storage, namespace: user.namespace) }
 
         it 'removes repository' do
-          expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
+          expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be_falsey
         end
       end
 
@@ -184,7 +184,7 @@ describe Users::DestroyService do
         let!(:project) { create(:project, :empty_repo, namespace: user.namespace) }
 
         it 'removes repository' do
-          expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
+          expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be_falsey
         end
       end
     end

spec/spec_helper.rb

@@ -113,10 +113,10 @@ RSpec.configure do |config|
       m.call(*args)
 
       shard_name, repository_relative_path = args
-      shard_path = Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path
       # We can't leave the hooks in place after a fork, as those would fail in tests
       # The "internal" API is not available
-      FileUtils.rm_rf(File.join(shard_path, repository_relative_path, 'hooks'))
+      Gitlab::Shell.new.rm_directory(shard_name,
+                                     File.join(repository_relative_path, 'hooks'))
     end
 
     # Enable all features by default for testing

spec/support/chunked_io/chunked_io_helpers.rb

@@ -5,7 +5,7 @@ module ChunkedIOHelpers
   end
 
   def stub_buffer_size(size)
-    stub_const('Ci::JobTraceChunk::CHUNK_SIZE', size)
+    stub_const('Ci::BuildTraceChunk::CHUNK_SIZE', size)
     stub_const('Gitlab::Ci::Trace::ChunkedIO::CHUNK_SIZE', size)
   end
 

spec/support/helpers/javascript_fixtures_helpers.rb

@@ -31,7 +31,7 @@ module JavaScriptFixturesHelpers
   end
 
   def remove_repository(project)
-    Gitlab::Shell.new.remove_repository(project.repository_storage_path, project.disk_path)
+    Gitlab::Shell.new.remove_repository(project.repository_storage, project.disk_path)
   end
 
   private

spec/support/helpers/test_env.rb

@@ -218,7 +218,8 @@ module TestEnv
   end
 
   def copy_repo(project, bare_repo:, refs:)
-    target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.disk_path}.git")
+    target_repo_path = File.expand_path(repos_path + "/#{project.disk_path}.git")
+
     FileUtils.mkdir_p(target_repo_path)
     FileUtils.cp_r("#{File.expand_path(bare_repo)}/.", target_repo_path)
     FileUtils.chmod_R 0755, target_repo_path
@@ -226,7 +227,7 @@ module TestEnv
   end
 
   def repos_path
-    Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
+    @repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
   end
 
   def backup_path

spec/tasks/gitlab/backup_rake_spec.rb

@@ -195,15 +195,12 @@ describe 'gitlab:app namespace rake task' do
     end
 
     context 'multiple repository storages' do
-      let(:storage_default) do
-        Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage'))
-      end
       let(:test_second_storage) do
         Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/custom_storage'))
       end
       let(:storages) do
         {
-          'default' => storage_default,
+          'default' => Gitlab.config.repositories.storages.default,
           'test_second_storage' => test_second_storage
         }
       end
@@ -215,8 +212,7 @@ describe 'gitlab:app namespace rake task' do
       before do
         # We only need a backup of the repositories for this test
         stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
-        FileUtils.mkdir(Settings.absolute('tmp/tests/default_storage'))
-        FileUtils.mkdir(Settings.absolute('tmp/tests/custom_storage'))
+
         allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
 
         # Avoid asking gitaly about the root ref (which will fail beacuse of the
@@ -225,14 +221,23 @@ describe 'gitlab:app namespace rake task' do
       end
 
       after do
-        FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage'))
         FileUtils.rm_rf(Settings.absolute('tmp/tests/custom_storage'))
       end
 
       it 'includes repositories in all repository storages' do
-        project_a = create(:project, :repository, repository_storage: 'default')
+        project_a = create(:project, :repository)
         project_b = create(:project, :repository, repository_storage: 'test_second_storage')
 
+        b_storage_dir = File.join(Settings.absolute('tmp/tests/custom_storage'), File.dirname(project_b.disk_path))
+
+        FileUtils.mkdir_p(b_storage_dir)
+
+        # Even when overriding the storage, we have to move it there, so it exists
+        FileUtils.mv(
+          File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'),
+          Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git')
+        )
+
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
 
         tar_contents, exit_status = Gitlab::Popen.popen(