Merge branch '759-tooling-to-move-projects-between-shards' into 'master'

Allow projects to be moved between repository storages

## What are the relevant issue numbers?

Closes #759 

See merge request !533

CHANGELOG-EE

 Please view this file on the master branch, on stable branches it's out of date.
 
 v 8.11.0 (unreleased)
+  - Allow projects to be moved between repository storages
+  - Add rake task to remove old repository copies from repositories moved to another storage
   - Performance improvement of push rules
   - Change LdapGroupSyncWorker to use new LDAP group sync classes
   - [Elastic][Fix] Commit search breaks for some URLs on gitlab-ce project

GITLAB_SHELL_VERSION

-3.2.1
+3.3.0

app/models/ability.rb

@@ -184,6 +184,8 @@ class Ability
         # Push abilities on the users team role
         rules.push(*project_team_rules(project.team, user))
 
+        rules << :change_repository_storage if user.admin?
+
         owner = user.admin? ||
                 project.owner == user ||
                 (project.group && project.group.has_owner?(user))

app/models/project.rb

@@ -1497,6 +1497,16 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def change_repository_storage(new_repository_storage_key)
+    return if repository_read_only?
+    return if repository_storage == new_repository_storage_key
+
+    raise ArgumentError unless Gitlab.config.repositories.storages.keys.include?(new_repository_storage_key)
+
+    run_after_commit { ProjectUpdateRepositoryStorageWorker.perform_async(id, new_repository_storage_key) }
+    self.repository_read_only = true
+  end
+
   private
 
   def default_branch_protected?

app/services/projects/update_repository_storage_service.rb

+module Projects
+  class UpdateRepositoryStorageService < BaseService
+    include Gitlab::ShellAdapter
+
+    def initialize(project)
+      @project = project
+    end
+
+    def execute(new_repository_storage_key)
+      new_storage_path = Gitlab.config.repositories.storages[new_repository_storage_key]
+      result = move_storage(project.path_with_namespace, new_storage_path)
+
+      if project.wiki.repository_exists?
+        result &&= move_storage("#{project.path_with_namespace}.wiki", new_storage_path)
+      end
+
+      if result
+        mark_old_paths_for_archive
+
+        project.update(repository_storage: new_repository_storage_key, repository_read_only: false)
+      else
+        project.update(repository_read_only: false)
+      end
+    end
+
+    private
+
+    def move_storage(project_path, new_storage_path)
+      gitlab_shell.mv_storage(project.repository_storage_path, project_path, new_storage_path)
+    end
+
+    def mark_old_paths_for_archive
+      old_repository_storage_path = project.repository_storage_path
+      new_project_path = moved_path(project.path_with_namespace)
+
+      # Notice that the block passed to `run_after_commit` will run with `project`
+      # as its context
+      project.run_after_commit do
+        GitlabShellWorker.perform_async(:mv_repository,
+                                        old_repository_storage_path,
+                                        path_with_namespace,
+                                        new_project_path)
+
+        if wiki.repository_exists?
+          GitlabShellWorker.perform_async(:mv_repository,
+                                          old_repository_storage_path,
+                                          "#{path_with_namespace}.wiki",
+                                          "#{new_project_path}.wiki")
+        end
+      end
+    end
+
+    def moved_path(path)
+      "#{path}+#{project.id}+moved+#{Time.now.to_i}"
+    end
+  end
+end

app/services/projects/update_service.rb

@@ -13,13 +13,20 @@ module Projects
         end
       end
 
-      new_branch = params[:default_branch]
+      new_branch = params.delete(:default_branch)
+      new_repository_storage = params.delete(:repository_storage)
 
-      if project.repository.exists? && new_branch && new_branch != project.default_branch
-        project.change_head(new_branch)
+      if project.repository.exists?
+        if new_branch && new_branch != project.default_branch
+          project.change_head(new_branch)
+        end
+
+        if new_repository_storage && can?(current_user, :change_repository_storage, project)
+          project.change_repository_storage(new_repository_storage)
+        end
       end
 
-      if project.update_attributes(params.except(:default_branch))
+      if project.update_attributes(params)
         if project.previous_changes.include?('path')
           project.rename_repo
         end

app/workers/project_update_repository_storage_worker.rb

+class ProjectUpdateRepositoryStorageWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :default
+
+  def perform(project_id, new_repository_storage_key)
+    project = Project.find(project_id)
+
+    ::Projects::UpdateRepositoryStorageService.new(project).execute(new_repository_storage_key)
+  end
+end

db/migrate/20160713200638_add_repository_read_only_to_projects.rb

+class AddRepositoryReadOnlyToProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :repository_read_only, :boolean
+  end
+end

db/schema.rb

@@ -969,6 +969,7 @@ ActiveRecord::Schema.define(version: 20160810153405) do
     t.string   "repository_storage",                 default: "default", null: false
     t.boolean  "request_access_enabled",             default: true,      null: false
     t.boolean  "has_external_wiki"
+    t.boolean  "repository_read_only"
   end
 
   add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree

doc/api/projects.md

@@ -280,7 +280,8 @@ Parameters:
       "group_name": "Gitlab Org",
       "group_access_level": 10
     }
-  ]
+  ],
+  "repository_storage": "default"
 }
 ```
 
@@ -448,6 +449,7 @@ Parameters:
 - `visibility_level` (optional)
 - `import_url` (optional)
 - `public_builds` (optional)
+- `repository_storage` (optional, available only for admins)
 
 ### Create project for user
 
@@ -473,6 +475,7 @@ Parameters:
 - `visibility_level` (optional)
 - `import_url` (optional)
 - `public_builds` (optional)
+- `repository_storage` (optional, available only for admins)
 
 ### Edit project
 
@@ -499,6 +502,7 @@ Parameters:
 - `public` (optional) - if `true` same as setting visibility_level = 20
 - `visibility_level` (optional)
 - `public_builds` (optional)
+- `repository_storage` (optional, available only for admins)
 
 On success, method returns 200 with the updated project. If parameters are
 invalid, 400 is returned.

doc/raketasks/cleanup.md

@@ -22,3 +22,13 @@ sudo gitlab-rake gitlab:cleanup:repos
 # installation from source
 bundle exec rake gitlab:cleanup:repos RAILS_ENV=production
 ```
+
+Remove old repository copies from repositories moved to another storage.
+
+```
+# omnibus-gitlab
+sudo gitlab-rake gitlab:cleanup:moved
+
+# installation from source
+bundle exec rake gitlab:cleanup:moved RAILS_ENV=production
+```

lib/api/entities.rb

@@ -94,6 +94,7 @@ module API
       expose :shared_with_groups do |project, options|
         SharedGroup.represent(project.project_group_links.all, options)
       end
+      expose :repository_storage, if: lambda { |_project, options| options[:user].try(:admin?) }
     end
 
     class ProjectMember < UserBasic

lib/api/projects.rb

@@ -105,6 +105,7 @@ module API
       #   visibility_level (optional) - 0 by default
       #   import_url (optional)
       #   public_builds (optional)
+      #   repository_storage (optional)
       # Example Request
       #   POST /projects
       post do
@@ -123,7 +124,8 @@ module API
                                      :public,
                                      :visibility_level,
                                      :import_url,
-                                     :public_builds]
+                                     :public_builds,
+                                     :repository_storage]
         attrs = map_public_to_visibility_level(attrs)
         @project = ::Projects::CreateService.new(current_user, attrs).execute
         if @project.saved?
@@ -155,6 +157,7 @@ module API
       #   visibility_level (optional)
       #   import_url (optional)
       #   public_builds (optional)
+      #   repository_storage (optional)
       # Example Request
       #   POST /projects/user/:user_id
       post "user/:user_id" do
@@ -172,7 +175,8 @@ module API
                                      :public,
                                      :visibility_level,
                                      :import_url,
-                                     :public_builds]
+                                     :public_builds,
+                                     :repository_storage]
         attrs = map_public_to_visibility_level(attrs)
         @project = ::Projects::CreateService.new(user, attrs).execute
         if @project.saved?
@@ -218,6 +222,7 @@ module API
       #   public (optional) - if true same as setting visibility_level = 20
       #   visibility_level (optional) - visibility level of a project
       #   public_builds (optional)
+      #   repository_storage (optional)
       # Example Request
       #   PUT /projects/:id
       put ':id' do
@@ -234,7 +239,8 @@ module API
                                      :shared_runners_enabled,
                                      :public,
                                      :visibility_level,
-                                     :public_builds]
+                                     :public_builds,
+                                     :repository_storage]
         attrs = map_public_to_visibility_level(attrs)
         authorize_admin_project
         authorize! :rename_project, user_project if attrs[:name].present?

lib/gitlab/backend/shell.rb

@@ -106,6 +106,20 @@ module Gitlab
                                    storage, "#{path}.git", "#{new_path}.git"])
     end
 
+    # Move repository storage
+    #
+    # current_storage - project's current storage path
+    # path - project path with namespace
+    # new_storage - new storage path
+    #
+    # Ex.
+    #   mv_storage("/path/to/storage", "randx/gitlab-ci", "/new/storage/path")
+    #
+    def mv_storage(current_storage, path, new_storage)
+      Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-storage',
+                                   current_storage, "#{path}.git", new_storage])
+    end
+
     # Fork repository to new namespace
     # forked_from_storage - forked-from project's storage path
     # path - project path with namespace

lib/gitlab/git_access.rb

@@ -59,6 +59,10 @@ module Gitlab
     end
 
     def push_access_check(changes)
+      if project.repository_read_only?
+        return build_status_object(false, 'The repository is temporarily read-only. Please try again later.')
+      end
+
       if Gitlab::Geo.secondary?
         return build_status_object(false, "You can't push code on a secondary GitLab Geo node.")
       end

lib/tasks/gitlab/cleanup.rake

@@ -27,7 +27,7 @@ namespace :gitlab do
         all_dirs.each do |dir_path|
 
           if remove_flag
-            if FileUtils.rm_rf dir_path
+            if FileUtils.rm_rf(dir_path)
               puts "Removed...#{dir_path}".color(:red)
             else
               puts "Cannot remove #{dir_path}".color(:red)
@@ -43,6 +43,34 @@ namespace :gitlab do
       end
     end
 
+    desc "GitLab | Cleanup | Delete moved repositories"
+    task moved: :environment  do
+      warn_user_is_not_gitlab
+      remove_flag = ENV['REMOVE']
+
+      Gitlab.config.repositories.storages.each do |name, repo_root|
+        # Look for global repos (legacy, depth 1) and normal repos (depth 2)
+        IO.popen(%W(find #{repo_root.chomp('/')} -mindepth 1 -maxdepth 2 -name *+moved*.git)) do |find|
+          find.each_line do |path|
+            path.chomp!
+            if remove_flag
+              if FileUtils.rm_rf(path)
+                puts "Removed...#{path}".color(:green)
+              else
+                puts "Cannot remove #{path}".color(:red)
+              end
+            else
+              puts "Can be removed: #{path}".color(:green)
+            end
+          end
+        end
+      end
+
+      unless remove_flag
+        puts "To cleanup these repositories run this command with REMOVE=true".color(:yellow)
+      end
+    end
+
     desc "GitLab | Cleanup | Clean repositories"
     task repos: :environment  do
       warn_user_is_not_gitlab

spec/factories/projects.rb

@@ -44,6 +44,10 @@ FactoryGirl.define do
         project.create_repository
       end
     end
+
+    trait :read_only_repository do
+      repository_read_only true
+    end
   end
 
   # Project with empty repository

spec/lib/gitlab/backend/shell_spec.rb

@@ -44,6 +44,30 @@ describe Gitlab::Shell, lib: true do
     end
   end
 
+  describe 'projects commands' do
+    let(:projects_path) { 'tmp/tests/shell-projects-test/bin/gitlab-projects' }
+
+    before do
+      allow(Gitlab.config.gitlab_shell).to receive(:path).and_return('tmp/tests/shell-projects-test')
+    end
+
+    describe 'mv_repository' do
+      it 'executes the command' do
+        expect(Gitlab::Utils).to receive(:system_silent).
+          with([projects_path, 'mv-project', 'storage/path', 'project/path.git', 'new/path.git'])
+        gitlab_shell.mv_repository('storage/path', 'project/path', 'new/path')
+      end
+    end
+
+    describe 'mv_storage' do
+      it 'executes the command' do
+        expect(Gitlab::Utils).to receive(:system_silent).
+          with([projects_path, 'mv-storage', 'current/storage', 'project/path.git', 'new/storage'])
+        gitlab_shell.mv_storage('current/storage', 'project/path', 'new/storage')
+      end
+    end
+  end
+
   describe Gitlab::Shell::KeyAdder, lib: true do
     describe '#add_key' do
       it 'normalizes space characters in the key' do

spec/lib/gitlab/git_access_spec.rb

@@ -555,4 +555,15 @@ describe Gitlab::GitAccess, lib: true do
       end
     end
   end
+
+  context 'when the repository is read only' do
+    it 'denies push access' do
+      project = create(:project, :read_only_repository)
+      project.team << [user, :master]
+
+      check = access.check('git-receive-pack')
+
+      expect(check).not_to be_allowed
+    end
+  end
 end

spec/models/project_spec.rb

@@ -1580,4 +1580,51 @@ describe Project, models: true do
       expect(shared_project.authorized_for_user?(master, Gitlab::Access::MASTER)).to be(true)
     end
   end
+
+  describe '#change_repository_storage' do
+    let(:project) { create(:project, repository_storage: 'a') }
+    let(:read_only_project) { create(:project, repository_storage: 'a', repository_read_only: true) }
+
+    before do
+      FileUtils.mkdir('tmp/tests/storage_a')
+      FileUtils.mkdir('tmp/tests/storage_b')
+
+      storages = { 'a' => 'tmp/tests/storage_a', 'b' => 'tmp/tests/storage_b' }
+      allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+    end
+
+    after do
+      FileUtils.rm_rf('tmp/tests/storage_a')
+      FileUtils.rm_rf('tmp/tests/storage_b')
+    end
+
+    it 'schedule the transfer of the repository to the new storage and locks the project' do
+      expect(ProjectUpdateRepositoryStorageWorker).to receive(:perform_async).with(project.id, 'b')
+
+      project.change_repository_storage('b')
+      project.save
+
+      expect(project).to be_repository_read_only
+    end
+
+    it "doesn't schedule the transfer if the repository is already read-only" do
+      expect(ProjectUpdateRepositoryStorageWorker).not_to receive(:perform_async)
+
+      read_only_project.change_repository_storage('b')
+      read_only_project.save
+    end
+
+    it "doesn't lock or schedule the transfer if the storage hasn't changed" do
+      expect(ProjectUpdateRepositoryStorageWorker).not_to receive(:perform_async)
+
+      project.change_repository_storage('a')
+      project.save
+
+      expect(project).not_to be_repository_read_only
+    end
+
+    it 'throws an error if an invalid repository storage is provided' do
+      expect { project.change_repository_storage('c') }.to raise_error
+    end
+  end
 end

spec/services/projects/update_repository_storage_service_spec.rb

+require 'spec_helper'
+
+describe Projects::UpdateRepositoryStorageService, services: true do
+
+  subject { described_class.new(project) }
+
+  describe "#execute" do
+    let(:gitlab_shell) { Gitlab::Shell.new }
+    let(:time) { Time.now }
+
+    before do
+      FileUtils.mkdir('tmp/tests/storage_a')
+      FileUtils.mkdir('tmp/tests/storage_b')
+
+      storages = { 'a' => 'tmp/tests/storage_a', 'b' => 'tmp/tests/storage_b' }
+      allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+      allow(subject).to receive(:gitlab_shell).and_return(gitlab_shell)
+
+      allow(Time).to receive(:now).and_return(time)
+    end
+
+    after do
+      FileUtils.rm_rf('tmp/tests/storage_a')
+      FileUtils.rm_rf('tmp/tests/storage_b')
+    end
+
+    context 'without wiki' do
+      let(:project) { create(:project, repository_storage: 'a', repository_read_only: true, wiki_enabled: false) }
+
+      context 'when the move succeeds' do
+        it 'moves the repository to the new storage and unmarks the repository as read only' do
+          expect(gitlab_shell).to receive(:mv_storage).
+            with('tmp/tests/storage_a', project.path_with_namespace, 'tmp/tests/storage_b').
+            and_return(true)
+          expect(GitlabShellWorker).to receive(:perform_async).
+            with(:mv_repository,
+              'tmp/tests/storage_a',
+              project.path_with_namespace,
+              "#{project.path_with_namespace}+#{project.id}+moved+#{time.to_i}")
+
+          subject.execute('b')
+
+          expect(project).not_to be_repository_read_only
+          expect(project.repository_storage).to eq('b')
+        end
+      end
+
+      context 'when the move fails' do
+        it 'unmarks the repository as read-only without updating the repository storage' do
+          expect(gitlab_shell).to receive(:mv_storage).
+            with('tmp/tests/storage_a', project.path_with_namespace, 'tmp/tests/storage_b').
+            and_return(false)
+          expect(GitlabShellWorker).not_to receive(:perform_async)
+
+          subject.execute('b')
+
+          expect(project).not_to be_repository_read_only
+          expect(project.repository_storage).to eq('a')
+        end
+      end
+    end
+
+    context 'with wiki' do
+      let(:project) { create(:project, repository_storage: 'a', repository_read_only: true, wiki_enabled: true) }
+
+      before { project.create_wiki }
+
+      context 'when the move succeeds' do
+        it 'moves the repository and its wiki to the new storage and unmarks the repository as read only' do
+          expect(gitlab_shell).to receive(:mv_storage).
+            with('tmp/tests/storage_a', project.path_with_namespace, 'tmp/tests/storage_b').
+            and_return(true)
+          expect(GitlabShellWorker).to receive(:perform_async).
+            with(:mv_repository,
+              'tmp/tests/storage_a',
+              project.path_with_namespace,
+              "#{project.path_with_namespace}+#{project.id}+moved+#{time.to_i}")
+
+          expect(gitlab_shell).to receive(:mv_storage).
+            with('tmp/tests/storage_a', "#{project.path_with_namespace}.wiki", 'tmp/tests/storage_b').
+            and_return(true)
+          expect(GitlabShellWorker).to receive(:perform_async).
+            with(:mv_repository,
+              'tmp/tests/storage_a',
+              "#{project.path_with_namespace}.wiki",
+              "#{project.path_with_namespace}+#{project.id}+moved+#{time.to_i}.wiki")
+
+          subject.execute('b')
+
+          expect(project).not_to be_repository_read_only
+          expect(project.repository_storage).to eq('b')
+        end
+      end
+
+      context 'when the move of the wiki fails' do
+        it 'unmarks the repository as read-only without updating the repository storage' do
+          expect(gitlab_shell).to receive(:mv_storage).
+            with('tmp/tests/storage_a', project.path_with_namespace, 'tmp/tests/storage_b').
+            and_return(true)
+          expect(gitlab_shell).to receive(:mv_storage).
+            with('tmp/tests/storage_a', "#{project.path_with_namespace}.wiki", 'tmp/tests/storage_b').
+            and_return(false)
+          expect(GitlabShellWorker).not_to receive(:perform_async)
+
+          subject.execute('b')
+
+          expect(project).not_to be_repository_read_only
+          expect(project.repository_storage).to eq('a')
+        end
+      end
+    end
+  end
+end

spec/services/projects/update_service_spec.rb

@@ -139,6 +139,38 @@ describe Projects::UpdateService, services: true do
     end
   end
 
+  describe 'repository_storage' do
+    let(:admin_user) { create(:user, admin: true) }
+    let(:user) { create(:user) }
+    let(:project) { create(:project, repository_storage: 'a') }
+    let(:opts) { { repository_storage: 'b' } }
+
+    before do
+      FileUtils.mkdir('tmp/tests/storage_a')
+      FileUtils.mkdir('tmp/tests/storage_b')
+
+      storages = { 'a' => 'tmp/tests/storage_a', 'b' => 'tmp/tests/storage_b' }
+      allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+    end
+
+    after do
+      FileUtils.rm_rf('tmp/tests/storage_a')
+      FileUtils.rm_rf('tmp/tests/storage_b')
+    end
+
+    it 'calls the change repository storage method if the storage changed' do
+      expect(project).to receive(:change_repository_storage).with('b')
+
+      update_project(project, admin_user, opts).inspect
+    end
+
+    it "doesn't call the change repository storage for non-admin users" do
+      expect(project).not_to receive(:change_repository_storage)
+
+      update_project(project, user, opts).inspect
+    end
+  end
+
   def update_project(project, user, opts)
     Projects::UpdateService.new(project, user, opts).execute
   end

spec/workers/project_update_repository_storage_worker_spec.rb

+require 'spec_helper'
+
+describe ProjectUpdateRepositoryStorageWorker do
+  let(:project) { create(:project) }
+
+  subject { ProjectUpdateRepositoryStorageWorker.new }
+
+  describe "#perform" do
+    it "should call the update repository storage service" do
+      expect_any_instance_of(Projects::UpdateRepositoryStorageService).
+        to receive(:execute).with('new_storage')
+
+      subject.perform(project.id, 'new_storage')
+    end
+  end
+end