Merge branch 'ee-gitaly-create-repository' into 'master'

(EE) Create repositories via Gitaly

See merge request gitlab-org/gitlab-ee!3038

GITALY_SERVER_VERSION

-0.40.0
+0.42.0

Gemfile

@@ -414,7 +414,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.33.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.38.0', require: 'gitaly'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -299,7 +299,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly-proto (0.33.0)
+    gitaly-proto (0.38.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -1060,7 +1060,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly-proto (~> 0.33.0)
+  gitaly-proto (~> 0.38.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)

app/models/project.rb

@@ -1035,7 +1035,7 @@ class Project < ActiveRecord::Base
     # Forked import is handled asynchronously
     return if forked? && !force
 
-    if gitlab_shell.add_repository(repository_storage_path, disk_path)
+    if gitlab_shell.add_repository(repository_storage, disk_path)
       repository.after_create
       true
     else

app/models/project_wiki.rb

@@ -189,7 +189,7 @@ class ProjectWiki
   private
 
   def init_repo(disk_path)
-    gitlab_shell.add_repository(project.repository_storage_path, disk_path)
+    gitlab_shell.add_repository(project.repository_storage, disk_path)
   end
 
   def commit_details(action, message = nil, title = nil)

lib/gitlab/git/repository.rb

@@ -22,11 +22,8 @@ module Gitlab
       CreateTreeError = Class.new(StandardError)
 
       class << self
-        # Unlike `new`, `create` takes the storage path, not the storage name
-        def create(storage_path, name, bare: true, symlink_hooks_to: nil)
-          repo_path = File.join(storage_path, name)
-          repo_path += '.git' unless repo_path.end_with?('.git')
-
+        # Unlike `new`, `create` takes the repository path
+        def create(repo_path, bare: true, symlink_hooks_to: nil)
           FileUtils.mkdir_p(repo_path, mode: 0770)
 
           # Equivalent to `git --git-path=#{repo_path} init [--bare]`

lib/gitlab/gitaly_client/repository_service.rb

@@ -53,6 +53,11 @@ module Gitlab
 
         GitalyClient.call(@storage, :repository_service, :fetch_remote, request)
       end
+
+      def create_repository
+        request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo)
+        GitalyClient.call(@storage, :repository_service, :create_repository, request)
+      end
     end
   end
 end

lib/gitlab/shell.rb

@@ -65,7 +65,7 @@ module Gitlab
 
     # Init new repository
     #
-    # storage - project's storage path
+    # storage - project's storage name
     # name - project path with namespace
     #
     # Ex.
@@ -73,7 +73,19 @@ module Gitlab
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def add_repository(storage, name)
-      Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path)
+      relative_path = name.dup
+      relative_path << '.git' unless relative_path.end_with?('.git')
+
+      gitaly_migrate(:create_repository) do |is_enabled|
+        if is_enabled
+          repository = Gitlab::Git::Repository.new(storage, relative_path, '')
+          repository.gitaly_repository_client.create_repository
+          true
+        else
+          repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path)
+          Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path)
+        end
+      end
     rescue => err
       Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}")
       false

lib/tasks/gitlab/shell.rake

@@ -79,7 +79,7 @@ namespace :gitlab do
         if File.exist?(path_to_repo)
           print '-'
         else
-          if Gitlab::Shell.new.add_repository(project.repository_storage_path,
+          if Gitlab::Shell.new.add_repository(project.repository_storage,
                                               project.disk_path)
             print '.'
           else

spec/lib/gitlab/shell_spec.rb

@@ -444,30 +444,42 @@ describe Gitlab::Shell do
     end
 
     describe '#add_repository' do
-      it 'creates a repository' do
-        created_path = File.join(TestEnv.repos_path, 'project', 'path.git')
-        hooks_path = File.join(created_path, 'hooks')
+      shared_examples '#add_repository' do
+        let(:repository_storage) { 'default' }
+        let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
+        let(:repo_name) { 'project/path' }
+        let(:created_path) { File.join(repository_storage_path, repo_name + '.git') }
 
-        begin
-          result = gitlab_shell.add_repository(TestEnv.repos_path, 'project/path')
-
-          repo_stat = File.stat(created_path) rescue nil
-          hooks_stat = File.lstat(hooks_path) rescue nil
-          hooks_dir = File.realpath(hooks_path)
-        ensure
+        after do
           FileUtils.rm_rf(created_path)
         end
 
-        expect(result).to be_truthy
-        expect(repo_stat.mode & 0o777).to eq(0o770)
-        expect(hooks_stat.symlink?).to be_truthy
-        expect(hooks_dir).to eq(gitlab_shell_hooks_path)
+        it 'creates a repository' do
+          expect(gitlab_shell.add_repository(repository_storage, repo_name)).to be_truthy
+
+          expect(File.stat(created_path).mode & 0o777).to eq(0o770)
+
+          hooks_path = File.join(created_path, 'hooks')
+          expect(File.lstat(hooks_path)).to be_symlink
+          expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path)
+        end
+
+        it 'returns false when the command fails' do
+          FileUtils.mkdir_p(File.dirname(created_path))
+          # This file will block the creation of the repo's .git directory. That
+          # should cause #add_repository to fail.
+          FileUtils.touch(created_path)
+
+          expect(gitlab_shell.add_repository(repository_storage, repo_name)).to be_falsy
+        end
       end
 
-      it 'returns false when the command fails' do
-        expect(FileUtils).to receive(:mkdir_p).and_raise(Errno::EEXIST)
+      context 'with gitlay' do
+        it_behaves_like '#add_repository'
+      end
 
-        expect(gitlab_shell.add_repository('current/storage', 'project/path')).to be_falsy
+      context 'without gitaly', skip_gitaly_mock: true do
+        it_behaves_like '#add_repository'
       end
     end
 

spec/lib/gitlab/workhorse_spec.rb

@@ -214,14 +214,12 @@ describe Gitlab::Workhorse do
       end
 
       it 'includes a Repository param' do
-        repo_param = { Repository: {
+        repo_param = {
           storage_name: 'default',
-          relative_path: project.full_path + '.git',
-          git_object_directory: '',
-          git_alternate_object_directories: []
-        } }
+          relative_path: project.full_path + '.git'
+        }
 
-        expect(subject).to include(repo_param)
+        expect(subject[:Repository]).to include(repo_param)
       end
 
       context "when git_upload_pack action is passed" do

spec/models/project_spec.rb

@@ -1484,7 +1484,7 @@ describe Project do
     context 'using a regular repository' do
       it 'creates the repository' do
         expect(shell).to receive(:add_repository)
-          .with(project.repository_storage_path, project.disk_path)
+          .with(project.repository_storage, project.disk_path)
           .and_return(true)
 
         expect(project.repository).to receive(:after_create)
@@ -1494,7 +1494,7 @@ describe Project do
 
       it 'adds an error if the repository could not be created' do
         expect(shell).to receive(:add_repository)
-          .with(project.repository_storage_path, project.disk_path)
+          .with(project.repository_storage, project.disk_path)
           .and_return(false)
 
         expect(project.repository).not_to receive(:after_create)
@@ -1551,7 +1551,7 @@ describe Project do
         .and_return(false)
 
       expect(shell).to receive(:add_repository)
-        .with(project.repository_storage_path, project.disk_path)
+        .with(project.repository_storage, project.disk_path)
         .and_return(true)
 
       project.ensure_repository

spec/services/projects/create_service_spec.rb

@@ -156,10 +156,11 @@ describe Projects::CreateService, '#execute' do
         }
       end
 
-      let(:repository_storage_path) { Gitlab.config.repositories.storages['default']['path'] }
+      let(:repository_storage) { 'default' }
+      let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
       before do
-        gitlab_shell.add_repository(repository_storage_path, "#{user.namespace.full_path}/existing")
+        gitlab_shell.add_repository(repository_storage, "#{user.namespace.full_path}/existing")
       end
 
       after do

spec/services/projects/fork_service_spec.rb

@@ -76,10 +76,11 @@ describe Projects::ForkService do
     end
 
     context 'repository already exists' do
-      let(:repository_storage_path) { Gitlab.config.repositories.storages['default']['path'] }
+      let(:repository_storage) { 'default' }
+      let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
       before do
-        gitlab_shell.add_repository(repository_storage_path, "#{@to_user.namespace.full_path}/#{@from_project.path}")
+        gitlab_shell.add_repository(repository_storage, "#{@to_user.namespace.full_path}/#{@from_project.path}")
       end
 
       after do

spec/services/projects/transfer_service_spec.rb

@@ -121,11 +121,14 @@ describe Projects::TransferService do
   end
 
   context 'namespace which contains orphan repository with same projects path name' do
-    let(:repository_storage_path) { Gitlab.config.repositories.storages['default']['path'] }
+    let(:repository_storage) { 'default' }
+    let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
     before do
       group.add_owner(user)
-      gitlab_shell.add_repository(repository_storage_path, "#{group.full_path}/#{project.path}")
+      unless gitlab_shell.add_repository(repository_storage, "#{group.full_path}/#{project.path}")
+        raise 'failed to add repository'
+      end
 
       @result = transfer_project(project, user, group)
     end

spec/services/projects/update_service_spec.rb

@@ -150,10 +150,11 @@ describe Projects::UpdateService, '#execute' do
   end
 
   context 'when renaming a project' do
-    let(:repository_storage_path) { Gitlab.config.repositories.storages['default']['path'] }
+    let(:repository_storage) { 'default' }
+    let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
     before do
-      gitlab_shell.add_repository(repository_storage_path, "#{user.namespace.full_path}/existing")
+      gitlab_shell.add_repository(repository_storage, "#{user.namespace.full_path}/existing")
     end
 
     after do