Merge branch 'gitaly-refs' into 'master'

Implement Gitaly refs client

See merge request !9291

app/models/repository.rb

@@ -59,7 +59,7 @@ class Repository
   def raw_repository
     return nil unless path_with_namespace
 
-    @raw_repository ||= Gitlab::Git::Repository.new(path_to_repo)
+    @raw_repository ||= initialize_raw_repository
   end
 
   # Return absolute path to repository
@@ -146,12 +146,7 @@ class Repository
     # may cause the branch to "disappear" erroneously or have the wrong SHA.
     #
     # See: https://github.com/libgit2/libgit2/issues/1534 and https://gitlab.com/gitlab-org/gitlab-ce/issues/15392
-    raw_repo =
-      if fresh_repo
-        Gitlab::Git::Repository.new(path_to_repo)
-      else
-        raw_repository
-      end
+    raw_repo = fresh_repo ? initialize_raw_repository : raw_repository
 
     raw_repo.find_branch(name)
   end
@@ -505,9 +500,7 @@ class Repository
     end
   end
 
-  def branch_names
-    branches.map(&:name)
-  end
+  delegate :branch_names, to: :raw_repository
   cache_method :branch_names, fallback: []
 
   delegate :tag_names, to: :raw_repository
@@ -1168,4 +1161,8 @@ class Repository
   def repository_storage_path
     @project.repository_storage_path
   end
+
+  def initialize_raw_repository
+    Gitlab::Git::Repository.new(project.repository_storage, path_with_namespace + '.git')
+  end
 end

changelogs/unreleased/gitaly-refs.yml

+---
+title: Incorporate Gitaly client for refs service
+merge_request: 9291
+author:

db/migrate/20140502125220_migrate_repo_size.rb

@@ -8,11 +8,10 @@ class MigrateRepoSize < ActiveRecord::Migration
     project_data.each do |project|
       id = project['id']
       namespace_path = project['namespace_path'] || ''
-      repos_path = Gitlab.config.gitlab_shell['repos_path'] || Gitlab.config.repositories.storages.default['path']
-      path = File.join(repos_path, namespace_path, project['project_path'] + '.git')
+      path = File.join(namespace_path, project['project_path'] + '.git')
 
       begin
-        repo = Gitlab::Git::Repository.new(path)
+        repo = Gitlab::Git::Repository.new('default', path)
         if repo.empty?
           print '-'
         else

lib/api/internal.rb

@@ -138,8 +138,11 @@ module API
 
         return unless Gitlab::GitalyClient.enabled?
 
+        relative_path = Gitlab::RepoPath.strip_storage_path(params[:repo_path])
+        project = Project.find_by_full_path(relative_path.sub(/\.(git|wiki)\z/, ''))
+
         begin
-          Gitlab::GitalyClient::Notifications.new(params[:repo_path]).post_receive
+          Gitlab::GitalyClient::Notifications.new(project.repository_storage, relative_path).post_receive
         rescue GRPC::Unavailable => e
           render_api_error(e, 500)
         end

lib/gitlab/git.rb

@@ -4,6 +4,8 @@ module Gitlab
     TAG_REF_PREFIX = "refs/tags/".freeze
     BRANCH_REF_PREFIX = "refs/heads/".freeze
 
+    CommandError = Class.new(StandardError)
+
     class << self
       def ref_name(ref)
         ref.sub(/\Arefs\/(tags|heads)\//, '')

lib/gitlab/git/repository.rb

@@ -25,9 +25,13 @@ module Gitlab
 
       # 'path' must be the path to a _bare_ git repository, e.g.
       # /path/to/my-repo.git
-      def initialize(path)
-        @path = path
-        @name = path.split("/").last
+      def initialize(repository_storage, relative_path)
+        @repository_storage = repository_storage
+        @relative_path = relative_path
+
+        storage_path = Gitlab.config.repositories.storages[@repository_storage]['path']
+        @path = File.join(storage_path, @relative_path)
+        @name = @relative_path.split("/").last
         @attributes = Gitlab::Git::Attributes.new(path)
       end
 
@@ -37,7 +41,15 @@ module Gitlab
 
       # Default branch in the repository
       def root_ref
-        @root_ref ||= discover_default_branch
+        @root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.default_branch_name
+          else
+            discover_default_branch
+          end
+        end
+      rescue GRPC::BadStatus => e
+        raise CommandError.new(e)
       end
 
       # Alias to old method for compatibility
@@ -54,7 +66,15 @@ module Gitlab
       # Returns an Array of branch names
       # sorted by name ASC
       def branch_names
-        branches.map(&:name)
+        Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.branch_names
+          else
+            branches.map(&:name)
+          end
+        end
+      rescue GRPC::BadStatus => e
+        raise CommandError.new(e)
       end
 
       # Returns an Array of Branches
@@ -107,7 +127,15 @@ module Gitlab
 
       # Returns an Array of tag names
       def tag_names
-        rugged.tags.map { |t| t.name }
+        Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.tag_names
+          else
+            rugged.tags.map { |t| t.name }
+          end
+        end
+      rescue GRPC::BadStatus => e
+        raise CommandError.new(e)
       end
 
       # Returns an Array of Tags
@@ -1202,6 +1230,10 @@ module Gitlab
         diff.find_similar!(break_rewrites: break_rewrites)
         diff.each_patch
       end
+
+      def gitaly_ref_client
+        @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(@repository_storage, @relative_path)
+      end
     end
   end
 end

lib/gitlab/gitaly_client/notifications.rb

@@ -3,18 +3,13 @@ module Gitlab
     class Notifications
       attr_accessor :stub
 
-      def initialize(repo_path)
-        full_path = Gitlab::RepoPath.strip_storage_path(repo_path).
-          sub(/\.git\z/, '').sub(/\.wiki\z/, '')
-        @project = Project.find_by_full_path(full_path)
-
-        channel = GitalyClient.get_channel(@project.repository_storage)
-        @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: channel)
+      def initialize(repository_storage, relative_path)
+        @channel, @repository = Util.process_path(repository_storage, relative_path)
+        @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: @channel)
       end
 
       def post_receive
-        repository = Gitaly::Repository.new(path: @project.repository.path_to_repo)
-        request = Gitaly::PostReceiveRequest.new(repository: repository)
+        request = Gitaly::PostReceiveRequest.new(repository: @repository)
         @stub.post_receive(request)
       end
     end

lib/gitlab/gitaly_client/ref.rb

+module Gitlab
+  module GitalyClient
+    class Ref
+      attr_accessor :stub
+
+      def initialize(repository_storage, relative_path)
+        @channel, @repository = Util.process_path(repository_storage, relative_path)
+        @stub = Gitaly::Ref::Stub.new(nil, nil, channel_override: @channel)
+      end
+
+      def default_branch_name
+        request = Gitaly::FindDefaultBranchNameRequest.new(repository: @repository)
+        stub.find_default_branch_name(request).name.gsub(/^refs\/heads\//, '')
+      end
+
+      def branch_names
+        request = Gitaly::FindAllBranchNamesRequest.new(repository: @repository)
+        consume_refs_response(stub.find_all_branch_names(request), prefix: 'refs/heads/')
+      end
+
+      def tag_names
+        request = Gitaly::FindAllTagNamesRequest.new(repository: @repository)
+        consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/')
+      end
+
+      private
+
+      def consume_refs_response(response, prefix:)
+        response.flat_map do |r|
+          r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') }
+        end
+      end
+    end
+  end
+end

lib/gitlab/gitaly_client/util.rb

+module Gitlab
+  module GitalyClient
+    module Util
+      def self.process_path(repository_storage, relative_path)
+        channel = GitalyClient.get_channel(repository_storage)
+        storage_path = Gitlab.config.repositories.storages[repository_storage]['path']
+        repository = Gitaly::Repository.new(path: File.join(storage_path, relative_path))
+
+        [channel, repository]
+      end
+    end
+  end
+end

spec/lib/gitlab/git/attributes_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe Gitlab::Git::Attributes, seed_helper: true do
   let(:path) do
-    File.join(SEED_REPOSITORY_PATH, 'with-git-attributes.git')
+    File.join(SEED_STORAGE_PATH, 'with-git-attributes.git')
   end
 
   subject { described_class.new(path) }
@@ -141,7 +141,7 @@ describe Gitlab::Git::Attributes, seed_helper: true do
     end
 
     it 'does not yield when the attributes file has an unsupported encoding' do
-      path = File.join(SEED_REPOSITORY_PATH, 'with-invalid-git-attributes.git')
+      path = File.join(SEED_STORAGE_PATH, 'with-invalid-git-attributes.git')
       attrs = described_class.new(path)
 
       expect { |b| attrs.each_line(&b) }.not_to yield_control

spec/lib/gitlab/git/blame_spec.rb

@@ -2,7 +2,7 @@
 require "spec_helper"
 
 describe Gitlab::Git::Blame, seed_helper: true do
-  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
   let(:blame) do
     Gitlab::Git::Blame.new(repository, SeedRepo::Commit::ID, "CONTRIBUTING.md")
   end

spec/lib/gitlab/git/blob_spec.rb

@@ -3,7 +3,7 @@
 require "spec_helper"
 
 describe Gitlab::Git::Blob, seed_helper: true do
-  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
 
   describe 'initialize' do
     let(:blob) { Gitlab::Git::Blob.new(name: 'test') }

spec/lib/gitlab/git/branch_spec.rb

 require "spec_helper"
 
 describe Gitlab::Git::Branch, seed_helper: true do
-  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
 
   subject { repository.branches }
 

spec/lib/gitlab/git/commit_spec.rb

 require "spec_helper"
 
 describe Gitlab::Git::Commit, seed_helper: true do
-  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
   let(:commit) { Gitlab::Git::Commit.find(repository, SeedRepo::Commit::ID) }
   let(:rugged_commit) do
     repository.rugged.lookup(SeedRepo::Commit::ID)
@@ -9,7 +9,7 @@ describe Gitlab::Git::Commit, seed_helper: true do
 
   describe "Commit info" do
     before do
-      repo = Gitlab::Git::Repository.new(TEST_REPO_PATH).rugged
+      repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged
 
       @committer = {
         email: 'mike@smith.com',
@@ -59,7 +59,7 @@ describe Gitlab::Git::Commit, seed_helper: true do
 
     after do
       # Erase the new commit so other tests get the original repo
-      repo = Gitlab::Git::Repository.new(TEST_REPO_PATH).rugged
+      repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged
       repo.references.update("refs/heads/master", SeedRepo::LastCommit::ID)
     end
   end
@@ -95,7 +95,7 @@ describe Gitlab::Git::Commit, seed_helper: true do
       end
 
       context 'with broken repo' do
-        let(:repository) { Gitlab::Git::Repository.new(TEST_BROKEN_REPO_PATH) }
+        let(:repository) { Gitlab::Git::Repository.new('default', TEST_BROKEN_REPO_PATH) }
 
         it 'returns nil' do
           expect(Gitlab::Git::Commit.find(repository, SeedRepo::Commit::ID)).to be_nil

spec/lib/gitlab/git/compare_spec.rb

 require "spec_helper"
 
 describe Gitlab::Git::Compare, seed_helper: true do
-  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
   let(:compare) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, SeedRepo::Commit::ID, false) }
   let(:compare_straight) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, SeedRepo::Commit::ID, true) }
 

spec/lib/gitlab/git/diff_spec.rb

 require "spec_helper"
 
 describe Gitlab::Git::Diff, seed_helper: true do
-  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
 
   before do
     @raw_diff_hash = {

spec/lib/gitlab/git/encoding_helper_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 
 describe Gitlab::Git::EncodingHelper do
   let(:ext_class) { Class.new { extend Gitlab::Git::EncodingHelper } }
-  let(:binary_string) { File.join(SEED_REPOSITORY_PATH, 'gitlab_logo.png') }
+  let(:binary_string) { File.join(SEED_STORAGE_PATH, 'gitlab_logo.png') }
 
   describe '#encode!' do
     [

spec/lib/gitlab/git/index_spec.rb

 require 'spec_helper'
 
 describe Gitlab::Git::Index, seed_helper: true do
-  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
   let(:index) { described_class.new(repository) }
 
   before do

spec/lib/gitlab/git/tag_spec.rb

 require "spec_helper"
 
 describe Gitlab::Git::Tag, seed_helper: true do
-  let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+  let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
 
   describe 'first tag' do
     let(:tag) { repository.tags.first }

spec/lib/gitlab/git/tree_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 
 describe Gitlab::Git::Tree, seed_helper: true do
   context :repo do
-    let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
+    let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
     let(:tree) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID) }
 
     it { expect(tree).to be_kind_of Array }

spec/lib/gitlab/gitaly_client/notifications_spec.rb

@@ -2,12 +2,15 @@ require 'spec_helper'
 
 describe Gitlab::GitalyClient::Notifications do
   describe '#post_receive' do
+    let(:project) { create(:empty_project) }
+    let(:repo_path) { project.repository.path_to_repo }
+    subject { described_class.new(project.repository_storage, project.full_path + '.git') }
+
     it 'sends a post_receive message' do
-      repo_path = create(:empty_project).repository.path_to_repo
       expect_any_instance_of(Gitaly::Notifications::Stub).
-        to receive(:post_receive).with(post_receive_request_with_repo_path(repo_path))
+        to receive(:post_receive).with(gitaly_request_with_repo_path(repo_path))
 
-      described_class.new(repo_path).post_receive
+      subject.post_receive
     end
   end
 end

spec/lib/gitlab/gitaly_client/ref_spec.rb

+require 'spec_helper'
+
+describe Gitlab::GitalyClient::Ref do
+  let(:project) { create(:empty_project) }
+  let(:repo_path) { project.repository.path_to_repo }
+  let(:client) { Gitlab::GitalyClient::Ref.new(project.repository_storage, project.full_path + '.git') }
+
+  before do
+    allow(Gitlab.config.gitaly).to receive(:enabled).and_return(true)
+  end
+
+  describe '#branch_names' do
+    it 'sends a find_all_branch_names message' do
+      expect_any_instance_of(Gitaly::Ref::Stub).
+        to receive(:find_all_branch_names).with(gitaly_request_with_repo_path(repo_path)).
+          and_return([])
+
+      client.branch_names
+    end
+  end
+
+  describe '#tag_names' do
+    it 'sends a find_all_tag_names message' do
+      expect_any_instance_of(Gitaly::Ref::Stub).
+        to receive(:find_all_tag_names).with(gitaly_request_with_repo_path(repo_path)).
+          and_return([])
+
+      client.tag_names
+    end
+  end
+
+  describe '#default_branch_name' do
+    it 'sends a find_default_branch_name message' do
+      expect_any_instance_of(Gitaly::Ref::Stub).
+        to receive(:find_default_branch_name).with(gitaly_request_with_repo_path(repo_path)).
+        and_return(double(name: 'foo'))
+
+      client.default_branch_name
+    end
+  end
+end

spec/support/matchers/gitaly_matchers.rb

-RSpec::Matchers.define :post_receive_request_with_repo_path do |path|
+RSpec::Matchers.define :gitaly_request_with_repo_path do |path|
   match { |actual| actual.repository.path == path }
 end

spec/support/seed_helper.rb

+require_relative 'test_env'
+
 # This file is specific to specs in spec/lib/gitlab/git/
 
-SEED_REPOSITORY_PATH   = File.expand_path('../../tmp/repositories', __dir__)
-TEST_REPO_PATH         = File.join(SEED_REPOSITORY_PATH, 'gitlab-git-test.git')
-TEST_NORMAL_REPO_PATH  = File.join(SEED_REPOSITORY_PATH, "not-bare-repo.git")
-TEST_MUTABLE_REPO_PATH = File.join(SEED_REPOSITORY_PATH, "mutable-repo.git")
-TEST_BROKEN_REPO_PATH  = File.join(SEED_REPOSITORY_PATH, "broken-repo.git")
+SEED_STORAGE_PATH      = TestEnv.repos_path
+TEST_REPO_PATH         = 'gitlab-git-test.git'.freeze
+TEST_NORMAL_REPO_PATH  = 'not-bare-repo.git'.freeze
+TEST_MUTABLE_REPO_PATH = 'mutable-repo.git'.freeze
+TEST_BROKEN_REPO_PATH  = 'broken-repo.git'.freeze
 
 module SeedHelper
   GITLAB_GIT_TEST_REPO_URL = ENV.fetch('GITLAB_GIT_TEST_REPO_URL', 'https://gitlab.com/gitlab-org/gitlab-git-test.git').freeze
 
   def ensure_seeds
-    if File.exist?(SEED_REPOSITORY_PATH)
-      FileUtils.rm_r(SEED_REPOSITORY_PATH)
+    if File.exist?(SEED_STORAGE_PATH)
+      FileUtils.rm_r(SEED_STORAGE_PATH)
     end
 
-    FileUtils.mkdir_p(SEED_REPOSITORY_PATH)
+    FileUtils.mkdir_p(SEED_STORAGE_PATH)
 
     create_bare_seeds
     create_normal_seeds
@@ -26,41 +28,45 @@ module SeedHelper
 
   def create_bare_seeds
     system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --bare #{GITLAB_GIT_TEST_REPO_URL}),
-           chdir: SEED_REPOSITORY_PATH,
+           chdir: SEED_STORAGE_PATH,
            out:   '/dev/null',
            err:   '/dev/null')
   end
 
   def create_normal_seeds
     system(git_env, *%W(#{Gitlab.config.git.bin_path} clone #{TEST_REPO_PATH} #{TEST_NORMAL_REPO_PATH}),
+           chdir: SEED_STORAGE_PATH,
            out: '/dev/null',
            err: '/dev/null')
   end
 
   def create_mutable_seeds
     system(git_env, *%W(#{Gitlab.config.git.bin_path} clone #{TEST_REPO_PATH} #{TEST_MUTABLE_REPO_PATH}),
+           chdir: SEED_STORAGE_PATH,
            out: '/dev/null',
            err: '/dev/null')
 
-    system(git_env, *%w(git branch -t feature origin/feature),
-           chdir: TEST_MUTABLE_REPO_PATH, out: '/dev/null', err: '/dev/null')
+    mutable_repo_full_path = File.join(SEED_STORAGE_PATH, TEST_MUTABLE_REPO_PATH)
+    system(git_env, *%W(#{Gitlab.config.git.bin_path} branch -t feature origin/feature),
+           chdir: mutable_repo_full_path, out: '/dev/null', err: '/dev/null')
 
     system(git_env, *%W(#{Gitlab.config.git.bin_path} remote add expendable #{GITLAB_GIT_TEST_REPO_URL}),
-           chdir: TEST_MUTABLE_REPO_PATH, out: '/dev/null', err: '/dev/null')
+           chdir: mutable_repo_full_path, out: '/dev/null', err: '/dev/null')
   end
 
   def create_broken_seeds
     system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --bare #{TEST_REPO_PATH} #{TEST_BROKEN_REPO_PATH}),
+           chdir: SEED_STORAGE_PATH,
            out: '/dev/null',
            err: '/dev/null')
 
-    refs_path = File.join(TEST_BROKEN_REPO_PATH, 'refs')
+    refs_path = File.join(SEED_STORAGE_PATH, TEST_BROKEN_REPO_PATH, 'refs')
 
     FileUtils.rm_r(refs_path)
   end
 
   def create_git_attributes
-    dir = File.join(SEED_REPOSITORY_PATH, 'with-git-attributes.git', 'info')
+    dir = File.join(SEED_STORAGE_PATH, 'with-git-attributes.git', 'info')
 
     FileUtils.mkdir_p(dir)
 
@@ -85,7 +91,7 @@ bla/bla.txt
   end
 
   def create_invalid_git_attributes
-    dir = File.join(SEED_REPOSITORY_PATH, 'with-invalid-git-attributes.git', 'info')
+    dir = File.join(SEED_STORAGE_PATH, 'with-invalid-git-attributes.git', 'info')
 
     FileUtils.mkdir_p(dir)