Merge branch 'jc-client-for-fetch-objects-into-pool' into 'master'

Add client methods for FetchIntoObjectPool RPC

See merge request gitlab-org/gitlab-ce!27767

Gemfile

@@ -417,7 +417,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 1.22.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 1.26.0', require: 'gitaly'
 
 gem 'grpc', '~> 1.19.0'
 

Gemfile.lock

@@ -283,7 +283,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (1.22.0)
+    gitaly-proto (1.26.0)
       grpc (~> 1.0)
     github-markup (1.7.0)
     gitlab-default_value_for (3.1.1)
@@ -1056,7 +1056,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 1.22.0)
+  gitaly-proto (~> 1.26.0)
   github-markup (~> 1.7.0)
   gitlab-default_value_for (~> 3.1.1)
   gitlab-labkit (~> 0.1.2)

lib/gitlab/git/object_pool.rb

@@ -40,6 +40,10 @@ module Gitlab
         @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path)
       end
 
+      def fetch
+        object_pool_service.fetch(source_repository)
+      end
+
       private
 
       def object_pool_service

lib/gitlab/gitaly_client/object_pool_service.rb

@@ -33,6 +33,15 @@ module Gitlab
         GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool,
                           request, timeout: GitalyClient.fast_timeout)
       end
+
+      def fetch(repository)
+        request = Gitaly::FetchIntoObjectPoolRequest.new(
+          object_pool: object_pool,
+          origin: repository.gitaly_repository
+        )
+
+        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
+      end
     end
   end
 end

spec/lib/gitlab/git/object_pool_spec.rb

@@ -3,8 +3,12 @@
 require 'spec_helper'
 
 describe Gitlab::Git::ObjectPool do
+  include RepoHelpers
+
   let(:pool_repository) { create(:pool_repository) }
   let(:source_repository) { pool_repository.source_project.repository }
+  let(:source_repository_path) { File.join(TestEnv.repos_path, source_repository.relative_path) }
+  let(:source_repository_rugged) { Rugged::Repository.new(source_repository_path) }
 
   subject { pool_repository.object_pool }
 
@@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do
       end
     end
   end
+
+  describe '#fetch' do
+    let(:commit_count) { source_repository.commit_count }
+
+    context "when the object's pool repository exists" do
+      it 'does not raise an error' do
+        expect { subject.fetch }.not_to raise_error
+      end
+    end
+
+    context "when the object's pool repository does not exist" do
+      before do
+        subject.delete
+      end
+
+      it "re-creates the object pool's repository" do
+        subject.fetch
+
+        expect(subject.repository.exists?).to be(true)
+      end
+
+      it 'does not raise an error' do
+        expect { subject.fetch }.not_to raise_error
+      end
+
+      it 'fetches objects from the source repository' do
+        new_commit_id = new_commit_edit_old_file(source_repository_rugged).oid
+
+        expect(subject.repository.exists?).to be false
+
+        subject.fetch
+
+        expect(subject.repository.commit_count('refs/remotes/origin/master')).to eq(commit_count)
+        expect(subject.repository.commit(new_commit_id).id).to eq(new_commit_id)
+      end
+    end
+  end
 end

spec/lib/gitlab/git/repository_spec.rb

@@ -3,6 +3,7 @@ require "spec_helper"
 
 describe Gitlab::Git::Repository, :seed_helper do
   include Gitlab::EncodingHelper
+  include RepoHelpers
   using RSpec::Parameterized::TableSyntax
 
   shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method|
@@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do
     repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha)
   end
 
-  # Build the options hash that's passed to Rugged::Commit#create
-  def commit_options(repo, index, target, ref, message)
-    options = {}
-    options[:tree] = index.write_tree(repo)
-    options[:author] = {
-      email: "test@example.com",
-      name: "Test Author",
-      time: Time.gm(2014, "mar", 3, 20, 15, 1)
-    }
-    options[:committer] = {
-      email: "test@example.com",
-      name: "Test Author",
-      time: Time.gm(2014, "mar", 3, 20, 15, 1)
-    }
-    options[:message] ||= message
-    options[:parents] = repo.empty? ? [] : [target].compact
-    options[:update_ref] = ref
-
-    options
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of CHANGELOG with a single new line of text.
-  def new_commit_edit_old_file(repo)
-    oid = repo.write("I replaced the changelog with this text", :blob)
-    index = repo.index
-    index.read_tree(repo.head.target.tree)
-    index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
-
-    options = commit_options(
-      repo,
-      index,
-      repo.head.target,
-      "HEAD",
-      "Edit CHANGELOG in its original location"
-    )
-
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of the specified file_path with new text.
-  def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
-    oid = repo.write(text, :blob)
-    index = repo.index
-    index.read_tree(branch.target.tree)
-    index.add(path: file_path, oid: oid, mode: 0100644)
-    options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of encoding/CHANGELOG with new text.
-  def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
-    branch = repo.branches[branch_name]
-    new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Moves the
-  # CHANGELOG file to the encoding/ directory.
-  def new_commit_move_file(repo)
-    blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
-    file_content = repo.lookup(blob_oid).content
-    oid = repo.write(file_content, :blob)
-    index = repo.index
-    index.read_tree(repo.head.target.tree)
-    index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
-    index.remove("CHANGELOG")
-
-    options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
-
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
   def refs(dir)
     IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line|
       line.split("\t").last

spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb

@@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do
       end
     end
   end
+
+  describe '#fetch' do
+    before do
+      subject.delete
+    end
+
+    it 'restores the pool repository objects' do
+      subject.fetch(project.repository)
+
+      expect(object_pool.repository.exists?).to be(true)
+    end
+
+    context 'when called twice' do
+      it "doesn't raise an error" do
+        subject.delete
+
+        expect { subject.fetch(project.repository) }.not_to raise_error
+      end
+    end
+  end
 end

spec/support/helpers/repo_helpers.rb

@@ -11,6 +11,8 @@ module RepoHelpers
   #   blob.path # => 'files/js/commit.js.coffee'
   #   blob.data # => 'class Commit...'
   #
+  # Build the options hash that's passed to Rugged::Commit#create
+
   def sample_blob
     OpenStruct.new(
       oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3',
@@ -129,4 +131,80 @@ eos
       file_content: content
     ).execute
   end
+
+  def commit_options(repo, index, target, ref, message)
+    options = {}
+    options[:tree] = index.write_tree(repo)
+    options[:author] = {
+      email: "test@example.com",
+      name: "Test Author",
+      time: Time.gm(2014, "mar", 3, 20, 15, 1)
+    }
+    options[:committer] = {
+      email: "test@example.com",
+      name: "Test Author",
+      time: Time.gm(2014, "mar", 3, 20, 15, 1)
+    }
+    options[:message] ||= message
+    options[:parents] = repo.empty? ? [] : [target].compact
+    options[:update_ref] = ref
+
+    options
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of CHANGELOG with a single new line of text.
+  def new_commit_edit_old_file(repo)
+    oid = repo.write("I replaced the changelog with this text", :blob)
+    index = repo.index
+    index.read_tree(repo.head.target.tree)
+    index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
+
+    options = commit_options(
+      repo,
+      index,
+      repo.head.target,
+      "HEAD",
+      "Edit CHANGELOG in its original location"
+    )
+
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of the specified file_path with new text.
+  def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
+    oid = repo.write(text, :blob)
+    index = repo.index
+    index.read_tree(branch.target.tree)
+    index.add(path: file_path, oid: oid, mode: 0100644)
+    options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of encoding/CHANGELOG with new text.
+  def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
+    branch = repo.branches[branch_name]
+    new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Moves the
+  # CHANGELOG file to the encoding/ directory.
+  def new_commit_move_file(repo)
+    blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
+    file_content = repo.lookup(blob_oid).content
+    oid = repo.write(file_content, :blob)
+    index = repo.index
+    index.read_tree(repo.head.target.tree)
+    index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
+    index.remove("CHANGELOG")
+
+    options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
+
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
 end