Commit f4cc9438 authored by Stan Hu's avatar Stan Hu

Merge branch 'jc-client-for-fetch-objects-into-pool' into 'master'

Add client methods for FetchIntoObjectPool RPC

See merge request gitlab-org/gitlab-ce!27767
parents a78877bf 5ee78765
...@@ -417,7 +417,7 @@ group :ed25519 do ...@@ -417,7 +417,7 @@ group :ed25519 do
end end
# Gitaly GRPC client # Gitaly GRPC client
gem 'gitaly-proto', '~> 1.22.0', require: 'gitaly' gem 'gitaly-proto', '~> 1.26.0', require: 'gitaly'
gem 'grpc', '~> 1.19.0' gem 'grpc', '~> 1.19.0'
......
...@@ -283,7 +283,7 @@ GEM ...@@ -283,7 +283,7 @@ GEM
gettext_i18n_rails (>= 0.7.1) gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0) po_to_json (>= 1.0.0)
rails (>= 3.2.0) rails (>= 3.2.0)
gitaly-proto (1.22.0) gitaly-proto (1.26.0)
grpc (~> 1.0) grpc (~> 1.0)
github-markup (1.7.0) github-markup (1.7.0)
gitlab-default_value_for (3.1.1) gitlab-default_value_for (3.1.1)
...@@ -1056,7 +1056,7 @@ DEPENDENCIES ...@@ -1056,7 +1056,7 @@ DEPENDENCIES
gettext (~> 3.2.2) gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3) gettext_i18n_rails_js (~> 1.3)
gitaly-proto (~> 1.22.0) gitaly-proto (~> 1.26.0)
github-markup (~> 1.7.0) github-markup (~> 1.7.0)
gitlab-default_value_for (~> 3.1.1) gitlab-default_value_for (~> 3.1.1)
gitlab-labkit (~> 0.1.2) gitlab-labkit (~> 0.1.2)
......
...@@ -40,6 +40,10 @@ module Gitlab ...@@ -40,6 +40,10 @@ module Gitlab
@repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path) @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path)
end end
def fetch
object_pool_service.fetch(source_repository)
end
private private
def object_pool_service def object_pool_service
......
...@@ -33,6 +33,15 @@ module Gitlab ...@@ -33,6 +33,15 @@ module Gitlab
GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool,
request, timeout: GitalyClient.fast_timeout) request, timeout: GitalyClient.fast_timeout)
end end
def fetch(repository)
request = Gitaly::FetchIntoObjectPoolRequest.new(
object_pool: object_pool,
origin: repository.gitaly_repository
)
GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
end
end end
end end
end end
...@@ -3,8 +3,12 @@ ...@@ -3,8 +3,12 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Git::ObjectPool do describe Gitlab::Git::ObjectPool do
include RepoHelpers
let(:pool_repository) { create(:pool_repository) } let(:pool_repository) { create(:pool_repository) }
let(:source_repository) { pool_repository.source_project.repository } let(:source_repository) { pool_repository.source_project.repository }
let(:source_repository_path) { File.join(TestEnv.repos_path, source_repository.relative_path) }
let(:source_repository_rugged) { Rugged::Repository.new(source_repository_path) }
subject { pool_repository.object_pool } subject { pool_repository.object_pool }
...@@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do ...@@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do
end end
end end
end end
describe '#fetch' do
let(:commit_count) { source_repository.commit_count }
context "when the object's pool repository exists" do
it 'does not raise an error' do
expect { subject.fetch }.not_to raise_error
end
end
context "when the object's pool repository does not exist" do
before do
subject.delete
end
it "re-creates the object pool's repository" do
subject.fetch
expect(subject.repository.exists?).to be(true)
end
it 'does not raise an error' do
expect { subject.fetch }.not_to raise_error
end
it 'fetches objects from the source repository' do
new_commit_id = new_commit_edit_old_file(source_repository_rugged).oid
expect(subject.repository.exists?).to be false
subject.fetch
expect(subject.repository.commit_count('refs/remotes/origin/master')).to eq(commit_count)
expect(subject.repository.commit(new_commit_id).id).to eq(new_commit_id)
end
end
end
end end
...@@ -3,6 +3,7 @@ require "spec_helper" ...@@ -3,6 +3,7 @@ require "spec_helper"
describe Gitlab::Git::Repository, :seed_helper do describe Gitlab::Git::Repository, :seed_helper do
include Gitlab::EncodingHelper include Gitlab::EncodingHelper
include RepoHelpers
using RSpec::Parameterized::TableSyntax using RSpec::Parameterized::TableSyntax
shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method| shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method|
...@@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do
repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha) repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha)
end end
# Build the options hash that's passed to Rugged::Commit#create
def commit_options(repo, index, target, ref, message)
options = {}
options[:tree] = index.write_tree(repo)
options[:author] = {
email: "test@example.com",
name: "Test Author",
time: Time.gm(2014, "mar", 3, 20, 15, 1)
}
options[:committer] = {
email: "test@example.com",
name: "Test Author",
time: Time.gm(2014, "mar", 3, 20, 15, 1)
}
options[:message] ||= message
options[:parents] = repo.empty? ? [] : [target].compact
options[:update_ref] = ref
options
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of CHANGELOG with a single new line of text.
def new_commit_edit_old_file(repo)
oid = repo.write("I replaced the changelog with this text", :blob)
index = repo.index
index.read_tree(repo.head.target.tree)
index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
options = commit_options(
repo,
index,
repo.head.target,
"HEAD",
"Edit CHANGELOG in its original location"
)
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of the specified file_path with new text.
def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
oid = repo.write(text, :blob)
index = repo.index
index.read_tree(branch.target.tree)
index.add(path: file_path, oid: oid, mode: 0100644)
options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of encoding/CHANGELOG with new text.
def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
branch = repo.branches[branch_name]
new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Moves the
# CHANGELOG file to the encoding/ directory.
def new_commit_move_file(repo)
blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
file_content = repo.lookup(blob_oid).content
oid = repo.write(file_content, :blob)
index = repo.index
index.read_tree(repo.head.target.tree)
index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
index.remove("CHANGELOG")
options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
def refs(dir) def refs(dir)
IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line| IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line|
line.split("\t").last line.split("\t").last
......
...@@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do ...@@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do
end end
end end
end end
describe '#fetch' do
before do
subject.delete
end
it 'restores the pool repository objects' do
subject.fetch(project.repository)
expect(object_pool.repository.exists?).to be(true)
end
context 'when called twice' do
it "doesn't raise an error" do
subject.delete
expect { subject.fetch(project.repository) }.not_to raise_error
end
end
end
end end
...@@ -11,6 +11,8 @@ module RepoHelpers ...@@ -11,6 +11,8 @@ module RepoHelpers
# blob.path # => 'files/js/commit.js.coffee' # blob.path # => 'files/js/commit.js.coffee'
# blob.data # => 'class Commit...' # blob.data # => 'class Commit...'
# #
# Build the options hash that's passed to Rugged::Commit#create
def sample_blob def sample_blob
OpenStruct.new( OpenStruct.new(
oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3', oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3',
...@@ -129,4 +131,80 @@ eos ...@@ -129,4 +131,80 @@ eos
file_content: content file_content: content
).execute ).execute
end end
def commit_options(repo, index, target, ref, message)
options = {}
options[:tree] = index.write_tree(repo)
options[:author] = {
email: "test@example.com",
name: "Test Author",
time: Time.gm(2014, "mar", 3, 20, 15, 1)
}
options[:committer] = {
email: "test@example.com",
name: "Test Author",
time: Time.gm(2014, "mar", 3, 20, 15, 1)
}
options[:message] ||= message
options[:parents] = repo.empty? ? [] : [target].compact
options[:update_ref] = ref
options
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of CHANGELOG with a single new line of text.
def new_commit_edit_old_file(repo)
oid = repo.write("I replaced the changelog with this text", :blob)
index = repo.index
index.read_tree(repo.head.target.tree)
index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
options = commit_options(
repo,
index,
repo.head.target,
"HEAD",
"Edit CHANGELOG in its original location"
)
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of the specified file_path with new text.
def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
oid = repo.write(text, :blob)
index = repo.index
index.read_tree(branch.target.tree)
index.add(path: file_path, oid: oid, mode: 0100644)
options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of encoding/CHANGELOG with new text.
def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
branch = repo.branches[branch_name]
new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Moves the
# CHANGELOG file to the encoding/ directory.
def new_commit_move_file(repo)
blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
file_content = repo.lookup(blob_oid).content
oid = repo.write(file_content, :blob)
index = repo.index
index.read_tree(repo.head.target.tree)
index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
index.remove("CHANGELOG")
options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment