Commit 27e18f18 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'gitaly-fetch-ref' into 'master'

Let fetch_ref pull from Gitaly instead of from disk

Closes gitaly#585

See merge request gitlab-org/gitlab-ce!14588
parents 22bb4872 147e2b21
...@@ -989,7 +989,7 @@ class Repository ...@@ -989,7 +989,7 @@ class Repository
end end
def create_ref(ref, ref_path) def create_ref(ref, ref_path)
fetch_ref(path_to_repo, ref, ref_path) raw_repository.write_ref(ref_path, ref)
end end
def ls_files(ref) def ls_files(ref)
......
...@@ -499,6 +499,8 @@ production: &base ...@@ -499,6 +499,8 @@ production: &base
# Gitaly settings # Gitaly settings
gitaly: gitaly:
# Path to the directory containing Gitaly client executables.
client_path: /home/git/gitaly
# Default Gitaly authentication token. Can be overriden per storage. Can # Default Gitaly authentication token. Can be overriden per storage. Can
# be left blank when Gitaly is running locally on a Unix socket, which # be left blank when Gitaly is running locally on a Unix socket, which
# is the normal way to deploy Gitaly. # is the normal way to deploy Gitaly.
...@@ -664,7 +666,7 @@ test: ...@@ -664,7 +666,7 @@ test:
gitaly_address: unix:tmp/tests/gitaly/gitaly.socket gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
gitaly: gitaly:
enabled: true client_path: tmp/tests/gitaly
token: secret token: secret
backup: backup:
path: tmp/tests/backups path: tmp/tests/backups
......
...@@ -302,7 +302,7 @@ range of inputs, might look like this: ...@@ -302,7 +302,7 @@ range of inputs, might look like this:
```ruby ```ruby
describe "#==" do describe "#==" do
using Rspec::Parameterized::TableSyntax using RSpec::Parameterized::TableSyntax
let(:project1) { create(:project) } let(:project1) { create(:project) }
let(:project2) { create(:project) } let(:project2) { create(:project) }
......
...@@ -53,14 +53,15 @@ module Gitlab ...@@ -53,14 +53,15 @@ module Gitlab
# Rugged repo object # Rugged repo object
attr_reader :rugged attr_reader :rugged
attr_reader :storage, :gl_repository, :relative_path attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver
# 'path' must be the path to a _bare_ git repository, e.g. # This initializer method is only used on the client side (gitlab-ce).
# /path/to/my-repo.git # Gitaly-ruby uses a different initializer.
def initialize(storage, relative_path, gl_repository) def initialize(storage, relative_path, gl_repository)
@storage = storage @storage = storage
@relative_path = relative_path @relative_path = relative_path
@gl_repository = gl_repository @gl_repository = gl_repository
@gitaly_resolver = Gitlab::GitalyClient
storage_path = Gitlab.config.repositories.storages[@storage]['path'] storage_path = Gitlab.config.repositories.storages[@storage]['path']
@path = File.join(storage_path, @relative_path) @path = File.join(storage_path, @relative_path)
...@@ -987,9 +988,9 @@ module Gitlab ...@@ -987,9 +988,9 @@ module Gitlab
def with_repo_tmp_commit(start_repository, start_branch_name, sha) def with_repo_tmp_commit(start_repository, start_branch_name, sha)
tmp_ref = fetch_ref( tmp_ref = fetch_ref(
start_repository.path, start_repository,
"#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}",
"refs/tmp/#{SecureRandom.hex}/head" target_ref: "refs/tmp/#{SecureRandom.hex}/head"
) )
yield commit(sha) yield commit(sha)
...@@ -1021,13 +1022,27 @@ module Gitlab ...@@ -1021,13 +1022,27 @@ module Gitlab
end end
end end
def write_ref(ref_path, sha) def write_ref(ref_path, ref)
rugged.references.create(ref_path, sha, force: true) raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ')
raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00")
command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z]
input = "update #{ref_path}\x00#{ref}\x00\x00"
output, status = circuit_breaker.perform do
popen(command, path) { |stdin| stdin.write(input) }
end
raise GitError, output unless status.zero?
end end
def fetch_ref(source_path, source_ref, target_ref) def fetch_ref(source_repository, source_ref:, target_ref:)
args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled|
message, status = run_git(args) if is_enabled
gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref)
else
local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref)
end
end
# Make sure ref was created, and raise Rugged::ReferenceError when not # Make sure ref was created, and raise Rugged::ReferenceError when not
raise Rugged::ReferenceError, message if status != 0 raise Rugged::ReferenceError, message if status != 0
...@@ -1036,9 +1051,9 @@ module Gitlab ...@@ -1036,9 +1051,9 @@ module Gitlab
end end
# Refactoring aid; allows us to copy code from app/models/repository.rb # Refactoring aid; allows us to copy code from app/models/repository.rb
def run_git(args) def run_git(args, env: {})
circuit_breaker.perform do circuit_breaker.perform do
popen([Gitlab.config.git.bin_path, *args], path) popen([Gitlab.config.git.bin_path, *args], path, env)
end end
end end
...@@ -1498,6 +1513,30 @@ module Gitlab ...@@ -1498,6 +1513,30 @@ module Gitlab
rescue Rugged::ReferenceError => ex rescue Rugged::ReferenceError => ex
raise InvalidRef, ex raise InvalidRef, ex
end end
def local_fetch_ref(source_path, source_ref:, target_ref:)
args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
run_git(args)
end
def gitaly_fetch_ref(source_repository, source_ref:, target_ref:)
gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh'))
gitaly_address = gitaly_resolver.address(source_repository.storage)
gitaly_token = gitaly_resolver.token(source_repository.storage)
request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository)
env = {
'GITALY_ADDRESS' => gitaly_address,
'GITALY_PAYLOAD' => request.to_json,
'GITALY_WD' => Dir.pwd,
'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack"
}
env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present?
args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref})
run_git(args, env: env)
end
end end
end end
end end
...@@ -12,7 +12,7 @@ FactoryGirl.define do ...@@ -12,7 +12,7 @@ FactoryGirl.define do
deployment.project ||= deployment.environment.project deployment.project ||= deployment.environment.project
unless deployment.project.repository_exists? unless deployment.project.repository_exists?
allow(deployment.project.repository).to receive(:fetch_ref) allow(deployment.project.repository).to receive(:create_ref)
end end
end end
end end
......
...@@ -1472,6 +1472,23 @@ describe Gitlab::Git::Repository, seed_helper: true do ...@@ -1472,6 +1472,23 @@ describe Gitlab::Git::Repository, seed_helper: true do
end end
end end
describe '#write_ref' do
context 'validations' do
using RSpec::Parameterized::TableSyntax
where(:ref_path, :ref) do
'foo bar' | '123'
'foobar' | "12\x003"
end
with_them do
it 'raises ArgumentError' do
expect { repository.write_ref(ref_path, ref) }.to raise_error(ArgumentError)
end
end
end
end
def create_remote_branch(repository, remote_name, branch_name, source_branch_name) def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
source_branch = repository.branches.find { |branch| branch.name == source_branch_name } source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
rugged = repository.rugged rugged = repository.rugged
......
...@@ -636,18 +636,18 @@ describe Repository do ...@@ -636,18 +636,18 @@ describe Repository do
describe '#fetch_ref' do describe '#fetch_ref' do
describe 'when storage is broken', broken_storage: true do describe 'when storage is broken', broken_storage: true do
it 'should raise a storage error' do it 'should raise a storage error' do
path = broken_repository.path_to_repo expect_to_raise_storage_error do
broken_repository.fetch_ref(broken_repository, source_ref: '1', target_ref: '2')
expect_to_raise_storage_error { broken_repository.fetch_ref(path, '1', '2') } end
end end
end end
end end
describe '#create_ref' do describe '#create_ref' do
it 'redirects the call to fetch_ref' do it 'redirects the call to write_ref' do
ref, ref_path = '1', '2' ref, ref_path = '1', '2'
expect(repository).to receive(:fetch_ref).with(repository.path_to_repo, ref, ref_path) expect(repository.raw_repository).to receive(:write_ref).with(ref_path, ref)
repository.create_ref(ref, ref_path) repository.create_ref(ref, ref_path)
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment