Commit 91a55ed6 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'gitaly-440-shell-fetch-remote' into 'master'

Migrate Repository.FetchRemote to Gitaly

See merge request !13625
parents 72a04002 b77176d1
...@@ -1044,7 +1044,7 @@ class Repository ...@@ -1044,7 +1044,7 @@ class Repository
end end
def fetch_remote(remote, forced: false, no_tags: false) def fetch_remote(remote, forced: false, no_tags: false)
gitlab_shell.fetch_remote(repository_storage_path, disk_path, remote, forced: forced, no_tags: no_tags) gitlab_shell.fetch_remote(raw_repository, remote, forced: forced, no_tags: no_tags)
end end
def fetch_ref(source_path, source_ref, target_ref) def fetch_ref(source_path, source_ref, target_ref)
......
...@@ -47,6 +47,9 @@ module Gitlab ...@@ -47,6 +47,9 @@ module Gitlab
# Directory name of repo # Directory name of repo
attr_reader :name attr_reader :name
# Relative path of repo
attr_reader :relative_path
# Rugged repo object # Rugged repo object
attr_reader :rugged attr_reader :rugged
......
...@@ -37,6 +37,22 @@ module Gitlab ...@@ -37,6 +37,22 @@ module Gitlab
request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: revision) request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: revision)
GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request)
end end
def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false)
request = Gitaly::FetchRemoteRequest.new(repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags)
if ssh_auth&.ssh_import?
if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
request.ssh_key = ssh_auth.ssh_private_key
end
if ssh_auth.ssh_known_hosts.present?
request.known_hosts = ssh_auth.ssh_known_hosts
end
end
GitalyClient.call(@storage, :repository_service, :fetch_remote, request)
end
end end
end end
end end
...@@ -98,35 +98,26 @@ module Gitlab ...@@ -98,35 +98,26 @@ module Gitlab
# Fetch remote for repository # Fetch remote for repository
# #
# name - project path with namespace # repository - an instance of Git::Repository
# remote - remote name # remote - remote name
# forced - should we use --force flag? # forced - should we use --force flag?
# no_tags - should we use --no-tags flag? # no_tags - should we use --no-tags flag?
# #
# Ex. # Ex.
# fetch_remote("gitlab/gitlab-ci", "upstream") # fetch_remote(my_repo, "upstream")
# #
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false)
args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] gitaly_migrate(:fetch_remote) do |is_enabled|
args << '--force' if forced if is_enabled
args << '--no-tags' if no_tags repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
else
vars = {} storage_path = Gitlab.config.repositories.storages[repository.storage]["path"]
local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
if ssh_auth&.ssh_import?
if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
end end
if ssh_auth.ssh_known_hosts.present?
vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
end end
end end
gitlab_shell_fast_execute_raise_error(args, vars)
end
# Move repository # Move repository
# storage - project's storage path # storage - project's storage path
# path - project path with namespace # path - project path with namespace
...@@ -302,6 +293,26 @@ module Gitlab ...@@ -302,6 +293,26 @@ module Gitlab
private private
def local_fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false)
args = [gitlab_shell_projects_path, 'fetch-remote', storage, name, remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
args << '--force' if forced
args << '--no-tags' if no_tags
vars = {}
if ssh_auth&.ssh_import?
if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
end
if ssh_auth.ssh_known_hosts.present?
vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
end
end
gitlab_shell_fast_execute_raise_error(args, vars)
end
def gitlab_shell_fast_execute(cmd) def gitlab_shell_fast_execute(cmd)
output, status = gitlab_shell_fast_execute_helper(cmd) output, status = gitlab_shell_fast_execute_helper(cmd)
...@@ -325,5 +336,13 @@ module Gitlab ...@@ -325,5 +336,13 @@ module Gitlab
# from wasting I/O by searching through GEM_PATH # from wasting I/O by searching through GEM_PATH
Bundler.with_original_env { Popen.popen(cmd, nil, vars) } Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
end end
def gitaly_migrate(method, &block)
Gitlab::GitalyClient.migrate(method, &block)
rescue GRPC::NotFound, GRPC::BadStatus => e
# Old Popen code returns [Error, output] to the caller, so we
# need to do the same here...
raise Error, e
end
end end
end end
...@@ -186,22 +186,48 @@ describe Gitlab::Shell do ...@@ -186,22 +186,48 @@ describe Gitlab::Shell do
end end
end end
describe '#fetch_remote' do shared_examples 'fetch_remote' do |gitaly_on|
let(:project2) { create(:project, :repository) }
let(:repository) { project2.repository }
def fetch_remote(ssh_auth = nil) def fetch_remote(ssh_auth = nil)
gitlab_shell.fetch_remote('current/storage', 'project/path', 'new/storage', ssh_auth: ssh_auth) gitlab_shell.fetch_remote(repository.raw_repository, 'new/storage', ssh_auth: ssh_auth)
end end
def expect_popen(vars = {}) def expect_popen(fail = false, vars = {})
popen_args = [ popen_args = [
projects_path, projects_path,
'fetch-remote', 'fetch-remote',
'current/storage', TestEnv.repos_path,
'project/path.git', repository.relative_path,
'new/storage', 'new/storage',
Gitlab.config.gitlab_shell.git_timeout.to_s Gitlab.config.gitlab_shell.git_timeout.to_s
] ]
expect(Gitlab::Popen).to receive(:popen).with(popen_args, nil, popen_vars.merge(vars)) return_value = fail ? ["error", 1] : [nil, 0]
expect(Gitlab::Popen).to receive(:popen).with(popen_args, nil, popen_vars.merge(vars)).and_return(return_value)
end
def expect_gitaly_call(fail, vars = {})
receive_fetch_remote =
if fail
receive(:fetch_remote).and_raise(GRPC::NotFound)
else
receive(:fetch_remote).and_return(true)
end
expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive_fetch_remote
end
if gitaly_on
def expect_call(fail, vars = {})
expect_gitaly_call(fail, vars)
end
else
def expect_call(fail, vars = {})
expect_popen(fail, vars)
end
end end
def build_ssh_auth(opts = {}) def build_ssh_auth(opts = {})
...@@ -216,20 +242,20 @@ describe Gitlab::Shell do ...@@ -216,20 +242,20 @@ describe Gitlab::Shell do
end end
it 'returns true when the command succeeds' do it 'returns true when the command succeeds' do
expect_popen.and_return([nil, 0]) expect_call(false)
expect(fetch_remote).to be_truthy expect(fetch_remote).to be_truthy
end end
it 'raises an exception when the command fails' do it 'raises an exception when the command fails' do
expect_popen.and_return(["error", 1]) expect_call(true)
expect { fetch_remote }.to raise_error(Gitlab::Shell::Error, "error") expect { fetch_remote }.to raise_error(Gitlab::Shell::Error)
end end
context 'SSH auth' do context 'SSH auth' do
it 'passes the SSH key if specified' do it 'passes the SSH key if specified' do
expect_popen('GITLAB_SHELL_SSH_KEY' => 'foo').and_return([nil, 0]) expect_call(false, 'GITLAB_SHELL_SSH_KEY' => 'foo')
ssh_auth = build_ssh_auth(ssh_key_auth?: true, ssh_private_key: 'foo') ssh_auth = build_ssh_auth(ssh_key_auth?: true, ssh_private_key: 'foo')
...@@ -237,7 +263,7 @@ describe Gitlab::Shell do ...@@ -237,7 +263,7 @@ describe Gitlab::Shell do
end end
it 'does not pass an empty SSH key' do it 'does not pass an empty SSH key' do
expect_popen.and_return([nil, 0]) expect_call(false)
ssh_auth = build_ssh_auth(ssh_key_auth: true, ssh_private_key: '') ssh_auth = build_ssh_auth(ssh_key_auth: true, ssh_private_key: '')
...@@ -245,7 +271,7 @@ describe Gitlab::Shell do ...@@ -245,7 +271,7 @@ describe Gitlab::Shell do
end end
it 'does not pass the key unless SSH key auth is to be used' do it 'does not pass the key unless SSH key auth is to be used' do
expect_popen.and_return([nil, 0]) expect_call(false)
ssh_auth = build_ssh_auth(ssh_key_auth: false, ssh_private_key: 'foo') ssh_auth = build_ssh_auth(ssh_key_auth: false, ssh_private_key: 'foo')
...@@ -253,7 +279,7 @@ describe Gitlab::Shell do ...@@ -253,7 +279,7 @@ describe Gitlab::Shell do
end end
it 'passes the known_hosts data if specified' do it 'passes the known_hosts data if specified' do
expect_popen('GITLAB_SHELL_KNOWN_HOSTS' => 'foo').and_return([nil, 0]) expect_call(false, 'GITLAB_SHELL_KNOWN_HOSTS' => 'foo')
ssh_auth = build_ssh_auth(ssh_known_hosts: 'foo') ssh_auth = build_ssh_auth(ssh_known_hosts: 'foo')
...@@ -261,7 +287,7 @@ describe Gitlab::Shell do ...@@ -261,7 +287,7 @@ describe Gitlab::Shell do
end end
it 'does not pass empty known_hosts data' do it 'does not pass empty known_hosts data' do
expect_popen.and_return([nil, 0]) expect_call(false)
ssh_auth = build_ssh_auth(ssh_known_hosts: '') ssh_auth = build_ssh_auth(ssh_known_hosts: '')
...@@ -269,7 +295,7 @@ describe Gitlab::Shell do ...@@ -269,7 +295,7 @@ describe Gitlab::Shell do
end end
it 'does not pass known_hosts data unless SSH is to be used' do it 'does not pass known_hosts data unless SSH is to be used' do
expect_popen(popen_vars).and_return([nil, 0]) expect_call(false, popen_vars)
ssh_auth = build_ssh_auth(ssh_import?: false, ssh_known_hosts: 'foo') ssh_auth = build_ssh_auth(ssh_import?: false, ssh_known_hosts: 'foo')
...@@ -278,6 +304,14 @@ describe Gitlab::Shell do ...@@ -278,6 +304,14 @@ describe Gitlab::Shell do
end end
end end
describe '#fetch_remote local', skip_gitaly_mock: true do
it_should_behave_like 'fetch_remote', false
end
describe '#fetch_remote gitaly' do
it_should_behave_like 'fetch_remote', true
end
describe '#import_repository' do describe '#import_repository' do
it 'returns true when the command succeeds' do it 'returns true when the command succeeds' do
expect(Gitlab::Popen).to receive(:popen) expect(Gitlab::Popen).to receive(:popen)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment