Skip to content

G
gitlab-ce
Commit a4e75e7a authored by Jacob Vosmaer's avatar Jacob Vosmaer
Browse files
Options

Use Gitaly for fetches and creating bundles

parent 5e471c78
Hide whitespace changes
Inline Side-by-side
Showing with 73 additions and 186 deletions
lib/gitlab/git/repository.rb
View file @ a4e75e7a
......@@ -898,12 +898,8 @@ module Gitlab
end
def fetch_source_branch!(source_repository, source_branch, local_ref)
Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled|
if is_enabled
gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
else
rugged_fetch_source_branch(source_repository, source_branch, local_ref)
end
wrapped_gitaly_errors do
gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
end
end
......@@ -1058,12 +1054,8 @@ module Gitlab
end
def bundle_to_disk(save_path)
gitaly_migrate(:bundle_to_disk) do |is_enabled|
if is_enabled
gitaly_repository_client.create_bundle(save_path)
else
run_git!(%W(bundle create #{save_path} --all))
end
wrapped_gitaly_errors do
gitaly_repository_client.create_bundle(save_path)
end
true
......
lib/gitlab/shell.rb
View file @ a4e75e7a
......@@ -92,21 +92,13 @@ module Gitlab
# Ex.
# import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
def import_repository(storage, name, url)
if url.start_with?('.', '/')
raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
end
relative_path = "#{name}.git"
cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
GitalyGitlabProjects.new(storage, relative_path)
else
# The timeout ensures the subprocess won't hang forever
gitlab_projects(storage, relative_path)
end
end
cmd = GitalyGitlabProjects.new(storage, relative_path)
success = cmd.import_project(url, git_timeout)
raise Error, cmd.output unless success
......@@ -126,12 +118,8 @@ module Gitlab
# fetch_remote(my_repo, "upstream")
#
def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
gitaly_migrate(:fetch_remote) do |is_enabled|
if is_enabled
repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune)
else
local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
end
wrapped_gitaly_errors do
repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune)
end
end
......@@ -389,28 +377,6 @@ module Gitlab
)
end
def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
vars = { force: forced, tags: !no_tags, prune: prune }
if ssh_auth&.ssh_import?
if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
vars[:ssh_key] = ssh_auth.ssh_private_key
end
if ssh_auth.ssh_known_hosts.present?
vars[:known_hosts] = ssh_auth.ssh_known_hosts
end
end
cmd = gitlab_projects(storage_name, repository_relative_path)
success = cmd.fetch_remote(remote, git_timeout, vars)
raise Error, cmd.output unless success
success
end
def gitlab_shell_fast_execute(cmd)
output, status = gitlab_shell_fast_execute_helper(cmd)
......@@ -440,10 +406,6 @@ module Gitlab
Gitlab.config.gitlab_shell.git_timeout
end
def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
wrapped_gitaly_errors { Gitlab::GitalyClient.migrate(method, status: status, &block) }
end
def wrapped_gitaly_errors
yield
rescue GRPC::NotFound, GRPC::BadStatus => e
......
spec/lib/gitlab/git/repository_spec.rb
View file @ a4e75e7a
......@@ -1716,59 +1716,51 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
describe '#fetch_source_branch!' do
shared_examples '#fetch_source_branch!' do
let(:local_ref) { 'refs/merge-requests/1/head' }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') }
after do
ensure_seeds
end
let(:local_ref) { 'refs/merge-requests/1/head' }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') }
context 'when the branch exists' do
context 'when the commit does not exist locally' do
let(:source_branch) { 'new-branch-for-fetch-source-branch' }
let(:source_rugged) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { source_repository.rugged } }
let(:new_oid) { new_commit_edit_old_file(source_rugged).oid }
after do
ensure_seeds
end
before do
source_rugged.branches.create(source_branch, new_oid)
end
context 'when the branch exists' do
context 'when the commit does not exist locally' do
let(:source_branch) { 'new-branch-for-fetch-source-branch' }
let(:source_rugged) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { source_repository.rugged } }
let(:new_oid) { new_commit_edit_old_file(source_rugged).oid }
it 'writes the ref' do
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
expect(repository.commit(local_ref).sha).to eq(new_oid)
end
before do
source_rugged.branches.create(source_branch, new_oid)
end
context 'when the commit exists locally' do
let(:source_branch) { 'master' }
let(:expected_oid) { SeedRepo::LastCommit::ID }
it 'writes the ref' do
# Sanity check: the commit should already exist
expect(repository.commit(expected_oid)).not_to be_nil
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
expect(repository.commit(local_ref).sha).to eq(expected_oid)
end
it 'writes the ref' do
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
expect(repository.commit(local_ref).sha).to eq(new_oid)
end
end
context 'when the branch does not exist' do
let(:source_branch) { 'definitely-not-master' }
context 'when the commit exists locally' do
let(:source_branch) { 'master' }
let(:expected_oid) { SeedRepo::LastCommit::ID }
it 'writes the ref' do
# Sanity check: the commit should already exist
expect(repository.commit(expected_oid)).not_to be_nil
it 'does not write the ref' do
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(false)
expect(repository.commit(local_ref)).to be_nil
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
expect(repository.commit(local_ref).sha).to eq(expected_oid)
end
end
end
it_behaves_like '#fetch_source_branch!'
context 'when the branch does not exist' do
let(:source_branch) { 'definitely-not-master' }
context 'without gitaly', :skip_gitaly_mock do
it_behaves_like '#fetch_source_branch!'
it 'does not write the ref' do
expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(false)
expect(repository.commit(local_ref)).to be_nil
end
end
end
......
spec/lib/gitlab/shell_spec.rb
View file @ a4e75e7a
......@@ -403,46 +403,36 @@ describe Gitlab::Shell do
end
describe '#create_repository' do
shared_examples '#create_repository' do
let(:repository_storage) { 'default' }
let(:repository_storage_path) do
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
end
end
let(:repo_name) { 'project/path' }
let(:created_path) { File.join(repository_storage_path, repo_name + '.git') }
after do
FileUtils.rm_rf(created_path)
let(:repository_storage) { 'default' }
let(:repository_storage_path) do
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
end
end
let(:repo_name) { 'project/path' }
let(:created_path) { File.join(repository_storage_path, repo_name + '.git') }
it 'creates a repository' do
expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_truthy
expect(File.stat(created_path).mode & 0o777).to eq(0o770)
after do
FileUtils.rm_rf(created_path)
end
hooks_path = File.join(created_path, 'hooks')
expect(File.lstat(hooks_path)).to be_symlink
expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path)
end
it 'creates a repository' do
expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_truthy
it 'returns false when the command fails' do
FileUtils.mkdir_p(File.dirname(created_path))
# This file will block the creation of the repo's .git directory. That
# should cause #create_repository to fail.
FileUtils.touch(created_path)
expect(File.stat(created_path).mode & 0o777).to eq(0o770)
expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_falsy
end
hooks_path = File.join(created_path, 'hooks')
expect(File.lstat(hooks_path)).to be_symlink
expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path)
end
context 'with gitaly' do
it_behaves_like '#create_repository'
end
it 'returns false when the command fails' do
FileUtils.mkdir_p(File.dirname(created_path))
# This file will block the creation of the repo's .git directory. That
# should cause #create_repository to fail.
FileUtils.touch(created_path)
context 'without gitaly', :skip_gitaly_mock do
it_behaves_like '#create_repository'
expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_falsy
end
end
......@@ -513,22 +503,12 @@ describe Gitlab::Shell do
end
end
shared_examples 'fetch_remote' do |gitaly_on|
describe '#fetch_remote' do
def fetch_remote(ssh_auth = nil, prune = true)
gitlab_shell.fetch_remote(repository.raw_repository, 'remote-name', ssh_auth: ssh_auth, prune: prune)
end
def expect_gitlab_projects(fail = false, options = {})
expect(gitlab_projects).to receive(:fetch_remote).with(
'remote-name',
timeout,
options
).and_return(!fail)
allow(gitlab_projects).to receive(:output).and_return('error') if fail
end
def expect_gitaly_call(fail, options = {})
def expect_call(fail, options = {})
receive_fetch_remote =
if fail
receive(:fetch_remote).and_raise(GRPC::NotFound)
......@@ -539,16 +519,6 @@ describe Gitlab::Shell do
expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive_fetch_remote
end
if gitaly_on
def expect_call(fail, options = {})
expect_gitaly_call(fail, options)
end
else
def expect_call(fail, options = {})
expect_gitlab_projects(fail, options)
end
end
def build_ssh_auth(opts = {})
defaults = {
ssh_import?: true,
......@@ -634,14 +604,6 @@ describe Gitlab::Shell do
expect(fetch_remote(ssh_auth)).to be_truthy
end
end
end
describe '#fetch_remote local', :skip_gitaly_mock do
it_should_behave_like 'fetch_remote', false
end
describe '#fetch_remote gitaly' do
it_should_behave_like 'fetch_remote', true
context 'gitaly call' do
let(:remote_name) { 'remote-name' }
......@@ -683,25 +645,6 @@ describe Gitlab::Shell do
end.to raise_error(Gitlab::Shell::Error, "error")
end
end
context 'without gitaly', :disable_gitaly do
it 'returns true when the command succeeds' do
expect(gitlab_projects).to receive(:import_project).with(import_url, timeout) { true }
result = gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url)
expect(result).to be_truthy
end
it 'raises an exception when the command fails' do
allow(gitlab_projects).to receive(:output) { 'error' }
expect(gitlab_projects).to receive(:import_project) { false }
expect do
gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url)
end.to raise_error(Gitlab::Shell::Error, "error")
end
end
end
end
......
spec/lib/gitlab/workhorse_spec.rb
View file @ a4e75e7a
......@@ -36,22 +36,20 @@ describe Gitlab::Workhorse do
allow(described_class).to receive(:git_archive_cache_disabled?).and_return(cache_disabled)
end
context 'when Gitaly workhorse_archive feature is enabled' do
it 'sets the header correctly' do
key, command, params = decode_workhorse_header(subject)
it 'sets the header correctly' do
key, command, params = decode_workhorse_header(subject)
expect(key).to eq('Gitlab-Workhorse-Send-Data')
expect(command).to eq('git-archive')
expect(params).to include(gitaly_params)
end
expect(key).to eq('Gitlab-Workhorse-Send-Data')
expect(command).to eq('git-archive')
expect(params).to include(gitaly_params)
end
context 'when archive caching is disabled' do
let(:cache_disabled) { true }
context 'when archive caching is disabled' do
let(:cache_disabled) { true }
it 'tells workhorse not to use the cache' do
_, _, params = decode_workhorse_header(subject)
expect(params).to include({ 'DisableCache' => true })
end
it 'tells workhorse not to use the cache' do
_, _, params = decode_workhorse_header(subject)
expect(params).to include({ 'DisableCache' => true })
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7