Commit 15ef5d66 authored by Robert Speicher's avatar Robert Speicher

Merge branch 'feature/migrate-import-repository-to-gitaly' into 'master'

Migrate importing repository to Gitaly

Closes gitaly#907

See merge request gitlab-org/gitlab-ce!16431
parents 5c51cbca 49d6ed9d
......@@ -403,7 +403,7 @@ group :ed25519 do
end
# Gitaly GRPC client
gem 'gitaly-proto', '~> 0.69.0', require: 'gitaly'
gem 'gitaly-proto', '~> 0.73.0', require: 'gitaly'
gem 'toml-rb', '~> 0.3.15', require: false
......
......@@ -284,7 +284,7 @@ GEM
po_to_json (>= 1.0.0)
rails (>= 3.2.0)
gherkin-ruby (0.3.2)
gitaly-proto (0.69.0)
gitaly-proto (0.73.0)
google-protobuf (~> 3.1)
grpc (~> 1.0)
github-linguist (4.7.6)
......@@ -1054,7 +1054,7 @@ DEPENDENCIES
gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.2.0)
gitaly-proto (~> 0.69.0)
gitaly-proto (~> 0.73.0)
github-linguist (~> 4.7.0)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-markup (~> 1.6.2)
......
......@@ -44,29 +44,13 @@ module Gitlab
# Import project via git clone --bare
# URL must be publicly cloneable
def import_project(source, timeout)
# Skip import if repo already exists
return false if File.exist?(repository_absolute_path)
masked_source = mask_password_in_url(source)
logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
success = run_with_timeout(cmd, timeout, nil)
unless success
logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
FileUtils.rm_rf(repository_absolute_path)
return false
Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled|
if is_enabled
gitaly_import_repository(source)
else
git_import_repository(source, timeout)
end
end
Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
# The project was imported successfully.
# Remove the origin URL since it may contain password.
remove_origin_in_repo
true
end
def fork_repository(new_shard_path, new_repository_relative_path)
......@@ -231,6 +215,42 @@ module Gitlab
raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'")
end
def git_import_repository(source, timeout)
# Skip import if repo already exists
return false if File.exist?(repository_absolute_path)
masked_source = mask_password_in_url(source)
logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
success = run_with_timeout(cmd, timeout, nil)
unless success
logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
FileUtils.rm_rf(repository_absolute_path)
return false
end
Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
# The project was imported successfully.
# Remove the origin URL since it may contain password.
remove_origin_in_repo
true
end
def gitaly_import_repository(source)
raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
true
rescue GRPC::BadStatus => e
@output << e.message
false
end
def git_fork_repository(new_shard_path, new_repository_relative_path)
from_path = repository_absolute_path
to_path = File.join(new_shard_path, new_repository_relative_path)
......
......@@ -100,6 +100,21 @@ module Gitlab
)
end
def import_repository(source)
request = Gitaly::CreateRepositoryFromURLRequest.new(
repository: @gitaly_repo,
url: source
)
GitalyClient.call(
@storage,
:repository_service,
:create_repository_from_url,
request,
timeout: GitalyClient.default_timeout
)
end
def rebase_in_progress?(rebase_id)
request = Gitaly::IsRebaseInProgressRequest.new(
repository: @gitaly_repo,
......
......@@ -158,39 +158,55 @@ describe Gitlab::Git::GitlabProjects do
subject { gl_projects.import_project(import_url, timeout) }
context 'success import' do
it 'imports a repo' do
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
shared_examples 'importing repository' do
context 'success import' do
it 'imports a repo' do
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
expect(logger).to receive(:info).with(message)
is_expected.to be_truthy
is_expected.to be_truthy
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
end
end
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
context 'already exists' do
it "doesn't import" do
FileUtils.mkdir_p(tmp_repo_path)
is_expected.to be_falsy
end
end
end
context 'already exists' do
it "doesn't import" do
FileUtils.mkdir_p(tmp_repo_path)
context 'when Gitaly import_repository feature is enabled' do
it_behaves_like 'importing repository'
end
context 'when Gitaly import_repository feature is disabled', :disable_gitaly do
describe 'logging' do
it 'imports a repo' do
message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
expect(logger).to receive(:info).with(message)
is_expected.to be_falsy
subject
end
end
end
context 'timeout' do
it 'does not import a repo' do
stub_spawn_timeout(cmd, timeout, nil)
context 'timeout' do
it 'does not import a repo' do
stub_spawn_timeout(cmd, timeout, nil)
message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
expect(logger).to receive(:error).with(message)
message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
expect(logger).to receive(:error).with(message)
is_expected.to be_falsy
is_expected.to be_falsy
expect(gl_projects.output).to eq("Timed out\n")
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
expect(gl_projects.output).to eq("Timed out\n")
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
end
end
it_behaves_like 'importing repository'
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment