Commit 06d9d7f8 authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'gitaly-remote-mirror-prep' into 'master'

Move git operations for UpdateRemoteMirrorService into Gitlab::Git

Closes gitaly#841

See merge request gitlab-org/gitlab-ee!3824
parents 8b13452e feccd853
......@@ -31,14 +31,6 @@ module EE
refs.map { |sha| commit(sha.strip) }
end
def push_remote_branches(remote, branches)
gitlab_shell.push_remote_branches(repository_storage_path, disk_path, remote, branches)
end
def delete_remote_branches(remote, branches)
gitlab_shell.delete_remote_branches(repository_storage_path, disk_path, remote, branches)
end
def rebase(user, merge_request)
raw.rebase(user, merge_request.id, branch: merge_request.source_branch,
branch_sha: merge_request.source_branch_sha,
......
......@@ -81,6 +81,10 @@ class RemoteMirror < ActiveRecord::Base
update_status == 'started'
end
def update_repository(options)
raw.update(options)
end
def sync
return unless enabled?
return if Gitlab::Geo.secondary?
......@@ -144,6 +148,10 @@ class RemoteMirror < ActiveRecord::Base
private
def raw
@raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, ref_name)
end
def recently_scheduled?
return false unless self.last_update_started_at
......
module Projects
class UpdateRemoteMirrorService < BaseService
attr_reader :mirror, :errors
attr_reader :errors
def execute(remote_mirror)
@mirror = remote_mirror
@errors = []
return success unless remote_mirror.enabled?
begin
repository.fetch_remote(mirror.ref_name, no_tags: true)
repository.fetch_remote(remote_mirror.ref_name, no_tags: true)
push_branches if changed_branches.present?
delete_branches if deleted_branches.present?
opts = {}
if remote_mirror.only_protected_branches?
opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name)
end
push_tags if changed_tags.present?
delete_tags if deleted_tags.present?
remote_mirror.update_repository(opts)
rescue => e
errors << e.message.strip
end
......@@ -26,105 +26,5 @@ module Projects
success
end
end
private
def local_branches
@local_branches ||= repository.local_branches.each_with_object({}) do |branch, branches|
branches[branch.name] = branch unless skip_branch?(branch.name)
end
end
def remote_branches
@remote_branches ||= repository.remote_branches(mirror.ref_name).each_with_object({}) do |branch, branches|
branches[branch.name] = branch
end
end
def push_branches
default_branch, branches = changed_branches.partition { |name| project.default_branch == name }
# Push the default branch first so it works fine when remote mirror is empty.
branches.unshift(*default_branch)
repository.push_remote_branches(mirror.ref_name, branches)
end
def delete_branches
repository.delete_remote_branches(mirror.ref_name, deleted_branches)
end
def deleted_branches
@deleted_branches ||= refs_to_delete(:branches)
end
def changed_branches
@changed_branches ||= local_branches.each_with_object([]) do |(name, branch), branches|
remote_branch = remote_branches[name]
if remote_branch.nil?
branches << name
elsif branch.dereferenced_target == remote_branch.dereferenced_target
# Already up to date
else
branches << name
end
end
end
def local_tags
@local_tags ||= repository.tags.each_with_object({}) do |tag, tags|
tags[tag.name] = tag
end
end
def remote_tags
@remote_tags ||= repository.remote_tags(mirror.ref_name).each_with_object({}) do |tag, tags|
tags[tag.name] = tag
end
end
def push_tags
repository.push_remote_branches(mirror.ref_name, changed_tags)
end
def delete_tags
repository.delete_remote_branches(mirror.ref_name, deleted_tags)
end
def changed_tags
@changed_tags ||= local_tags.each_with_object([]) do |(name, tag), tags|
remote_tag = remote_tags[name]
if remote_tag.nil? || (tag.dereferenced_target != remote_tag.dereferenced_target)
tags << name
end
end
end
def deleted_tags
@deleted_tags ||= refs_to_delete(:tags)
end
def refs_to_delete(type)
remote_refs = send("remote_#{type}") # rubocop:disable GitlabSecurity/PublicSend
local_refs = send("local_#{type}") # rubocop:disable GitlabSecurity/PublicSend
default_branch_id = project.commit.id
remote_refs.each_with_object([]) do |(name, remote_ref), refs_to_delete|
next if local_refs[name] # skip if branch or tag exist in local repo
next if type == :branches && skip_branch?(name)
remote_ref_id = remote_ref.dereferenced_target.try(:id)
if remote_ref_id && project.repository.rugged_is_ancestor?(remote_ref_id, default_branch_id)
refs_to_delete << name
end
end
end
def skip_branch?(name)
mirror.only_protected_branches? && !ProtectedBranch.protected?(project, name)
end
end
end
module Gitlab
module Git
class RemoteMirror
def initialize(repository, ref_name)
@repository = repository
@ref_name = ref_name
end
def update(only_branches_matching: [], only_tags_matching: [])
local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching)
remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching)
updated_branches = changed_refs(local_branches, remote_branches)
push_branches(updated_branches.keys) if updated_branches.present?
delete_refs(local_branches, remote_branches)
local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching)
remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching)
updated_tags = changed_refs(local_tags, remote_tags)
@repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present?
delete_refs(local_tags, remote_tags)
end
private
def refs_obj(refs, only_refs_matching: [])
refs.each_with_object({}) do |ref, refs|
next if only_refs_matching.present? && !only_refs_matching.include?(ref.name)
refs[ref.name] = ref
end
end
def changed_refs(local_refs, remote_refs)
local_refs.select do |ref_name, ref|
remote_ref = remote_refs[ref_name]
remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target
end
end
def push_branches(branches)
default_branch, branches = branches.partition do |branch|
@repository.root_ref == branch
end
# Push the default branch first so it works fine when remote mirror is empty.
branches.unshift(*default_branch)
@repository.push_remote_branches(@ref_name, branches)
end
def delete_refs(local_refs, remote_refs)
refs = refs_to_delete(local_refs, remote_refs)
@repository.delete_remote_branches(@ref_name, refs.keys) if refs.present?
end
def refs_to_delete(local_refs, remote_refs)
default_branch_id = @repository.commit.id
remote_refs.select do |remote_ref_name, remote_ref|
next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo
remote_ref_id = remote_ref.dereferenced_target.try(:id)
remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id)
end
end
end
end
end
......@@ -21,6 +21,7 @@ module Gitlab
REBASE_WORKTREE_PREFIX = 'rebase'.freeze
SQUASH_WORKTREE_PREFIX = 'squash'.freeze
GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout
NoRepository = Class.new(StandardError)
InvalidBlobName = Class.new(StandardError)
......@@ -83,7 +84,7 @@ module Gitlab
# Rugged repo object
attr_reader :rugged
attr_reader :storage, :gl_repository, :relative_path
attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path
# This initializer method is only used on the client side (gitlab-ce).
# Gitaly-ruby uses a different initializer.
......@@ -93,6 +94,12 @@ module Gitlab
@gl_repository = gl_repository
storage_path = Gitlab.config.repositories.storages[@storage]['path']
@gitlab_projects = Gitlab::Git::GitlabProjects.new(
storage_path,
relative_path,
global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
logger: Rails.logger
)
@path = File.join(storage_path, @relative_path)
@name = @relative_path.split("/").last
@attributes = Gitlab::Git::Attributes.new(path)
......@@ -1281,6 +1288,18 @@ module Gitlab
fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id))
end
def push_remote_branches(remote_name, branch_names, forced: true)
success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names)
success || gitlab_projects_error
end
def delete_remote_branches(remote_name, branch_names)
success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)
success || gitlab_projects_error
end
def gitaly_repository
Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository)
end
......@@ -1927,6 +1946,10 @@ module Gitlab
def fetch_remote(remote_name = 'origin', env: nil)
run_git(['fetch', remote_name], env: env).last.zero?
end
def gitlab_projects_error
raise CommandError, @gitlab_projects.output
end
end
end
end
......@@ -364,47 +364,6 @@ module Gitlab
end
end
# Push branch to remote repository
#
# storage - project's storage path
# project_name - project's disk path
# remote_name - remote name
# branch_names - remote branch names to push
# forced - should we use --force flag
#
# Ex.
# push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature'])
#
def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true)
cmd = gitlab_projects(storage, "#{project_name}.git")
success = cmd.push_branches(remote_name, git_timeout, forced, branch_names)
raise Error, cmd.output unless success
success
end
# Delete branch from remote repository
#
# storage - project's storage path
# project_name - project's disk path
# remote_name - remote name
# branch_names - remote branch names
#
# Ex.
# delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature'])
#
def delete_remote_branches(storage, project_name, remote_name, branch_names)
cmd = gitlab_projects(storage, "#{project_name}.git")
success = cmd.delete_remote_branches(remote_name, branch_names)
raise Error, cmd.output unless success
success
end
protected
def gitlab_shell_path
......
......@@ -18,9 +18,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
let(:storage_path) { TestEnv.repos_path }
describe '.create_hooks' do
let(:repo_path) { File.join(TestEnv.repos_path, 'hook-test.git') }
let(:repo_path) { File.join(storage_path, 'hook-test.git') }
let(:hooks_dir) { File.join(repo_path, 'hooks') }
let(:target_hooks_dir) { Gitlab.config.gitlab_shell.hooks_path }
let(:existing_target) { File.join(repo_path, 'foobar') }
......@@ -645,7 +646,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
after do
Gitlab::Shell.new.remove_repository(TestEnv.repos_path, 'my_project')
Gitlab::Shell.new.remove_repository(storage_path, 'my_project')
end
it 'fetches a repository as a mirror remote' do
......@@ -1843,6 +1844,64 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
end
describe '#gitlab_projects' do
subject { repository.gitlab_projects }
it { expect(subject.shard_path).to eq(storage_path) }
it { expect(subject.repository_relative_path).to eq(repository.relative_path) }
end
context 'gitlab_projects commands' do
let(:gitlab_projects) { repository.gitlab_projects }
let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }
describe '#push_remote_branches' do
subject do
repository.push_remote_branches('downstream-remote', ['master'])
end
it 'executes the command' do
expect(gitlab_projects).to receive(:push_branches)
.with('downstream-remote', timeout, true, ['master'])
.and_return(true)
is_expected.to be_truthy
end
it 'raises an error if the command fails' do
allow(gitlab_projects).to receive(:output) { 'error' }
expect(gitlab_projects).to receive(:push_branches)
.with('downstream-remote', timeout, true, ['master'])
.and_return(false)
expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
end
end
describe '#delete_remote_branches' do
subject do
repository.delete_remote_branches('downstream-remote', ['master'])
end
it 'executes the command' do
expect(gitlab_projects).to receive(:delete_remote_branches)
.with('downstream-remote', ['master'])
.and_return(true)
is_expected.to be_truthy
end
it 'raises an error if the command fails' do
allow(gitlab_projects).to receive(:output) { 'error' }
expect(gitlab_projects).to receive(:delete_remote_branches)
.with('downstream-remote', ['master'])
.and_return(false)
expect { subject }.to raise_error(Gitlab::Git::CommandError, 'error')
end
end
end
def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
rugged = repository.rugged
......
......@@ -652,62 +652,6 @@ describe Gitlab::Shell do
end.to raise_error(Gitlab::Shell::Error, "error")
end
end
describe '#push_remote_branches' do
subject(:result) do
gitlab_shell.push_remote_branches(
project.repository_storage_path,
project.disk_path,
'downstream-remote',
['master']
)
end
it 'executes the command' do
expect(gitlab_projects).to receive(:push_branches)
.with('downstream-remote', timeout, true, ['master'])
.and_return(true)
is_expected.to be_truthy
end
it 'fails to execute the command' do
allow(gitlab_projects).to receive(:output) { 'error' }
expect(gitlab_projects).to receive(:push_branches)
.with('downstream-remote', timeout, true, ['master'])
.and_return(false)
expect { result }.to raise_error(Gitlab::Shell::Error, 'error')
end
end
describe '#delete_remote_branches' do
subject(:result) do
gitlab_shell.delete_remote_branches(
project.repository_storage_path,
project.disk_path,
'downstream-remote',
['master']
)
end
it 'executes the command' do
expect(gitlab_projects).to receive(:delete_remote_branches)
.with('downstream-remote', ['master'])
.and_return(true)
is_expected.to be_truthy
end
it 'fails to execute the command' do
allow(gitlab_projects).to receive(:output) { 'error' }
expect(gitlab_projects).to receive(:delete_remote_branches)
.with('downstream-remote', ['master'])
.and_return(false)
expect { result }.to raise_error(Gitlab::Shell::Error, 'error')
end
end
end
describe 'namespace actions' do
......
......@@ -2218,24 +2218,6 @@ describe Repository do
end
end
describe '#push_remote_branches' do
it 'push branches to the remote repo' do
expect_any_instance_of(Gitlab::Shell).to receive(:push_remote_branches)
.with(repository.repository_storage_path, repository.disk_path, 'remote_name', ['branch'])
repository.push_remote_branches('remote_name', ['branch'])
end
end
describe '#delete_remote_branches' do
it 'delete branches to the remote repo' do
expect_any_instance_of(Gitlab::Shell).to receive(:delete_remote_branches)
.with(repository.repository_storage_path, repository.disk_path, 'remote_name', ['branch'])
repository.delete_remote_branches('remote_name', ['branch'])
end
end
describe '#local_branches' do
it 'returns the local branches' do
masterrev = repository.find_branch('master').dereferenced_target
......
......@@ -4,6 +4,7 @@ describe Projects::UpdateRemoteMirrorService do
let(:project) { create(:project, :repository) }
let(:remote_project) { create(:forked_project_with_submodules) }
let(:repository) { project.repository }
let(:raw_repository) { repository.raw }
let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo, enabled: true, only_protected_branches: false) }
subject { described_class.new(project, project.creator) }
......@@ -11,11 +12,10 @@ describe Projects::UpdateRemoteMirrorService do
describe "#execute" do
before do
create_branch(repository, 'existing-branch')
allow(repository).to receive(:remote_tags) { generate_tags(repository, 'v1.0.0', 'v1.1.0') }
gitlab_shell = Gitlab::Shell.new
allow(repository).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(gitlab_shell).to receive(:push_remote_branches).and_return(true)
allow(raw_repository).to receive(:remote_tags) do
generate_tags(repository, 'v1.0.0', 'v1.1.0')
end
allow(raw_repository).to receive(:push_remote_branches).and_return(true)
end
it 'does nothing when unlicensed' do
......@@ -46,7 +46,7 @@ describe Projects::UpdateRemoteMirrorService do
it "push all the branches the first time" do
allow(repository).to receive(:fetch_remote)
expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, local_branch_names)
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, local_branch_names)
subject.execute(remote_mirror)
end
......@@ -54,7 +54,7 @@ describe Projects::UpdateRemoteMirrorService do
it "does not push anything is remote is up to date" do
allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, local_branch_names) }
expect(repository).not_to receive(:push_remote_branches)
expect(raw_repository).not_to receive(:push_remote_branches)
subject.execute(remote_mirror)
end
......@@ -65,7 +65,7 @@ describe Projects::UpdateRemoteMirrorService do
allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, current_branches) }
create_branch(repository, 'my-new-branch')
expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['my-new-branch'])
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['my-new-branch'])
subject.execute(remote_mirror)
end
......@@ -76,7 +76,7 @@ describe Projects::UpdateRemoteMirrorService do
update_branch(repository, 'existing-branch')
end
expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
subject.execute(remote_mirror)
end
......@@ -98,7 +98,7 @@ describe Projects::UpdateRemoteMirrorService do
update_branch(repository, protected_branch_name)
end
expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
subject.execute(remote_mirror)
end
......@@ -109,7 +109,7 @@ describe Projects::UpdateRemoteMirrorService do
update_branch(repository, 'existing-branch')
end
expect(repository).not_to receive(:push_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
expect(raw_repository).not_to receive(:push_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
subject.execute(remote_mirror)
end
......@@ -123,7 +123,7 @@ describe Projects::UpdateRemoteMirrorService do
update_remote_branch(repository, remote_mirror.ref_name, 'markdown')
end
expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['markdown'])
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['markdown'])
subject.execute(remote_mirror)
end
......@@ -138,7 +138,7 @@ describe Projects::UpdateRemoteMirrorService do
delete_branch(repository, 'existing-branch')
end
expect(repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
subject.execute(remote_mirror)
end
......@@ -163,7 +163,7 @@ describe Projects::UpdateRemoteMirrorService do
delete_branch(repository, protected_branch_name)
end
expect(repository).not_to receive(:delete_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
subject.execute(remote_mirror)
end
......@@ -174,7 +174,7 @@ describe Projects::UpdateRemoteMirrorService do
delete_branch(repository, 'existing-branch')
end
expect(repository).not_to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
subject.execute(remote_mirror)
end
......@@ -196,7 +196,7 @@ describe Projects::UpdateRemoteMirrorService do
create_remote_branch(repository, remote_mirror.ref_name, 'remote-branch', rev.id)
end
expect(repository).not_to receive(:delete_remote_branches)
expect(raw_repository).not_to receive(:delete_remote_branches)
subject.execute(remote_mirror)
end
......@@ -211,7 +211,7 @@ describe Projects::UpdateRemoteMirrorService do
create_remote_branch(repository, remote_mirror.ref_name, protected_branch_name, masterrev.id)
end
expect(repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
subject.execute(remote_mirror)
end
......@@ -229,7 +229,7 @@ describe Projects::UpdateRemoteMirrorService do
create_remote_branch(repository, remote_mirror.ref_name, 'remote-branch', rev.id)
end
expect(repository).not_to receive(:delete_remote_branches)
expect(raw_repository).not_to receive(:delete_remote_branches)
subject.execute(remote_mirror)
end
......@@ -244,7 +244,7 @@ describe Projects::UpdateRemoteMirrorService do
create_remote_branch(repository, remote_mirror.ref_name, 'remote-branch', masterrev.id)
end
expect(repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['remote-branch'])
expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['remote-branch'])
subject.execute(remote_mirror)
end
......@@ -271,9 +271,9 @@ describe Projects::UpdateRemoteMirrorService do
context 'when there are some tags to push' do
it 'pushes tags to remote' do
allow(repository).to receive(:remote_tags) { {} }
allow(raw_repository).to receive(:remote_tags) { {} }
expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['v1.0.0', 'v1.1.0'])
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['v1.0.0', 'v1.1.0'])
subject.execute(remote_mirror)
end
......@@ -282,11 +282,11 @@ describe Projects::UpdateRemoteMirrorService do
context 'when there are some tags to delete' do
it 'deletes tags from remote' do
remote_tags = generate_tags(repository, 'v1.0.0', 'v1.1.0')
allow(repository).to receive(:remote_tags) { remote_tags }
allow(raw_repository).to receive(:remote_tags) { remote_tags }
repository.rm_tag(create(:user), 'v1.0.0')
expect(repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['v1.0.0'])
expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['v1.0.0'])
subject.execute(remote_mirror)
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment