Commit d8d2b73b authored by Bob Van Landuyt's avatar Bob Van Landuyt

Improve bare repository import

- Allow imports into nested groups
- Make sure it sets the correct visibility level when creating new
  groups

While doing this, I moved the import into a testable class, that made
it easier to improve.
parent 0f74ba96
---
title: 'Improve bare project import: Allow subgroups, take default visibility level
into account'
merge_request: 13670
author:
type: fixed
module Gitlab
class BareRepositoryImporter
NoAdminError = Class.new(StandardError)
def self.execute
Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
git_base_path = repository_storage['path']
repos_to_import = Dir.glob(git_base_path + '/**/*.git')
repos_to_import.each do |repo_path|
if repo_path.end_with?('.wiki.git')
log " * Skipping wiki repo"
next
end
log "Processing #{repo_path}".color(:yellow)
repo_relative_path = repo_path[repository_storage['path'].length..-1]
.sub(/^\//, '') # Remove leading `/`
.sub(/\.git$/, '') # Remove `.git` at the end
new(storage_name, repo_relative_path).create_project_if_needed
end
end
end
attr_reader :storage_name, :full_path, :group_path, :project_path, :user
delegate :log, to: :class
def initialize(storage_name, repo_path)
@storage_name = storage_name
@full_path = repo_path
unless @user = User.admins.order_id_asc.first
raise NoAdminError.new('No admin user found to import repositories')
end
@group_path, @project_path = File.split(repo_path)
@group_path = nil if @group_path == '.'
end
def create_project_if_needed
if project = Project.find_by_full_path(full_path)
log " * #{project.name} (#{full_path}) exists"
return project
end
create_project
end
private
def create_project
group = find_or_create_group
project_params = {
name: project_path,
path: project_path,
repository_storage: storage_name,
namespace_id: group&.id
}
project = Projects::CreateService.new(user, project_params).execute
if project.persisted?
log " * Created #{project.name} (#{full_path})".color(:green)
ProjectCacheWorker.perform_async(project.id)
else
log " * Failed trying to create #{project.name} (#{full_path})".color(:red)
log " Errors: #{project.errors.messages}".color(:red)
end
project
end
def find_or_create_group
return nil unless group_path
if namespace = Namespace.find_by_full_path(group_path)
log " * Namespace #{group_path} exists.".color(:green)
return namespace
end
create_group_path
end
def create_group_path
group_path_segments = group_path.split('/')
new_group, parent_group = nil
partial_path_segments = []
while subgroup_name = group_path_segments.shift
partial_path_segments << subgroup_name
partial_path = partial_path_segments.join('/')
unless new_group = Group.find_by_full_path(partial_path)
log " * Creating group #{partial_path}.".color(:green)
params = {
path: subgroup_name,
name: subgroup_name,
parent: parent_group,
visibility_level: Gitlab::CurrentSettings.current_application_settings.default_group_visibility
}
new_group = Groups::CreateService.new(user, params).execute
end
if new_group.persisted?
log " * Group #{partial_path} (#{new_group.id}) available".color(:green)
else
log " * Failed trying to create group #{partial_path}.".color(:red)
log " * Errors: #{new_group.errors.messages}".color(:red)
end
parent_group = new_group
end
new_group
end
# This is called from within a rake task only used by Admins, so allow writing
# to STDOUT
#
# rubocop:disable Rails/Output
def self.log(message)
puts message
end
# rubocop:enable Rails/Output
end
end
......@@ -9,6 +9,7 @@ namespace :gitlab do
# * The project owner will set to the first administator of the system
# * Existing projects will be skipped
#
#
desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
task repos: :environment do
if Project.current_application_settings.hashed_storage_enabled
......@@ -17,69 +18,7 @@ namespace :gitlab do
exit 1
end
Gitlab.config.repositories.storages.each_value do |repository_storage|
git_base_path = repository_storage['path']
repos_to_import = Dir.glob(git_base_path + '/**/*.git')
repos_to_import.each do |repo_path|
# strip repo base path
repo_path[0..git_base_path.length] = ''
path = repo_path.sub(/\.git$/, '')
group_name, name = File.split(path)
group_name = nil if group_name == '.'
puts "Processing #{repo_path}".color(:yellow)
if path.end_with?('.wiki')
puts " * Skipping wiki repo"
next
end
project = Project.find_by_full_path(path)
if project
puts " * #{project.name} (#{repo_path}) exists"
else
user = User.admins.reorder("id").first
project_params = {
name: name,
path: name
}
# find group namespace
if group_name
group = Namespace.find_by(path: group_name)
# create group namespace
unless group
group = Group.new(name: group_name)
group.path = group_name
group.owner = user
if group.save
puts " * Created Group #{group.name} (#{group.id})".color(:green)
else
puts " * Failed trying to create group #{group.name}".color(:red)
end
end
# set project group
project_params[:namespace_id] = group.id
end
project = Projects::CreateService.new(user, project_params).execute
if project.persisted?
puts " * Created #{project.name} (#{repo_path})".color(:green)
ProjectCacheWorker.perform_async(project.id)
else
puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
puts " Errors: #{project.errors.messages}".color(:red)
end
end
end
end
puts "Done!".color(:green)
Gitlab::BareRepositoryImporter.execute
end
end
end
require 'spec_helper'
describe Gitlab::BareRepositoryImporter, repository: true do
subject(:importer) { described_class.new('default', project_path) }
let(:project_path) { 'a-group/a-sub-group/a-project' }
let!(:admin) { create(:admin) }
before do
allow(described_class).to receive(:log)
end
describe '.execute' do
it 'creates a project for a repository in storage' do
FileUtils.mkdir_p(File.join(TestEnv.repos_path, "#{project_path}.git"))
fake_importer = double
expect(described_class).to receive(:new).with('default', project_path)
.and_return(fake_importer)
expect(fake_importer).to receive(:create_project_if_needed)
described_class.execute
end
it 'skips wiki repos' do
FileUtils.mkdir_p(File.join(TestEnv.repos_path, 'the-group', 'the-project.wiki.git'))
expect(described_class).to receive(:log).with(' * Skipping wiki repo')
expect(described_class).not_to receive(:new)
described_class.execute
end
end
describe '#initialize' do
context 'without admin users' do
let(:admin) { nil }
it 'raises an error' do
expect { importer }.to raise_error(Gitlab::BareRepositoryImporter::NoAdminError)
end
end
end
describe '#create_project_if_needed' do
it 'starts an import for a project that did not exist' do
expect(importer).to receive(:create_project)
importer.create_project_if_needed
end
it 'skips importing when the project already exists' do
group = create(:group, path: 'a-group')
subgroup = create(:group, path: 'a-sub-group', parent: group)
project = create(:project, path: 'a-project', namespace: subgroup)
expect(importer).not_to receive(:create_project)
expect(importer).to receive(:log).with(" * #{project.name} (a-group/a-sub-group/a-project) exists")
importer.create_project_if_needed
end
it 'creates a project with the correct path in the database' do
importer.create_project_if_needed
expect(Project.find_by_full_path(project_path)).not_to be_nil
end
it 'creates group and subgroup in the database' do
importer.create_project_if_needed
parent = Group.find_by_full_path('a-group')
child = parent.children.find_by(path: 'a-sub-group')
expect(parent).not_to be_nil
expect(child).not_to be_nil
end
it 'creates the group with correct visibility level' do
allow(Gitlab::CurrentSettings.current_application_settings)
.to receive(:default_group_visibility) { Gitlab::VisibilityLevel::INTERNAL }
project = importer.create_project_if_needed
group = project.namespace
expect(group.visibility_level).to eq(Gitlab::VisibilityLevel::INTERNAL)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment