Commit d5c142b6 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre

Merge branch 'georgekoltsov/project_tree_restorer_refactor' into 'master'

Extract shared functionality from ProjectTreeRestorer

See merge request gitlab-org/gitlab!21243
parents 71f11b5b 3f3625f0
......@@ -5,6 +5,8 @@ module Gitlab
class FileImporter
include Gitlab::ImportExport::CommandLineUtil
ImporterError = Class.new(StandardError)
MAX_RETRIES = 8
IGNORED_FILENAMES = %w(. ..).freeze
......@@ -12,8 +14,8 @@ module Gitlab
new(*args).import
end
def initialize(project:, archive_file:, shared:)
@project = project
def initialize(importable:, archive_file:, shared:)
@importable = importable
@archive_file = archive_file
@shared = shared
end
......@@ -52,7 +54,7 @@ module Gitlab
def decompress_archive
result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
raise ImporterError.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
result
end
......@@ -60,9 +62,9 @@ module Gitlab
def copy_archive
return if @archive_file
@archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project))
@archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable))
download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
download_or_copy_upload(@importable.import_export_upload.import_file, @archive_file)
end
def remove_symlinks
......
......@@ -39,7 +39,7 @@ module Gitlab
end
def import_file
Gitlab::ImportExport::FileImporter.import(project: project,
Gitlab::ImportExport::FileImporter.import(importable: project,
archive_file: archive_file,
shared: shared)
end
......
......@@ -3,10 +3,10 @@
module Gitlab
module ImportExport
class MembersMapper
def initialize(exported_members:, user:, project:)
def initialize(exported_members:, user:, importable:)
@exported_members = user.admin? ? exported_members : []
@user = user
@project = project
@importable = importable
# This needs to run first, as second call would be from #map
# which means project members already exist.
......@@ -19,7 +19,7 @@ module Gitlab
@exported_members.inject(missing_keys_tracking_hash) do |hash, member|
if member['user']
old_user_id = member['user']['id']
existing_user = User.where(find_project_user_query(member)).first
existing_user = User.where(find_user_query(member)).first
hash[old_user_id] = existing_user.id if existing_user && add_team_member(member, existing_user)
else
add_team_member(member)
......@@ -47,39 +47,48 @@ module Gitlab
end
def ensure_default_member!
@project.project_members.destroy_all # rubocop: disable DestroyAll
@importable.members.destroy_all # rubocop: disable DestroyAll
ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true)
relation_class.create!(user: @user, access_level: relation_class::MAINTAINER, source_id: @importable.id, importing: true)
rescue => e
raise e, "Error adding importer user to project members. #{e.message}"
raise e, "Error adding importer user to #{@importable.class} members. #{e.message}"
end
def add_team_member(member, existing_user = nil)
member['user'] = existing_user
ProjectMember.create(member_hash(member)).persisted?
relation_class.create(member_hash(member)).persisted?
end
def member_hash(member)
parsed_hash(member).merge(
'source_id' => @project.id,
'source_id' => @importable.id,
'importing' => true,
'access_level' => [member['access_level'], ProjectMember::MAINTAINER].min
'access_level' => [member['access_level'], relation_class::MAINTAINER].min
).except('user_id')
end
def parsed_hash(member)
Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys,
relation_class: ProjectMember)
Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys,
relation_class: relation_class)
end
def find_project_user_query(member)
def find_user_query(member)
user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username']))
end
def user_arel
@user_arel ||= User.arel_table
end
def relation_class
case @importable
when Project
ProjectMember
when Group
GroupMember
end
end
end
end
end
......@@ -3,9 +3,6 @@
module Gitlab
module ImportExport
class ProjectTreeRestorer
# Relations which cannot be saved at project level (and have a group assigned)
GROUP_MODELS = [GroupLabel, Milestone].freeze
attr_reader :user
attr_reader :shared
attr_reader :project
......@@ -13,34 +10,23 @@ module Gitlab
def initialize(user:, shared:, project:)
@path = File.join(shared.export_path, 'project.json')
@user = user
@shared = shared
@shared = shared
@project = project
end
def restore
begin
@tree_hash = read_tree_hash
rescue => e
Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
end
@tree_hash = read_tree_hash
@project_members = @tree_hash.delete('project_members')
RelationRenameService.rename(@tree_hash)
ActiveRecord::Base.uncached do
ActiveRecord::Base.no_touching do
update_project_params!
create_project_relations!
post_import!
end
end
# ensure that we have latest version of the restore
@project.reload # rubocop:disable Cop/ActiveRecordAssociationReload
if relation_tree_restorer.restore
@project.merge_requests.set_latest_merge_request_diff_ids!
true
true
else
false
end
rescue => e
@shared.error(e)
false
......@@ -51,195 +37,36 @@ module Gitlab
def read_tree_hash
json = IO.read(@path)
ActiveSupport::JSON.decode(json)
end
def members_mapper
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
user: @user,
project: @project)
end
# A Hash of the imported merge request ID -> imported ID.
def merge_requests_mapping
@merge_requests_mapping ||= {}
end
# Loops through the tree of models defined in import_export.yml and
# finds them in the imported JSON so they can be instantiated and saved
# in the DB. The structure and relationships between models are guessed from
# the configuration yaml file too.
# Finally, it updates each attribute in the newly imported project.
def create_project_relations!
project_relations.each(&method(
:process_project_relation!))
end
def post_import!
@project.merge_requests.set_latest_merge_request_diff_ids!
end
def process_project_relation!(relation_key, relation_definition)
data_hashes = @tree_hash.delete(relation_key)
return unless data_hashes
# we do not care if we process array or hash
data_hashes = [data_hashes] unless data_hashes.is_a?(Array)
relation_index = 0
# consume and remove objects from memory
while data_hash = data_hashes.shift
process_project_relation_item!(relation_key, relation_definition, relation_index, data_hash)
relation_index += 1
end
end
def process_project_relation_item!(relation_key, relation_definition, relation_index, data_hash)
relation_object = build_relation(relation_key, relation_definition, data_hash)
return unless relation_object
return if group_model?(relation_object)
relation_object.project = @project
relation_object.save!
save_id_mapping(relation_key, data_hash, relation_object)
rescue => e
# re-raise if not enabled
raise e unless Feature.enabled?(:import_graceful_failures, @project.group, default_enabled: true)
log_import_failure(relation_key, relation_index, e)
end
def log_import_failure(relation_key, relation_index, exception)
Gitlab::Sentry.track_acceptable_exception(exception,
extra: { project_id: @project.id, relation_key: relation_key, relation_index: relation_index })
ImportFailure.create(
project: @project,
relation_key: relation_key,
relation_index: relation_index,
exception_class: exception.class.to_s,
exception_message: exception.message.truncate(255),
correlation_id_value: Labkit::Correlation::CorrelationId.current_id
)
Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
end
# Older, serialized CI pipeline exports may only have a
# merge_request_id and not the full hash of the merge request. To
# import these pipelines, we need to preserve the mapping between
# the old and new the merge request ID.
def save_id_mapping(relation_key, data_hash, relation_object)
return unless relation_key == 'merge_requests'
merge_requests_mapping[data_hash['id']] = relation_object.id
end
def project_relations
@project_relations ||=
reader
.attributes_finder
.find_relations_tree(:project)
.deep_stringify_keys
end
def update_project_params!
project_params = @tree_hash.reject do |key, value|
project_relations.include?(key)
end
project_params = project_params.merge(
present_project_override_params)
# Cleaning all imported and overridden params
project_params = Gitlab::ImportExport::AttributeCleaner.clean(
relation_hash: project_params,
relation_class: Project,
excluded_keys: excluded_keys_for_relation(:project))
@project.assign_attributes(project_params)
@project.drop_visibility_level!
Gitlab::Timeless.timeless(@project) do
@project.save!
end
end
def present_project_override_params
# we filter out the empty strings from the overrides
# keeping the default values configured
project_override_params.transform_values do |value|
value.is_a?(String) ? value.presence : value
end.compact
end
def project_override_params
@project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
end
def build_relations(relation_key, relation_definition, data_hashes)
data_hashes.map do |data_hash|
build_relation(relation_key, relation_definition, data_hash)
end.compact
end
def build_relation(relation_key, relation_definition, data_hash)
# TODO: This is hack to not create relation for the author
# Rather make `RelationFactory#set_note_author` to take care of that
return data_hash if relation_key == 'author'
# create relation objects recursively for all sub-objects
relation_definition.each do |sub_relation_key, sub_relation_definition|
transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
end
Gitlab::ImportExport::RelationFactory.create(
relation_sym: relation_key.to_sym,
relation_hash: data_hash,
members_mapper: members_mapper,
merge_requests_mapping: merge_requests_mapping,
def relation_tree_restorer
@relation_tree_restorer ||= RelationTreeRestorer.new(
user: @user,
project: @project,
excluded_keys: excluded_keys_for_relation(relation_key))
shared: @shared,
importable: @project,
tree_hash: @tree_hash,
members_mapper: members_mapper,
relation_factory: relation_factory,
reader: reader
)
end
def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
sub_data_hash = data_hash[sub_relation_key]
return unless sub_data_hash
# if object is a hash we can create simple object
# as it means that this is 1-to-1 vs 1-to-many
sub_data_hash =
if sub_data_hash.is_a?(Array)
build_relations(
sub_relation_key,
sub_relation_definition,
sub_data_hash).presence
else
build_relation(
sub_relation_key,
sub_relation_definition,
sub_data_hash)
end
# persist object(s) or delete from relation
if sub_data_hash
data_hash[sub_relation_key] = sub_data_hash
else
data_hash.delete(sub_relation_key)
end
def members_mapper
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
user: @user,
importable: @project)
end
def group_model?(relation_object)
GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
def relation_factory
Gitlab::ImportExport::RelationFactory
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
def excluded_keys_for_relation(relation)
reader.attributes_finder.find_excluded_keys(relation)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module ImportExport
class RelationTreeRestorer
# Relations which cannot be saved at project level (and have a group assigned)
GROUP_MODELS = [GroupLabel, Milestone].freeze
attr_reader :user
attr_reader :shared
attr_reader :importable
attr_reader :tree_hash
def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, relation_factory:, reader:)
@user = user
@shared = shared
@importable = importable
@tree_hash = tree_hash
@members_mapper = members_mapper
@relation_factory = relation_factory
@reader = reader
end
def restore
ActiveRecord::Base.uncached do
ActiveRecord::Base.no_touching do
update_params!
create_relations!
end
end
# ensure that we have latest version of the restore
@importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload
true
rescue => e
@shared.error(e)
false
end
private
# Loops through the tree of models defined in import_export.yml and
# finds them in the imported JSON so they can be instantiated and saved
# in the DB. The structure and relationships between models are guessed from
# the configuration yaml file too.
# Finally, it updates each attribute in the newly imported project/group.
def create_relations!
relations.each(&method(:process_relation!))
end
def process_relation!(relation_key, relation_definition)
data_hashes = @tree_hash.delete(relation_key)
return unless data_hashes
# we do not care if we process array or hash
data_hashes = [data_hashes] unless data_hashes.is_a?(Array)
relation_index = 0
# consume and remove objects from memory
while data_hash = data_hashes.shift
process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
relation_index += 1
end
end
def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
relation_object = build_relation(relation_key, relation_definition, data_hash)
return unless relation_object
return if importable_class == Project && group_model?(relation_object)
relation_object.assign_attributes(importable_class_sym => @importable)
relation_object.save!
save_id_mapping(relation_key, data_hash, relation_object)
rescue => e
# re-raise if not enabled
raise e unless Feature.enabled?(:import_graceful_failures, @importable.group, default_enabled: true)
log_import_failure(relation_key, relation_index, e)
end
def log_import_failure(relation_key, relation_index, exception)
Gitlab::Sentry.track_acceptable_exception(
exception,
extra: { project_id: @importable.id,
relation_key: relation_key,
relation_index: relation_index })
ImportFailure.create(
project: @importable,
relation_key: relation_key,
relation_index: relation_index,
exception_class: exception.class.to_s,
exception_message: exception.message.truncate(255),
correlation_id_value: Labkit::Correlation::CorrelationId.current_id
)
end
# Older, serialized CI pipeline exports may only have a
# merge_request_id and not the full hash of the merge request. To
# import these pipelines, we need to preserve the mapping between
# the old and new the merge request ID.
def save_id_mapping(relation_key, data_hash, relation_object)
return unless importable_class == Project
return unless relation_key == 'merge_requests'
merge_requests_mapping[data_hash['id']] = relation_object.id
end
def relations
@relations ||=
@reader
.attributes_finder
.find_relations_tree(importable_class_sym)
.deep_stringify_keys
end
def update_params!
params = @tree_hash.reject do |key, _|
relations.include?(key)
end
params = params.merge(present_override_params)
# Cleaning all imported and overridden params
params = Gitlab::ImportExport::AttributeCleaner.clean(
relation_hash: params,
relation_class: importable_class,
excluded_keys: excluded_keys_for_relation(importable_class_sym))
@importable.assign_attributes(params)
@importable.drop_visibility_level! if importable_class == Project
Gitlab::Timeless.timeless(@importable) do
@importable.save!
end
end
def present_override_params
# we filter out the empty strings from the overrides
# keeping the default values configured
override_params&.transform_values do |value|
value.is_a?(String) ? value.presence : value
end&.compact
end
def override_params
@importable_override_params ||= importable_override_params
end
def importable_override_params
if @importable.respond_to?(:import_data)
@importable.import_data&.data&.fetch('override_params', nil) || {}
else
{}
end
end
def build_relations(relation_key, relation_definition, data_hashes)
data_hashes.map do |data_hash|
build_relation(relation_key, relation_definition, data_hash)
end.compact
end
def build_relation(relation_key, relation_definition, data_hash)
# TODO: This is hack to not create relation for the author
# Rather make `RelationFactory#set_note_author` to take care of that
return data_hash if relation_key == 'author'
# create relation objects recursively for all sub-objects
relation_definition.each do |sub_relation_key, sub_relation_definition|
transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
end
@relation_factory.create(relation_factory_params(relation_key, data_hash))
end
def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
sub_data_hash = data_hash[sub_relation_key]
return unless sub_data_hash
# if object is a hash we can create simple object
# as it means that this is 1-to-1 vs 1-to-many
sub_data_hash =
if sub_data_hash.is_a?(Array)
build_relations(
sub_relation_key,
sub_relation_definition,
sub_data_hash).presence
else
build_relation(
sub_relation_key,
sub_relation_definition,
sub_data_hash)
end
# persist object(s) or delete from relation
if sub_data_hash
data_hash[sub_relation_key] = sub_data_hash
else
data_hash.delete(sub_relation_key)
end
end
def group_model?(relation_object)
GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
end
def excluded_keys_for_relation(relation)
@reader.attributes_finder.find_excluded_keys(relation)
end
def importable_class
@importable.class
end
def importable_class_sym
importable_class.to_s.downcase.to_sym
end
# A Hash of the imported merge request ID -> imported ID.
def merge_requests_mapping
@merge_requests_mapping ||= {}
end
def relation_factory_params(relation_key, data_hash)
base_params = {
relation_sym: relation_key.to_sym,
relation_hash: data_hash,
members_mapper: @members_mapper,
user: @user,
excluded_keys: excluded_keys_for_relation(relation_key)
}
base_params[:merge_requests_mapping] = merge_requests_mapping if importable_class == Project
base_params[importable_class_sym] = @importable
base_params
end
end
end
end
{
"description": "Nisi et repellendus ut enim quo accusamus vel magnam.",
"import_type": "gitlab_project",
"creator_id": 123,
"creator_id": 999,
"visibility_level": 10,
"archived": false,
"milestones": [
......
......@@ -31,7 +31,7 @@ describe Gitlab::ImportExport::FileImporter do
context 'normal run' do
before do
described_class.import(project: build(:project), archive_file: '', shared: shared)
described_class.import(importable: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
......@@ -70,7 +70,7 @@ describe Gitlab::ImportExport::FileImporter do
context 'error' do
before do
allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
described_class.import(project: build(:project), archive_file: '', shared: shared)
described_class.import(importable: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
......
......@@ -27,7 +27,7 @@ describe Gitlab::ImportExport::MembersMapper do
"email" => user2.email,
"username" => 'test'
},
"user_id" => 19
"user_id" => 19
},
{
"id" => 3,
......@@ -47,7 +47,7 @@ describe Gitlab::ImportExport::MembersMapper do
let(:members_mapper) do
described_class.new(
exported_members: exported_members, user: user, project: project)
exported_members: exported_members, user: user, importable: project)
end
it 'includes the exported user ID in the map' do
......@@ -83,7 +83,8 @@ describe Gitlab::ImportExport::MembersMapper do
end
it 'removes old user_id from member_hash to avoid conflict with user key' do
expect(ProjectMember).to receive(:create)
expect(ProjectMember)
.to receive(:create)
.twice
.with(hash_excluding('user_id'))
.and_call_original
......@@ -117,7 +118,7 @@ describe Gitlab::ImportExport::MembersMapper do
let(:project) { create(:project, :public, name: 'searchable_project', namespace: group) }
let(:members_mapper) do
described_class.new(
exported_members: exported_members, user: user2, project: project)
exported_members: exported_members, user: user2, importable: project)
end
before do
......@@ -140,7 +141,7 @@ describe Gitlab::ImportExport::MembersMapper do
let(:project) { create(:project, namespace: group) }
let(:members_mapper) do
described_class.new(
exported_members: exported_members, user: user, project: project)
exported_members: exported_members, user: user, importable: project)
end
before do
......@@ -163,7 +164,7 @@ describe Gitlab::ImportExport::MembersMapper do
it 'includes importer specific error message' do
expect(ProjectMember).to receive(:create!).and_raise(StandardError.new(exception_message))
expect { members_mapper.map }.to raise_error(StandardError, "Error adding importer user to project members. #{exception_message}")
expect { members_mapper.map }.to raise_error(StandardError, "Error adding importer user to Project members. #{exception_message}")
end
end
end
......
......@@ -31,9 +31,6 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
project_tree_restorer = described_class.new(user: @user, shared: @shared, project: @project)
expect(Gitlab::ImportExport::RelationFactory).to receive(:create).with(hash_including(excluded_keys: ['whatever'])).and_call_original.at_least(:once)
allow(project_tree_restorer).to receive(:excluded_keys_for_relation).and_return(['whatever'])
@restored_project_json = project_tree_restorer.restore
end
end
......@@ -557,8 +554,9 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
context 'Minimal JSON' do
let(:project) { create(:project) }
let(:user) { create(:user) }
let(:tree_hash) { { 'visibility_level' => visibility } }
let(:restorer) { described_class.new(user: nil, shared: shared, project: project) }
let(:restorer) { described_class.new(user: user, shared: shared, project: project) }
before do
expect(restorer).to receive(:read_tree_hash) { tree_hash }
......
......@@ -203,7 +203,7 @@ describe Gitlab::ImportExport::RelationFactory do
Gitlab::ImportExport::MembersMapper.new(
exported_members: [exported_member],
user: user,
project: project)
importable: project)
end
it 'maps the right author to the imported note' do
......
# frozen_string_literal: true
# This spec is a lightweight version of:
# * project_tree_restorer_spec.rb
#
# In depth testing is being done in the above specs.
# This spec tests that restore project works
# but does not have 100% relation coverage.
require 'spec_helper'
describe Gitlab::ImportExport::RelationTreeRestorer do
include ImportExport::CommonUtil
let(:user) { create(:user) }
let(:shared) { Gitlab::ImportExport::Shared.new(importable) }
let(:members_mapper) { Gitlab::ImportExport::MembersMapper.new(exported_members: {}, user: user, importable: importable) }
let(:importable_hash) do
json = IO.read(path)
ActiveSupport::JSON.decode(json)
end
let(:relation_tree_restorer) do
described_class.new(
user: user,
shared: shared,
tree_hash: tree_hash,
importable: importable,
members_mapper: members_mapper,
relation_factory: relation_factory,
reader: reader
)
end
subject { relation_tree_restorer.restore }
context 'when restoring a project' do
let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' }
let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') }
let(:relation_factory) { Gitlab::ImportExport::RelationFactory }
let(:reader) { Gitlab::ImportExport::Reader.new(shared: shared) }
let(:tree_hash) { importable_hash }
it 'restores project tree' do
expect(subject).to eq(true)
end
describe 'imported project' do
let(:project) { Project.find_by_path('project') }
before do
subject
end
it 'has the project attributes and relations' do
expect(project.description).to eq('Nisi et repellendus ut enim quo accusamus vel magnam.')
expect(project.labels.count).to eq(3)
expect(project.boards.count).to eq(1)
expect(project.project_feature).not_to be_nil
expect(project.custom_attributes.count).to eq(2)
expect(project.project_badges.count).to eq(2)
expect(project.snippets.count).to eq(1)
end
end
end
end
......@@ -32,7 +32,7 @@ RSpec.shared_examples 'restores project successfully' do |**results|
it 'does not set params that are excluded from import_export settings' do
expect(project.import_type).to be_nil
expect(project.creator_id).not_to eq 123
expect(project.creator_id).not_to eq 999
end
it 'records exact number of import failures' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment