Commit 79cb4d99 authored by Bob Van Landuyt's avatar Bob Van Landuyt

Import projects with LFS objects

If the LFS object already exist, we'll link it tot he existing one, if
not we'll create it.
parent 84ee2ddb
...@@ -13,7 +13,7 @@ module Gitlab ...@@ -13,7 +13,7 @@ module Gitlab
end end
def execute def execute
if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore) if import_file && check_version! && restorers.all?(&:restore)
project_tree.restored_project project_tree.restored_project
else else
raise Projects::ImportService::Error.new(@shared.errors.join(', ')) raise Projects::ImportService::Error.new(@shared.errors.join(', '))
...@@ -24,6 +24,11 @@ module Gitlab ...@@ -24,6 +24,11 @@ module Gitlab
private private
def restorers
[repo_restorer, wiki_restorer, project_tree, avatar_restorer,
uploads_restorer, lfs_restorer]
end
def import_file def import_file
Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
shared: @shared) shared: @shared)
...@@ -60,6 +65,10 @@ module Gitlab ...@@ -60,6 +65,10 @@ module Gitlab
Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
end end
def lfs_restorer
Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
end
def path_with_namespace def path_with_namespace
File.join(@project.namespace.full_path, @project.path) File.join(@project.namespace.full_path, @project.path)
end end
......
module Gitlab
module ImportExport
class LfsRestorer
def initialize(project:, shared:)
@project = project
@shared = shared
end
def restore
return true if lfs_file_paths.empty?
lfs_file_paths.each do |file_path|
link_or_create_lfs_object!(file_path)
end
true
rescue => e
@shared.error(e)
false
end
private
def link_or_create_lfs_object!(path)
size = File.size(path)
oid = LfsObject.calculate_oid(path)
lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size)
lfs_object.file = File.open(path) unless lfs_object.file&.exists?
@project.lfs_storage_project.lfs_objects << lfs_object
end
def lfs_file_paths
@lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*")
end
def lfs_storage_path
File.join(@shared.export_path, 'lfs-objects')
end
end
end
end
...@@ -11,7 +11,7 @@ module Gitlab ...@@ -11,7 +11,7 @@ module Gitlab
def save def save
return true if @project.lfs_objects.empty? return true if @project.lfs_objects.empty?
@project.lfs_objects.each do |lfs_object| @project.lfs_storage_project.lfs_objects.each do |lfs_object|
save_lfs_object(lfs_object) save_lfs_object(lfs_object)
end end
......
require 'spec_helper'
describe Gitlab::ImportExport::Importer do
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
FileUtils.mkdir_p(shared.export_path)
FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path)
end
after do
FileUtils.rm_rf(test_path)
end
describe '#execute' do
it 'succeeds' do
importer.execute
expect(shared.errors).to be_empty
end
it 'extracts the archive' do
expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
importer.execute
end
it 'checks the version' do
expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
importer.execute
end
context 'all restores are executed' do
[
Gitlab::ImportExport::AvatarRestorer,
Gitlab::ImportExport::RepoRestorer,
Gitlab::ImportExport::WikiRestorer,
Gitlab::ImportExport::UploadsRestorer,
Gitlab::ImportExport::LfsRestorer
].each do |restorer|
it "calls the #{restorer}" do
fake_restorer = double(restorer.to_s)
expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
importer.execute
end
end
it 'restores the ProjectTree' do
expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
importer.execute
end
end
end
end
require 'spec_helper'
describe Gitlab::ImportExport::LfsRestorer do
include UploadHelpers
let(:export_path) { "#{Dir.tmpdir}/lfs_object_restorer_spec" }
let(:project) { create(:project) }
let(:shared) { project.import_export_shared }
subject(:restorer) { described_class.new(project: project, shared: shared) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
FileUtils.mkdir_p(shared.export_path)
end
after do
FileUtils.rm_rf(shared.export_path)
end
describe '#restore' do
context 'when the archive contains lfs files' do
let(:dummy_lfs_file_path) { File.join(shared.export_path, 'lfs-objects', 'dummy') }
def create_lfs_object_with_content(content)
dummy_lfs_file = Tempfile.new('existing')
File.write(dummy_lfs_file.path, content)
size = dummy_lfs_file.size
oid = LfsObject.calculate_oid(dummy_lfs_file.path)
LfsObject.create!(oid: oid, size: size, file: dummy_lfs_file)
end
before do
FileUtils.mkdir_p(File.dirname(dummy_lfs_file_path))
File.write(dummy_lfs_file_path, 'not very large')
allow(restorer).to receive(:lfs_file_paths).and_return([dummy_lfs_file_path])
end
it 'creates an lfs object for the project' do
expect { restorer.restore }.to change { project.reload.lfs_objects.size }.by(1)
end
it 'assigns the file correctly' do
restorer.restore
expect(project.lfs_objects.first.file.read).to eq('not very large')
end
it 'links an existing LFS object if it existed' do
lfs_object = create_lfs_object_with_content('not very large')
restorer.restore
expect(project.lfs_objects).to include(lfs_object)
end
it 'succeeds' do
expect(restorer.restore).to be_truthy
expect(shared.errors).to be_empty
end
it 'stores the upload' do
expect_any_instance_of(LfsObjectUploader).to receive(:store!)
restorer.restore
end
end
context 'without any LFS-objects' do
it 'succeeds' do
expect(restorer.restore).to be_truthy
expect(shared.errors).to be_empty
end
end
end
end
...@@ -19,8 +19,9 @@ describe Gitlab::ImportExport::LfsSaver do ...@@ -19,8 +19,9 @@ describe Gitlab::ImportExport::LfsSaver do
describe '#save' do describe '#save' do
context 'when the project has LFS objects' do context 'when the project has LFS objects' do
let(:lfs_object) { create(:lfs_object, :with_file) } let(:lfs_object) { create(:lfs_object, :with_file) }
before do before do
project.lfs_objects << lfs_object\ project.lfs_objects << lfs_object
end end
it 'does not cause errors' do it 'does not cause errors' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment