Commit 2d82ed8a authored by Douwe Maan's avatar Douwe Maan

Merge branch '48778-remove-old-storage-logic-from-import-export-ee' into 'master'

[EE] Resolve "Remove old storage logic from Import/Export"

See merge request gitlab-org/gitlab-ee!7270
parents 9cca757e 051707e0
......@@ -193,10 +193,8 @@ class ProjectsController < Projects::ApplicationController
end
def download_export
if export_project_object_storage?
send_upload(@project.import_export_upload.export_file)
elsif export_project_path
send_file export_project_path, disposition: 'attachment'
if @project.export_file_exists?
send_upload(@project.export_file)
else
redirect_to(
edit_project_path(@project, anchor: 'js-export-project'),
......@@ -434,12 +432,4 @@ class ProjectsController < Projects::ApplicationController
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end
def export_project_path
@export_project_path ||= @project.export_project_path
end
def export_project_object_storage?
@project.export_project_object_exists?
end
end
......@@ -25,8 +25,6 @@ module Storage
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
end
remove_exports!
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
......@@ -101,8 +99,6 @@ module Storage
end
end
end
remove_exports!
end
def remove_legacy_exports!
......
......@@ -254,18 +254,6 @@ class Namespace < ActiveRecord::Base
end
end
# Exports belonging to projects with legacy storage are placed in a common
# subdirectory of the namespace, so a simple `rm -rf` is sufficient to remove
# them.
#
# Exports of projects using hashed storage are placed in a location defined
# only by the project ID, so each must be removed individually.
def remove_exports!
remove_legacy_exports!
all_projects.with_storage_feature(:repository).find_each(&:remove_exports)
end
def refresh_project_authorizations
owner.refresh_authorized_projects
end
......
......@@ -1751,16 +1751,12 @@ class Project < ActiveRecord::Base
import_export_shared.archive_path
end
def export_project_path
Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
end
def export_status
if export_in_progress?
:started
elsif after_export_in_progress?
:after_export_action
elsif export_project_path || export_project_object_exists?
elsif export_file_exists?
:finished
else
:none
......@@ -1775,21 +1771,19 @@ class Project < ActiveRecord::Base
import_export_shared.after_export_in_progress?
end
def remove_exports(path = export_path)
if path.present?
FileUtils.rm_rf(path)
elsif export_project_object_exists?
import_export_upload.remove_export_file!
import_export_upload.save
end
def remove_exports
return unless export_file_exists?
import_export_upload.remove_export_file!
import_export_upload.save
end
def remove_exported_project_file
remove_exports(export_project_path)
def export_file_exists?
export_file&.file
end
def export_project_object_exists?
Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
def export_file
import_export_upload&.export_file
end
def full_path_slug
......
......@@ -18,6 +18,10 @@ class AvatarUploader < GitlabUploader
false
end
def absolute_path
self.class.absolute_path(model.avatar)
end
private
def dynamic_segment
......
---
title: Update Import/Export to only use new storage uploaders logic
merge_request: 21409
author:
type: added
......@@ -9,6 +9,7 @@
> application settings (`/admin/application_settings`) under 'Import sources'.
> - The exports are stored in a temporary [shared directory][tmp] and are deleted
> every 24 hours by a specific worker.
> - ImportExport can use object storage automatically starting from GitLab 11.3
The GitLab Import/Export version can be checked by using:
......@@ -30,12 +31,6 @@ sudo gitlab-rake gitlab:import_export:data
bundle exec rake gitlab:import_export:data RAILS_ENV=production
```
In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]:
```
import_export_object_storage
```
[ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[feature-flags]: https://docs.gitlab.com/ee/api/features.html
[tmp]: ../../development/shared_files.md
......@@ -28,16 +28,12 @@ module EE
::RepositoryImportWorker.new.perform(export_into_project_id)
ensure
export_file.close if export_file.respond_to?(:close)
project.remove_exported_project_file
project.remove_exports
end
def export_file
strong_memoize(:export_file) do
if object_storage?
project.import_export_upload.export_file&.file
else
File.open(project.export_project_path)
end
project.export_file&.file
end
end
......
......@@ -71,23 +71,8 @@ describe ProjectsController do
stub_ee_application_setting(custom_project_templates_group_id: group.id)
end
context 'old upload' do
before do
stub_feature_flags(import_export_object_storage: false)
end
it 'creates the project from project template' do
post :create, project: templates_params
created_project = Project.find_by_path('foo')
expect(flash[:notice]).to eq "Project 'foo' was successfully created."
expect(created_project.repository.empty?).to be false
end
end
context 'object storage' do
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
end
......
......@@ -23,14 +23,13 @@ describe EE::Gitlab::ImportExport::AfterExportStrategies::CustomTemplateExportIm
describe '#execute' do
it 'updates the project import_source with the path to import' do
path = Tempfile.new.path
file = fixture_file_upload('spec/fixtures/project_export.tar.gz')
allow(subject).to receive(:import_upload_path).and_return(path)
expect(Project).to receive(:update).with(project.id, import_source: path).and_call_original
allow(subject).to receive(:export_file).and_return(file)
subject.execute(user, project_template)
expect(project.reload.import_source).to eq path
expect(project.reload.import_export_upload.import_file.file).not_to be_nil
end
it 'imports repository' do
......@@ -42,32 +41,22 @@ describe EE::Gitlab::ImportExport::AfterExportStrategies::CustomTemplateExportIm
end
it 'removes the exported project file after the import' do
expect(project_template).to receive(:remove_exported_project_file)
expect(project_template).to receive(:remove_exports)
subject.execute(user, project_template)
end
describe 'export_file' do
let(:project_template) { create(:project, :with_export) }
before do
allow(subject).to receive(:project).and_return(project_template)
end
context 'without object storage' do
it 'returns the local path' do
subject.execute(user, project_template)
expect(subject.send(:export_file)).not_to be_nil
end
end
context 'with object storage' do
let(:project_template) { create(:project, :with_object_export) }
it 'returns the path from object storage' do
subject.execute(user, project_template)
it 'returns the path from object storage' do
subject.execute(user, project_template)
expect(subject.send(:export_file)).not_to be_nil
end
expect(subject.send(:export_file)).not_to be_nil
end
end
end
......
......@@ -21,12 +21,8 @@ module API
detail 'This feature was introduced in GitLab 10.6.'
end
get ':id/export/download' do
path = user_project.export_project_path
if path
present_disk_file!(path, File.basename(path), 'application/gzip')
elsif user_project.export_project_object_exists?
present_carrierwave_file!(user_project.import_export_upload.export_file)
if user_project.export_file_exists?
present_carrierwave_file!(user_project.export_file)
else
render_api_error!('404 Not found or has expired', 404)
end
......
......@@ -40,10 +40,6 @@ module Gitlab
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end
def object_storage?
Feature.enabled?(:import_export_object_storage)
end
def version
VERSION
end
......
......@@ -53,7 +53,7 @@ module Gitlab
end
def self.lock_file_path(project)
return unless project.export_path || object_storage?
return unless project.export_path || export_file_exists?
lock_path = project.import_export_shared.archive_path
......@@ -83,8 +83,8 @@ module Gitlab
errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
end
def object_storage?
project.export_project_object_exists?
def export_file_exists?
project.export_file_exists?
end
end
end
......
......@@ -23,7 +23,7 @@ module Gitlab
def strategy_execute
handle_response_error(send_file)
project.remove_exported_project_file
project.remove_exports
end
def handle_response_error(response)
......@@ -40,15 +40,11 @@ module Gitlab
def send_file
Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
ensure
export_file.close if export_file && !object_storage?
export_file.close if export_file
end
def export_file
if object_storage?
project.import_export_upload.export_file.file.open
else
File.open(project.export_project_path)
end
project.export_file.open
end
def send_file_options
......@@ -63,11 +59,7 @@ module Gitlab
end
def export_size
if object_storage?
project.import_export_upload.export_file.file.size
else
File.size(project.export_project_path)
end
project.export_file.file.size
end
end
end
......
......@@ -19,7 +19,7 @@ module Gitlab
private
def avatar_export_file
@avatar_export_file ||= Dir["#{avatar_export_path}/*"].first
@avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first
end
def avatar_export_path
......
module Gitlab
module ImportExport
class AvatarSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:)
@project = project
@shared = shared
......@@ -14,19 +12,12 @@ module Gitlab
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared,
relative_export_path: 'avatar',
from: avatar_path
relative_export_path: 'avatar'
).save
rescue => e
@shared.error(e)
false
end
private
def avatar_path
@project.avatar.path
end
end
end
end
......@@ -92,8 +92,6 @@ module Gitlab
end
def remove_import_file
return unless Gitlab::ImportExport.object_storage?
upload = @project.import_export_upload
return unless upload&.import_file&.file
......
......@@ -199,7 +199,7 @@ module Gitlab
end
def excluded_keys_for_relation(relation)
@reader.attributes_finder.find_excluded_keys(relation)
reader.attributes_finder.find_excluded_keys(relation)
end
end
end
......
......@@ -18,7 +18,7 @@ module Gitlab
Rails.logger.info("Saved project export #{archive_file}")
save_on_object_storage if use_object_storage?
save_upload
else
@shared.error(Gitlab::ImportExport::Error.new(error_message))
false
......@@ -27,10 +27,8 @@ module Gitlab
@shared.error(e)
false
ensure
if use_object_storage?
remove_archive
remove_export_path
end
remove_archive
remove_export_path
end
private
......@@ -51,7 +49,7 @@ module Gitlab
@archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
end
def save_on_object_storage
def save_upload
upload = ImportExportUpload.find_or_initialize_by(project: @project)
File.open(archive_file) { |file| upload.export_file = file }
......@@ -59,12 +57,8 @@ module Gitlab
upload.save!
end
def use_object_storage?
Gitlab::ImportExport.object_storage?
end
def error_message
"Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
"Unable to save #{archive_file} into #{@shared.export_path}."
end
end
end
......
......@@ -5,18 +5,13 @@ module Gitlab
UPLOADS_BATCH_SIZE = 100
def initialize(project:, shared:, relative_export_path: 'uploads', from: nil)
def initialize(project:, shared:, relative_export_path: 'uploads')
@project = project
@shared = shared
@relative_export_path = relative_export_path
@from = from || default_uploads_path
end
def save
if File.file?(@from) && @relative_export_path == 'avatar'
copy_files(@from, File.join(uploads_export_path, @project.avatar.filename))
end
copy_project_uploads
true
......@@ -55,17 +50,11 @@ module Gitlab
copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path))
else
next unless Gitlab::ImportExport.object_storage?
download_and_copy(uploader)
end
end
end
def default_uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, @relative_export_path)
end
......
......@@ -2,30 +2,14 @@ module Gitlab
module ImportExport
class UploadsRestorer < UploadsSaver
def restore
if Gitlab::ImportExport.object_storage?
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared
).restore
elsif File.directory?(uploads_export_path)
copy_files(uploads_export_path, uploads_path)
true
else
true # Proceed without uploads
end
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared
).restore
rescue => e
@shared.error(e)
false
end
def uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, 'uploads')
end
end
end
end
module Gitlab
module ImportExport
class UploadsSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:)
@project = project
@shared = shared
......
module Gitlab
module TemplateHelper
include Gitlab::Utils::StrongMemoize
def prepare_template_environment(file)
return unless file
if Gitlab::ImportExport.object_storage?
params[:import_export_upload] = ImportExportUpload.new(import_file: file)
else
FileUtils.mkdir_p(File.dirname(import_upload_path))
FileUtils.copy_entry(file.path, import_upload_path)
params[:import_source] = import_upload_path
end
end
def import_upload_path
strong_memoize(:import_upload_path) do
Gitlab::ImportExport.import_upload_path(filename: tmp_filename)
end
params[:import_export_upload] = ImportExportUpload.new(import_file: file)
end
def tmp_filename
......
......@@ -6,6 +6,8 @@ namespace :gitlab do
desc "GitLab | Update project templates"
task :update_project_templates do
include Gitlab::ImportExport::CommandLineUtil
if Rails.env.production?
puts "This rake task is not meant fo production instances".red
exit(1)
......@@ -52,7 +54,7 @@ namespace :gitlab do
end
Projects::ImportExport::ExportService.new(project, admin).execute
FileUtils.cp(project.export_project_path, template.archive_path)
download_or_copy_upload(project.export_file, template.archive_path)
Projects::DestroyService.new(admin, project).execute
puts "Exported #{template.name}".green
end
......
......@@ -848,37 +848,7 @@ describe ProjectsController do
project.add_maintainer(user)
end
context 'object storage disabled' do
before do
stub_feature_flags(import_export_object_storage: false)
end
context 'when project export is enabled' do
it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(302)
end
end
context 'when project export is disabled' do
before do
stub_application_setting(project_export_enabled?: false)
end
it 'returns 404' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(404)
end
end
end
context 'object storage enabled' do
before do
stub_feature_flags(import_export_object_storage: true)
end
context 'when project export is enabled' do
it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
......
......@@ -141,27 +141,11 @@ FactoryBot.define do
end
trait :with_export do
before(:create) do |_project, _evaluator|
allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { false }
allow(Feature).to receive(:enabled?).with('import_export_object_storage') { false }
end
after(:create) do |project, _evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
end
trait :with_object_export do
before(:create) do |_project, _evaluator|
allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { true }
allow(Feature).to receive(:enabled?).with('import_export_object_storage') { true }
end
after(:create) do |project, evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
end
trait :broken_storage do
after(:create) do |project|
project.update_column(:repository_storage, 'broken')
......
......@@ -25,7 +25,6 @@ describe 'Import/Export - project export integration test', :js do
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
end
after do
......
require 'spec_helper'
describe 'Import/Export - project import integration test', :js do
include Select2Helper
let(:user) { create(:user) }
let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
gitlab_sign_in(user)
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
context 'when selecting the namespace' do
let(:user) { create(:admin) }
let!(:namespace) { user.namespace }
let(:project_path) { 'test-project-path' + SecureRandom.hex }
context 'prefilled the path' do
it 'user imports an exported project successfully' do
visit new_project_path
select2(namespace.id, from: '#project_namespace_id')
fill_in :project_path, with: project_path, visible: true
click_import_project_tab
click_link 'GitLab export'
expect(page).to have_content('Import an exported GitLab project')
expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
attach_file('file', file)
click_on 'Import project'
expect(Project.count).to eq(1)
project = Project.last
expect(project).not_to be_nil
expect(project.description).to eq("Foo Bar")
expect(project.issues).not_to be_empty
expect(project.merge_requests).not_to be_empty
expect(project_hook_exists?(project)).to be true
expect(wiki_exists?(project)).to be true
expect(project.import_state.status).to eq('finished')
end
end
context 'path is not prefilled' do
it 'user imports an exported project successfully' do
visit new_project_path
click_import_project_tab
click_link 'GitLab export'
fill_in :path, with: 'test-project-path', visible: true
attach_file('file', file)
expect { click_on 'Import project' }.to change { Project.count }.by(1)
project = Project.last
expect(project).not_to be_nil
expect(page).to have_content("Project 'test-project-path' is being imported")
end
end
end
it 'invalid project' do
project = create(:project, namespace: user.namespace)
visit new_project_path
select2(user.namespace.id, from: '#project_namespace_id')
fill_in :project_path, with: project.name, visible: true
click_import_project_tab
click_link 'GitLab export'
attach_file('file', file)
click_on 'Import project'
page.within('.flash-container') do
expect(page).to have_content('Project could not be imported')
end
end
def wiki_exists?(project)
wiki = ProjectWiki.new(project)
wiki.repository.exists? && !wiki.repository.empty?
end
def project_hook_exists?(project)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
end
end
def click_import_project_tab
find('#import-project-tab').click
end
end
......@@ -8,7 +8,7 @@ describe 'Import/Export - project import integration test', :js do
let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
before do
stub_feature_flags(import_export_object_storage: false)
stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
gitlab_sign_in(user)
end
......@@ -33,7 +33,6 @@ describe 'Import/Export - project import integration test', :js do
expect(page).to have_content('Import an exported GitLab project')
expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
expect(Gitlab::ImportExport).to receive(:import_upload_path).with(filename: /\A\h{32}\z/).and_call_original
attach_file('file', file)
click_on 'Import project'
......
require 'spec_helper'
describe 'Import/Export - Namespace export file cleanup', :js do
let(:export_path) { Dir.mktmpdir('namespace_export_file_spec') }
before do
allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
shared_examples_for 'handling project exports on namespace change' do
let!(:old_export_path) { project.export_path }
before do
sign_in(create(:admin))
setup_export_project
end
context 'moving the namespace' do
it 'removes the export file' do
expect(File).to exist(old_export_path)
project.namespace.update!(path: build(:namespace).path)
expect(File).not_to exist(old_export_path)
end
end
context 'deleting the namespace' do
it 'removes the export file' do
expect(File).to exist(old_export_path)
project.namespace.destroy
expect(File).not_to exist(old_export_path)
end
end
end
describe 'legacy storage' do
let(:project) { create(:project, :legacy_storage) }
it_behaves_like 'handling project exports on namespace change'
end
describe 'hashed storage' do
let(:project) { create(:project) }
it_behaves_like 'handling project exports on namespace change'
end
def setup_export_project
visit edit_project_path(project)
expect(page).to have_content('Export project')
find(:link, 'Export project').send_keys(:return)
visit edit_project_path(project)
expect(page).to have_content('Download export')
end
end
......@@ -132,7 +132,6 @@ describe Gitlab::Cleanup::ProjectUploads do
let!(:path) { File.join(FileUploader.root, orphaned.model.full_path, orphaned.path) }
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.dirname(path))
......@@ -156,7 +155,6 @@ describe Gitlab::Cleanup::ProjectUploads do
let!(:new_path) { File.join(FileUploader.root, '-', 'project-lost-found', 'wrong', orphaned.path) }
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.dirname(path))
......
require 'spec_helper'
describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
let!(:service) { described_class.new }
let!(:project) { create(:project, :with_object_export) }
let(:shared) { project.import_export_shared }
let!(:user) { create(:user) }
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
stub_feature_flags(import_export_object_storage: true)
end
it 'returns if project exported file is not found' do
allow(project).to receive(:export_project_object_exists?).and_return(false)
expect(service).not_to receive(:strategy_execute)
service.execute(user, project)
end
it 'creates a lock file in the export dir' do
allow(service).to receive(:delete_after_export_lock)
service.execute(user, project)
expect(lock_path_exist?).to be_truthy
end
context 'when the method succeeds' do
it 'removes the lock file' do
service.execute(user, project)
expect(lock_path_exist?).to be_falsey
end
end
context 'when the method fails' do
before do
allow(service).to receive(:strategy_execute).and_call_original
end
context 'when validation fails' do
before do
allow(service).to receive(:invalid?).and_return(true)
end
it 'does not create the lock file' do
expect(service).not_to receive(:create_or_update_after_export_lock)
service.execute(user, project)
end
it 'does not execute main logic' do
expect(service).not_to receive(:strategy_execute)
service.execute(user, project)
end
it 'logs validation errors in shared context' do
expect(service).to receive(:log_validation_errors)
service.execute(user, project)
end
end
context 'when an exception is raised' do
it 'removes the lock' do
expect { service.execute(user, project) }.to raise_error(NotImplementedError)
expect(lock_path_exist?).to be_falsey
end
end
end
end
describe '#log_validation_errors' do
it 'add the message to the shared context' do
errors = %w(test_message test_message2)
allow(service).to receive(:invalid?).and_return(true)
allow(service.errors).to receive(:full_messages).and_return(errors)
expect(shared).to receive(:add_error_message).twice.and_call_original
service.execute(user, project)
expect(shared.errors).to eq errors
end
end
describe '#to_json' do
it 'adds the current strategy class to the serialized attributes' do
params = { param1: 1 }
result = params.merge(klass: described_class.to_s).to_json
expect(described_class.new(params).to_json).to eq result
end
end
def lock_path_exist?
File.exist?(described_class.lock_file_path(project))
end
end
......@@ -9,11 +9,10 @@ describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
stub_feature_flags(import_export_object_storage: false)
end
it 'returns if project exported file is not found' do
allow(project).to receive(:export_project_path).and_return(nil)
allow(project).to receive(:export_file_exists?).and_return(false)
expect(service).not_to receive(:strategy_execute)
......
......@@ -24,34 +24,13 @@ describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
end
describe '#execute' do
context 'without object storage' do
before do
stub_feature_flags(import_export_object_storage: false)
end
it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
expect(project).to receive(:remove_exported_project_file)
strategy.execute(user, project)
end
end
context 'with object storage' do
before do
stub_feature_flags(import_export_object_storage: true)
end
it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
expect(project).to receive(:remove_exports)
expect(project).to receive(:remove_exported_project_file)
strategy.execute(user, project)
end
strategy.execute(user, project)
end
end
end
......@@ -8,8 +8,7 @@ describe Gitlab::ImportExport::AvatarSaver do
before do
FileUtils.mkdir_p("#{shared.export_path}/avatar/")
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:export_path).and_return(export_path)
end
after do
......@@ -19,7 +18,7 @@ describe Gitlab::ImportExport::AvatarSaver do
it 'saves a project avatar' do
described_class.new(project: project_with_avatar, shared: shared).save
expect(File).to exist("#{shared.export_path}/avatar/dk.png")
expect(File).to exist(Dir["#{shared.export_path}/avatar/**/dk.png"].first)
end
it 'is fine not to have an avatar' do
......
require 'spec_helper'
describe Gitlab::ImportExport::FileImporter do
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
let(:valid_file) { "#{shared.export_path}/valid.json" }
let(:symlink_file) { "#{shared.export_path}/invalid.json" }
let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
let(:subfolder_symlink_file) { "#{shared.export_path}/subfolder/invalid.json" }
let(:evil_symlink_file) { "#{shared.export_path}/.\nevil" }
before do
stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
allow(SecureRandom).to receive(:hex).and_return('abcd')
setup_files
end
after do
FileUtils.rm_rf(storage_path)
end
context 'normal run' do
before do
described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
expect(File.exist?(symlink_file)).to be false
end
it 'removes hidden symlinks in root folder' do
expect(File.exist?(hidden_symlink_file)).to be false
end
it 'removes evil symlinks in root folder' do
expect(File.exist?(evil_symlink_file)).to be false
end
it 'removes symlinks in subfolders' do
expect(File.exist?(subfolder_symlink_file)).to be false
end
it 'does not remove a valid file' do
expect(File.exist?(valid_file)).to be true
end
it 'creates the file in the right subfolder' do
expect(shared.export_path).to include('test/abcd')
end
end
context 'error' do
before do
allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
expect(File.exist?(symlink_file)).to be false
end
it 'removes hidden symlinks in root folder' do
expect(File.exist?(hidden_symlink_file)).to be false
end
it 'removes symlinks in subfolders' do
expect(File.exist?(subfolder_symlink_file)).to be false
end
it 'does not remove a valid file' do
expect(File.exist?(valid_file)).to be true
end
end
def setup_files
FileUtils.mkdir_p("#{shared.export_path}/subfolder/")
FileUtils.touch(valid_file)
FileUtils.ln_s(valid_file, symlink_file)
FileUtils.ln_s(valid_file, subfolder_symlink_file)
FileUtils.ln_s(valid_file, hidden_symlink_file)
FileUtils.ln_s(valid_file, evil_symlink_file)
end
end
......@@ -2,7 +2,7 @@ require 'spec_helper'
describe Gitlab::ImportExport::FileImporter do
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
let(:export_path) { "#{Dir.tmpdir}/file_importer_spec" }
let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
let(:valid_file) { "#{shared.export_path}/valid.json" }
let(:symlink_file) { "#{shared.export_path}/invalid.json" }
let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
......@@ -11,7 +11,9 @@ describe Gitlab::ImportExport::FileImporter do
before do
stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
allow(SecureRandom).to receive(:hex).and_return('abcd')
......@@ -19,12 +21,12 @@ describe Gitlab::ImportExport::FileImporter do
end
after do
FileUtils.rm_rf(export_path)
FileUtils.rm_rf(storage_path)
end
context 'normal run' do
before do
described_class.import(project: nil, archive_file: '', shared: shared)
described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
......@@ -55,7 +57,7 @@ describe Gitlab::ImportExport::FileImporter do
context 'error' do
before do
allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
described_class.import(project: nil, archive_file: '', shared: shared)
described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
......
require 'spec_helper'
describe Gitlab::ImportExport::Importer do
let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
let(:project) { create(:project) }
let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(shared.export_path)
ImportExportUpload.create(project: project, import_file: import_file)
end
after do
FileUtils.rm_rf(test_path)
end
describe '#execute' do
it 'succeeds' do
importer.execute
expect(shared.errors).to be_empty
end
it 'extracts the archive' do
expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
importer.execute
end
it 'checks the version' do
expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
importer.execute
end
context 'all restores are executed' do
[
Gitlab::ImportExport::AvatarRestorer,
Gitlab::ImportExport::RepoRestorer,
Gitlab::ImportExport::WikiRestorer,
Gitlab::ImportExport::UploadsRestorer,
Gitlab::ImportExport::LfsRestorer,
Gitlab::ImportExport::StatisticsRestorer
].each do |restorer|
it "calls the #{restorer}" do
fake_restorer = double(restorer.to_s)
expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
importer.execute
end
end
it 'restores the ProjectTree' do
expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
importer.execute
end
it 'removes the import file' do
expect(importer).to receive(:remove_import_file).and_call_original
importer.execute
expect(project.import_export_upload.import_file&.file).to be_nil
end
end
context 'when project successfully restored' do
let!(:existing_project) { create(:project, namespace: user.namespace) }
let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') }
before do
restorers = double(:restorers, all?: true)
allow(subject).to receive(:import_file).and_return(true)
allow(subject).to receive(:check_version!).and_return(true)
allow(subject).to receive(:restorers).and_return(restorers)
allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
end
context 'when import_data' do
context 'has original_path' do
it 'overwrites existing project' do
expect_any_instance_of(::Projects::OverwriteProjectService).to receive(:execute).with(existing_project)
subject.execute
end
end
context 'has not original_path' do
before do
allow(project).to receive(:import_data).and_return(double(data: {}))
end
it 'does not call the overwrite service' do
expect_any_instance_of(::Projects::OverwriteProjectService).not_to receive(:execute).with(existing_project)
subject.execute
end
end
end
end
end
end
......@@ -4,16 +4,18 @@ describe Gitlab::ImportExport::Importer do
let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) }
let(:project) { create(:project) }
let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(shared.export_path)
FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path)
ImportExportUpload.create(project: project, import_file: import_file)
end
after do
......@@ -64,6 +66,14 @@ describe Gitlab::ImportExport::Importer do
importer.execute
end
it 'removes the import file' do
expect(importer).to receive(:remove_import_file).and_call_original
importer.execute
expect(project.import_export_upload.import_file&.file).to be_nil
end
it 'sets the correct visibility_level when visibility level is a string' do
project.create_or_update_import_data(
data: { override_params: { visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s } }
......@@ -85,7 +95,6 @@ describe Gitlab::ImportExport::Importer do
allow(subject).to receive(:import_file).and_return(true)
allow(subject).to receive(:check_version!).and_return(true)
allow(subject).to receive(:restorers).and_return(restorers)
allow(restorers).to receive(:all?).and_return(true)
allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
end
......
......@@ -18,26 +18,12 @@ describe Gitlab::ImportExport::Saver do
FileUtils.rm_rf(export_path)
end
context 'local archive' do
it 'saves the repo to disk' do
stub_feature_flags(import_export_object_storage: false)
it 'saves the repo using object storage' do
stub_uploads_object_storage(ImportExportUploader)
subject.save
subject.save
expect(shared.errors).to be_empty
expect(Dir.empty?(shared.archive_path)).to be false
end
end
context 'object storage' do
it 'saves the repo using object storage' do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(ImportExportUploader)
subject.save
expect(ImportExportUpload.find_by(project: project).export_file.url)
.to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
end
expect(ImportExportUpload.find_by(project: project).export_file.url)
.to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
end
end
......@@ -4,6 +4,7 @@ describe Gitlab::ImportExport::UploadsManager do
let(:shared) { project.import_export_shared }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:project) { create(:project) }
let(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) }
let(:exported_file_path) { "#{shared.export_path}/uploads/#{upload.secret}/#{File.basename(upload.path)}" }
subject(:manager) { described_class.new(project: project, shared: shared) }
......@@ -69,44 +70,20 @@ describe Gitlab::ImportExport::UploadsManager do
end
end
end
end
context 'using object storage' do
let!(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) }
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
end
it 'saves the file' do
fake_uri = double
expect(fake_uri).to receive(:open).and_return(StringIO.new('File content'))
expect(URI).to receive(:parse).and_return(fake_uri)
manager.save
describe '#restore' do
before do
stub_uploads_object_storage(FileUploader)
expect(File.read(exported_file_path)).to eq('File content')
end
FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038'))
FileUtils.touch(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038', "dummy.txt"))
end
describe '#restore' do
context 'using object storage' do
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038'))
FileUtils.touch(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038', "dummy.txt"))
end
it 'restores the file' do
manager.restore
it 'restores the file' do
manager.restore
expect(project.uploads.size).to eq(1)
expect(project.uploads.first.build_uploader.filename).to eq('dummy.txt')
end
end
expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
end
......@@ -8,7 +8,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/random'))
FileUtils.touch(File.join(shared.export_path, 'uploads/random', "dummy.txt"))
FileUtils.touch(File.join(shared.export_path, 'uploads/random', 'dummy.txt'))
end
after do
......@@ -27,9 +27,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
it 'copies the uploads to the project path' do
subject.restore
uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('dummy.txt')
expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
......@@ -45,9 +43,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
it 'copies the uploads to the project path' do
subject.restore
uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) }
expect(uploads).to include('dummy.txt')
expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
end
end
end
......
......@@ -7,7 +7,6 @@ describe Gitlab::ImportExport::UploadsSaver do
let(:shared) { project.import_export_shared }
before do
stub_feature_flags(import_export_object_storage: false)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
end
......
......@@ -406,12 +406,6 @@ describe Namespace do
child.destroy
end
end
it 'removes the exports folder' do
expect(namespace).to receive(:remove_exports!)
namespace.destroy
end
end
context 'hashed storage' do
......@@ -426,12 +420,6 @@ describe Namespace do
expect(File.exist?(deleted_path_in_dir)).to be(false)
end
it 'removes the exports folder' do
expect(namespace).to receive(:remove_exports!)
namespace.destroy
end
end
end
......@@ -830,26 +818,6 @@ describe Namespace do
end
end
describe '#remove_exports' do
let(:legacy_project) { create(:project, :with_export, :legacy_storage, namespace: namespace) }
let(:hashed_project) { create(:project, :with_export, namespace: namespace) }
let(:export_path) { Dir.mktmpdir('namespace_remove_exports_spec') }
let(:legacy_export) { legacy_project.export_project_path }
let(:hashed_export) { hashed_project.export_project_path }
it 'removes exports for legacy and hashed projects' do
allow(Gitlab::ImportExport).to receive(:storage_path) { export_path }
expect(File.exist?(legacy_export)).to be_truthy
expect(File.exist?(hashed_export)).to be_truthy
namespace.remove_exports!
expect(File.exist?(legacy_export)).to be_falsy
expect(File.exist?(hashed_export)).to be_falsy
end
end
describe '#full_path_was' do
context 'when the group has no parent' do
it 'should return the path was' do
......
......@@ -3146,73 +3146,12 @@ describe Project do
end
describe '#remove_export' do
let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
let(:project) { create(:project, :with_export) }
before do
stub_feature_flags(import_export_object_storage: false)
end
it 'removes the exports directory for the project' do
expect(File.exist?(project.export_path)).to be_truthy
allow(FileUtils).to receive(:rm_rf).and_call_original
expect(FileUtils).to receive(:rm_rf).with(project.export_path).and_call_original
it 'removes the export' do
project.remove_exports
expect(File.exist?(project.export_path)).to be_falsy
end
it 'is a no-op on legacy projects when there is no namespace' do
export_path = legacy_project.export_path
legacy_project.namespace.delete
legacy_project.reload
expect(FileUtils).not_to receive(:rm_rf).with(export_path)
legacy_project.remove_exports
expect(File.exist?(export_path)).to be_truthy
end
it 'runs on hashed storage projects when there is no namespace' do
export_path = project.export_path
project.namespace.delete
legacy_project.reload
allow(FileUtils).to receive(:rm_rf).and_call_original
expect(FileUtils).to receive(:rm_rf).with(export_path).and_call_original
project.remove_exports
expect(File.exist?(export_path)).to be_falsy
end
it 'is run when the project is destroyed' do
expect(project).to receive(:remove_exports).and_call_original
project.destroy
end
end
describe '#remove_exported_project_file' do
let(:project) { create(:project, :with_export) }
it 'removes the exported project file' do
stub_feature_flags(import_export_object_storage: false)
exported_file = project.export_project_path
expect(File.exist?(exported_file)).to be_truthy
allow(FileUtils).to receive(:rm_rf).and_call_original
expect(FileUtils).to receive(:rm_rf).with(exported_file).and_call_original
project.remove_exported_project_file
expect(File.exist?(exported_file)).to be_falsy
expect(project.export_file_exists?).to be_falsey
end
end
......
......@@ -4,8 +4,8 @@ describe API::ProjectExport do
set(:project) { create(:project) }
set(:project_none) { create(:project) }
set(:project_started) { create(:project) }
set(:project_finished) { create(:project) }
set(:project_after_export) { create(:project) }
let(:project_finished) { create(:project, :with_export) }
let(:project_after_export) { create(:project, :with_export) }
set(:user) { create(:user) }
set(:admin) { create(:admin) }
......@@ -29,13 +29,7 @@ describe API::ProjectExport do
# simulate exporting work directory
FileUtils.mkdir_p File.join(project_started.export_path, 'securerandom-hex')
# simulate exported
FileUtils.mkdir_p project_finished.export_path
FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz')
# simulate in after export action
FileUtils.mkdir_p project_after_export.export_path
FileUtils.touch File.join(project_after_export.export_path, '_export.tar.gz')
FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export)
end
......@@ -191,14 +185,11 @@ describe API::ProjectExport do
context 'when upload complete' do
before do
FileUtils.rm_rf(project_after_export.export_path)
if project_after_export.export_project_object_exists?
upload = project_after_export.import_export_upload
project_after_export.remove_exports
end
upload.remove_export_file!
upload.save
end
it 'has removed the export' do
expect(project_after_export.export_file_exists?).to be_falsey
end
it_behaves_like '404 response' do
......@@ -273,13 +264,13 @@ describe API::ProjectExport do
before do
stub_uploads_object_storage(ImportExportUploader)
[project, project_finished, project_after_export].each do |p|
p.add_maintainer(user)
project.add_maintainer(user)
project_finished.add_maintainer(user)
project_after_export.add_maintainer(user)
upload = ImportExportUpload.new(project: p)
upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
upload.save!
end
upload = ImportExportUpload.new(project: project)
upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
upload.save!
end
it_behaves_like 'get project download by strategy'
......
......@@ -7,7 +7,6 @@ describe API::ProjectImport do
let(:namespace) { create(:group) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
namespace.add_owner(user)
......
......@@ -52,7 +52,7 @@ module ExportFileHelper
# Expands the compressed file for an exported project into +tmpdir+
def in_directory_with_expanded_export(project)
Dir.mktmpdir do |tmpdir|
export_file = project.export_project_path
export_file = project.export_file.path
_output, exit_status = Gitlab::Popen.popen(%W{tar -zxf #{export_file} -C #{tmpdir}})
yield(exit_status, tmpdir)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment