Commit 1f765ce9 authored by Douwe Maan's avatar Douwe Maan

Merge branch '48778-remove-old-storage-logic-from-import-export' into 'master'

Resolve "Remove old storage logic from Import/Export"

Closes #51144, #50109, #48778, and #51079

See merge request gitlab-org/gitlab-ce!21409
parents 766bb7fb 1499d176
...@@ -191,10 +191,8 @@ class ProjectsController < Projects::ApplicationController ...@@ -191,10 +191,8 @@ class ProjectsController < Projects::ApplicationController
end end
def download_export def download_export
if export_project_object_storage? if @project.export_file_exists?
send_upload(@project.import_export_upload.export_file) send_upload(@project.export_file)
elsif export_project_path
send_file export_project_path, disposition: 'attachment'
else else
redirect_to( redirect_to(
edit_project_path(@project, anchor: 'js-export-project'), edit_project_path(@project, anchor: 'js-export-project'),
...@@ -425,12 +423,4 @@ class ProjectsController < Projects::ApplicationController ...@@ -425,12 +423,4 @@ class ProjectsController < Projects::ApplicationController
def whitelist_query_limiting def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440') Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end end
def export_project_path
@export_project_path ||= @project.export_project_path
end
def export_project_object_storage?
@project.export_project_object_exists?
end
end end
...@@ -25,8 +25,6 @@ module Storage ...@@ -25,8 +25,6 @@ module Storage
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path) Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
end end
remove_exports!
# If repositories moved successfully we need to # If repositories moved successfully we need to
# send update instructions to users. # send update instructions to users.
# However we cannot allow rollback since we moved namespace dir # However we cannot allow rollback since we moved namespace dir
...@@ -101,8 +99,6 @@ module Storage ...@@ -101,8 +99,6 @@ module Storage
end end
end end
end end
remove_exports!
end end
def remove_legacy_exports! def remove_legacy_exports!
......
...@@ -253,18 +253,6 @@ class Namespace < ActiveRecord::Base ...@@ -253,18 +253,6 @@ class Namespace < ActiveRecord::Base
end end
end end
# Exports belonging to projects with legacy storage are placed in a common
# subdirectory of the namespace, so a simple `rm -rf` is sufficient to remove
# them.
#
# Exports of projects using hashed storage are placed in a location defined
# only by the project ID, so each must be removed individually.
def remove_exports!
remove_legacy_exports!
all_projects.with_storage_feature(:repository).find_each(&:remove_exports)
end
def refresh_project_authorizations def refresh_project_authorizations
owner.refresh_authorized_projects owner.refresh_authorized_projects
end end
......
...@@ -1735,16 +1735,12 @@ class Project < ActiveRecord::Base ...@@ -1735,16 +1735,12 @@ class Project < ActiveRecord::Base
import_export_shared.archive_path import_export_shared.archive_path
end end
def export_project_path
Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
end
def export_status def export_status
if export_in_progress? if export_in_progress?
:started :started
elsif after_export_in_progress? elsif after_export_in_progress?
:after_export_action :after_export_action
elsif export_project_path || export_project_object_exists? elsif export_file_exists?
:finished :finished
else else
:none :none
...@@ -1759,21 +1755,19 @@ class Project < ActiveRecord::Base ...@@ -1759,21 +1755,19 @@ class Project < ActiveRecord::Base
import_export_shared.after_export_in_progress? import_export_shared.after_export_in_progress?
end end
def remove_exports(path = export_path) def remove_exports
if path.present? return unless export_file_exists?
FileUtils.rm_rf(path)
elsif export_project_object_exists?
import_export_upload.remove_export_file! import_export_upload.remove_export_file!
import_export_upload.save import_export_upload.save
end end
end
def remove_exported_project_file def export_file_exists?
remove_exports(export_project_path) export_file&.file
end end
def export_project_object_exists? def export_file
Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file import_export_upload&.export_file
end end
def full_path_slug def full_path_slug
......
...@@ -18,6 +18,10 @@ class AvatarUploader < GitlabUploader ...@@ -18,6 +18,10 @@ class AvatarUploader < GitlabUploader
false false
end end
def absolute_path
self.class.absolute_path(model.avatar)
end
private private
def dynamic_segment def dynamic_segment
......
---
title: Update Import/Export to only use new storage uploaders logic
merge_request: 21409
author:
type: added
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
> application settings (`/admin/application_settings`) under 'Import sources'. > application settings (`/admin/application_settings`) under 'Import sources'.
> - The exports are stored in a temporary [shared directory][tmp] and are deleted > - The exports are stored in a temporary [shared directory][tmp] and are deleted
> every 24 hours by a specific worker. > every 24 hours by a specific worker.
> - ImportExport can use object storage automatically starting from GitLab 11.3
The GitLab Import/Export version can be checked by using: The GitLab Import/Export version can be checked by using:
...@@ -30,12 +31,6 @@ sudo gitlab-rake gitlab:import_export:data ...@@ -30,12 +31,6 @@ sudo gitlab-rake gitlab:import_export:data
bundle exec rake gitlab:import_export:data RAILS_ENV=production bundle exec rake gitlab:import_export:data RAILS_ENV=production
``` ```
In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]:
```
import_export_object_storage
```
[ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050 [ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[feature-flags]: https://docs.gitlab.com/ee/api/features.html [feature-flags]: https://docs.gitlab.com/ee/api/features.html
[tmp]: ../../development/shared_files.md [tmp]: ../../development/shared_files.md
...@@ -21,12 +21,8 @@ module API ...@@ -21,12 +21,8 @@ module API
detail 'This feature was introduced in GitLab 10.6.' detail 'This feature was introduced in GitLab 10.6.'
end end
get ':id/export/download' do get ':id/export/download' do
path = user_project.export_project_path if user_project.export_file_exists?
present_carrierwave_file!(user_project.export_file)
if path
present_disk_file!(path, File.basename(path), 'application/gzip')
elsif user_project.export_project_object_exists?
present_carrierwave_file!(user_project.import_export_upload.export_file)
else else
render_api_error!('404 Not found or has expired', 404) render_api_error!('404 Not found or has expired', 404)
end end
......
...@@ -40,10 +40,6 @@ module Gitlab ...@@ -40,10 +40,6 @@ module Gitlab
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz" "#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end end
def object_storage?
Feature.enabled?(:import_export_object_storage)
end
def version def version
VERSION VERSION
end end
......
...@@ -53,7 +53,7 @@ module Gitlab ...@@ -53,7 +53,7 @@ module Gitlab
end end
def self.lock_file_path(project) def self.lock_file_path(project)
return unless project.export_path || object_storage? return unless project.export_path || export_file_exists?
lock_path = project.import_export_shared.archive_path lock_path = project.import_export_shared.archive_path
...@@ -83,8 +83,8 @@ module Gitlab ...@@ -83,8 +83,8 @@ module Gitlab
errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
end end
def object_storage? def export_file_exists?
project.export_project_object_exists? project.export_file_exists?
end end
end end
end end
......
...@@ -23,7 +23,7 @@ module Gitlab ...@@ -23,7 +23,7 @@ module Gitlab
def strategy_execute def strategy_execute
handle_response_error(send_file) handle_response_error(send_file)
project.remove_exported_project_file project.remove_exports
end end
def handle_response_error(response) def handle_response_error(response)
...@@ -40,15 +40,11 @@ module Gitlab ...@@ -40,15 +40,11 @@ module Gitlab
def send_file def send_file
Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
ensure ensure
export_file.close if export_file && !object_storage? export_file.close if export_file
end end
def export_file def export_file
if object_storage? project.export_file.open
project.import_export_upload.export_file.file.open
else
File.open(project.export_project_path)
end
end end
def send_file_options def send_file_options
...@@ -63,11 +59,7 @@ module Gitlab ...@@ -63,11 +59,7 @@ module Gitlab
end end
def export_size def export_size
if object_storage? project.export_file.file.size
project.import_export_upload.export_file.file.size
else
File.size(project.export_project_path)
end
end end
end end
end end
......
...@@ -19,7 +19,7 @@ module Gitlab ...@@ -19,7 +19,7 @@ module Gitlab
private private
def avatar_export_file def avatar_export_file
@avatar_export_file ||= Dir["#{avatar_export_path}/*"].first @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first
end end
def avatar_export_path def avatar_export_path
......
module Gitlab module Gitlab
module ImportExport module ImportExport
class AvatarSaver class AvatarSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:) def initialize(project:, shared:)
@project = project @project = project
@shared = shared @shared = shared
...@@ -14,19 +12,12 @@ module Gitlab ...@@ -14,19 +12,12 @@ module Gitlab
Gitlab::ImportExport::UploadsManager.new( Gitlab::ImportExport::UploadsManager.new(
project: @project, project: @project,
shared: @shared, shared: @shared,
relative_export_path: 'avatar', relative_export_path: 'avatar'
from: avatar_path
).save ).save
rescue => e rescue => e
@shared.error(e) @shared.error(e)
false false
end end
private
def avatar_path
@project.avatar.path
end
end end
end end
end end
...@@ -92,8 +92,6 @@ module Gitlab ...@@ -92,8 +92,6 @@ module Gitlab
end end
def remove_import_file def remove_import_file
return unless Gitlab::ImportExport.object_storage?
upload = @project.import_export_upload upload = @project.import_export_upload
return unless upload&.import_file&.file return unless upload&.import_file&.file
......
...@@ -199,7 +199,7 @@ module Gitlab ...@@ -199,7 +199,7 @@ module Gitlab
end end
def excluded_keys_for_relation(relation) def excluded_keys_for_relation(relation)
@reader.attributes_finder.find_excluded_keys(relation) reader.attributes_finder.find_excluded_keys(relation)
end end
end end
end end
......
...@@ -18,7 +18,7 @@ module Gitlab ...@@ -18,7 +18,7 @@ module Gitlab
Rails.logger.info("Saved project export #{archive_file}") Rails.logger.info("Saved project export #{archive_file}")
save_on_object_storage if use_object_storage? save_upload
else else
@shared.error(Gitlab::ImportExport::Error.new(error_message)) @shared.error(Gitlab::ImportExport::Error.new(error_message))
false false
...@@ -27,11 +27,9 @@ module Gitlab ...@@ -27,11 +27,9 @@ module Gitlab
@shared.error(e) @shared.error(e)
false false
ensure ensure
if use_object_storage?
remove_archive remove_archive
remove_export_path remove_export_path
end end
end
private private
...@@ -51,7 +49,7 @@ module Gitlab ...@@ -51,7 +49,7 @@ module Gitlab
@archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
end end
def save_on_object_storage def save_upload
upload = ImportExportUpload.find_or_initialize_by(project: @project) upload = ImportExportUpload.find_or_initialize_by(project: @project)
File.open(archive_file) { |file| upload.export_file = file } File.open(archive_file) { |file| upload.export_file = file }
...@@ -59,12 +57,8 @@ module Gitlab ...@@ -59,12 +57,8 @@ module Gitlab
upload.save! upload.save!
end end
def use_object_storage?
Gitlab::ImportExport.object_storage?
end
def error_message def error_message
"Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}" "Unable to save #{archive_file} into #{@shared.export_path}."
end end
end end
end end
......
...@@ -5,18 +5,13 @@ module Gitlab ...@@ -5,18 +5,13 @@ module Gitlab
UPLOADS_BATCH_SIZE = 100 UPLOADS_BATCH_SIZE = 100
def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) def initialize(project:, shared:, relative_export_path: 'uploads')
@project = project @project = project
@shared = shared @shared = shared
@relative_export_path = relative_export_path @relative_export_path = relative_export_path
@from = from || default_uploads_path
end end
def save def save
if File.file?(@from) && @relative_export_path == 'avatar'
copy_files(@from, File.join(uploads_export_path, @project.avatar.filename))
end
copy_project_uploads copy_project_uploads
true true
...@@ -55,17 +50,11 @@ module Gitlab ...@@ -55,17 +50,11 @@ module Gitlab
copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path)) copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path))
else else
next unless Gitlab::ImportExport.object_storage?
download_and_copy(uploader) download_and_copy(uploader)
end end
end end
end end
def default_uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path)
end end
......
...@@ -2,30 +2,14 @@ module Gitlab ...@@ -2,30 +2,14 @@ module Gitlab
module ImportExport module ImportExport
class UploadsRestorer < UploadsSaver class UploadsRestorer < UploadsSaver
def restore def restore
if Gitlab::ImportExport.object_storage?
Gitlab::ImportExport::UploadsManager.new( Gitlab::ImportExport::UploadsManager.new(
project: @project, project: @project,
shared: @shared shared: @shared
).restore ).restore
elsif File.directory?(uploads_export_path)
copy_files(uploads_export_path, uploads_path)
true
else
true # Proceed without uploads
end
rescue => e rescue => e
@shared.error(e) @shared.error(e)
false false
end end
def uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, 'uploads')
end
end end
end end
end end
module Gitlab module Gitlab
module ImportExport module ImportExport
class UploadsSaver class UploadsSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:) def initialize(project:, shared:)
@project = project @project = project
@shared = shared @shared = shared
......
module Gitlab module Gitlab
module TemplateHelper module TemplateHelper
include Gitlab::Utils::StrongMemoize
def prepare_template_environment(file) def prepare_template_environment(file)
return unless file return unless file
if Gitlab::ImportExport.object_storage?
params[:import_export_upload] = ImportExportUpload.new(import_file: file) params[:import_export_upload] = ImportExportUpload.new(import_file: file)
else
FileUtils.mkdir_p(File.dirname(import_upload_path))
FileUtils.copy_entry(file.path, import_upload_path)
params[:import_source] = import_upload_path
end
end
def import_upload_path
strong_memoize(:import_upload_path) do
Gitlab::ImportExport.import_upload_path(filename: tmp_filename)
end
end end
def tmp_filename def tmp_filename
......
...@@ -6,6 +6,8 @@ namespace :gitlab do ...@@ -6,6 +6,8 @@ namespace :gitlab do
desc "GitLab | Update project templates" desc "GitLab | Update project templates"
task :update_project_templates do task :update_project_templates do
include Gitlab::ImportExport::CommandLineUtil
if Rails.env.production? if Rails.env.production?
puts "This rake task is not meant fo production instances".red puts "This rake task is not meant fo production instances".red
exit(1) exit(1)
...@@ -52,7 +54,7 @@ namespace :gitlab do ...@@ -52,7 +54,7 @@ namespace :gitlab do
end end
Projects::ImportExport::ExportService.new(project, admin).execute Projects::ImportExport::ExportService.new(project, admin).execute
FileUtils.cp(project.export_project_path, template.archive_path) download_or_copy_upload(project.export_file, template.archive_path)
Projects::DestroyService.new(admin, project).execute Projects::DestroyService.new(admin, project).execute
puts "Exported #{template.name}".green puts "Exported #{template.name}".green
end end
......
...@@ -803,37 +803,7 @@ describe ProjectsController do ...@@ -803,37 +803,7 @@ describe ProjectsController do
project.add_maintainer(user) project.add_maintainer(user)
end end
context 'object storage disabled' do
before do
stub_feature_flags(import_export_object_storage: false)
end
context 'when project export is enabled' do
it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(302)
end
end
context 'when project export is disabled' do
before do
stub_application_setting(project_export_enabled?: false)
end
it 'returns 404' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(404)
end
end
end
context 'object storage enabled' do context 'object storage enabled' do
before do
stub_feature_flags(import_export_object_storage: true)
end
context 'when project export is enabled' do context 'when project export is enabled' do
it 'returns 302' do it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project get :download_export, namespace_id: project.namespace, id: project
......
...@@ -103,27 +103,11 @@ FactoryBot.define do ...@@ -103,27 +103,11 @@ FactoryBot.define do
end end
trait :with_export do trait :with_export do
before(:create) do |_project, _evaluator|
allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { false }
allow(Feature).to receive(:enabled?).with('import_export_object_storage') { false }
end
after(:create) do |project, _evaluator| after(:create) do |project, _evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id) ProjectExportWorker.new.perform(project.creator.id, project.id)
end end
end end
trait :with_object_export do
before(:create) do |_project, _evaluator|
allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { true }
allow(Feature).to receive(:enabled?).with('import_export_object_storage') { true }
end
after(:create) do |project, evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
end
trait :broken_storage do trait :broken_storage do
after(:create) do |project| after(:create) do |project|
project.update_column(:repository_storage, 'broken') project.update_column(:repository_storage, 'broken')
......
...@@ -25,7 +25,6 @@ describe 'Import/Export - project export integration test', :js do ...@@ -25,7 +25,6 @@ describe 'Import/Export - project export integration test', :js do
before do before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
end end
after do after do
......
require 'spec_helper'
describe 'Import/Export - project import integration test', :js do
include Select2Helper
let(:user) { create(:user) }
let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
gitlab_sign_in(user)
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
context 'when selecting the namespace' do
let(:user) { create(:admin) }
let!(:namespace) { user.namespace }
let(:project_path) { 'test-project-path' + SecureRandom.hex }
context 'prefilled the path' do
it 'user imports an exported project successfully' do
visit new_project_path
select2(namespace.id, from: '#project_namespace_id')
fill_in :project_path, with: project_path, visible: true
click_import_project_tab
click_link 'GitLab export'
expect(page).to have_content('Import an exported GitLab project')
expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
attach_file('file', file)
click_on 'Import project'
expect(Project.count).to eq(1)
project = Project.last
expect(project).not_to be_nil
expect(project.description).to eq("Foo Bar")
expect(project.issues).not_to be_empty
expect(project.merge_requests).not_to be_empty
expect(project_hook_exists?(project)).to be true
expect(wiki_exists?(project)).to be true
expect(project.import_state.status).to eq('finished')
end
end
context 'path is not prefilled' do
it 'user imports an exported project successfully' do
visit new_project_path
click_import_project_tab
click_link 'GitLab export'
fill_in :path, with: 'test-project-path', visible: true
attach_file('file', file)
expect { click_on 'Import project' }.to change { Project.count }.by(1)
project = Project.last
expect(project).not_to be_nil
expect(page).to have_content("Project 'test-project-path' is being imported")
end
end
end
it 'invalid project' do
project = create(:project, namespace: user.namespace)
visit new_project_path
select2(user.namespace.id, from: '#project_namespace_id')
fill_in :project_path, with: project.name, visible: true
click_import_project_tab
click_link 'GitLab export'
attach_file('file', file)
click_on 'Import project'
page.within('.flash-container') do
expect(page).to have_content('Project could not be imported')
end
end
def wiki_exists?(project)
wiki = ProjectWiki.new(project)
wiki.repository.exists? && !wiki.repository.empty?
end
def project_hook_exists?(project)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
end
end
def click_import_project_tab
find('#import-project-tab').click
end
end
...@@ -8,7 +8,7 @@ describe 'Import/Export - project import integration test', :js do ...@@ -8,7 +8,7 @@ describe 'Import/Export - project import integration test', :js do
let(:export_path) { "#{Dir.tmpdir}/import_file_spec" } let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
before do before do
stub_feature_flags(import_export_object_storage: false) stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
gitlab_sign_in(user) gitlab_sign_in(user)
end end
...@@ -33,7 +33,6 @@ describe 'Import/Export - project import integration test', :js do ...@@ -33,7 +33,6 @@ describe 'Import/Export - project import integration test', :js do
expect(page).to have_content('Import an exported GitLab project') expect(page).to have_content('Import an exported GitLab project')
expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}") expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
expect(Gitlab::ImportExport).to receive(:import_upload_path).with(filename: /\A\h{32}\z/).and_call_original
attach_file('file', file) attach_file('file', file)
click_on 'Import project' click_on 'Import project'
......
require 'spec_helper'
describe 'Import/Export - Namespace export file cleanup', :js do
let(:export_path) { Dir.mktmpdir('namespace_export_file_spec') }
before do
allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
shared_examples_for 'handling project exports on namespace change' do
let!(:old_export_path) { project.export_path }
before do
sign_in(create(:admin))
setup_export_project
end
context 'moving the namespace' do
it 'removes the export file' do
expect(File).to exist(old_export_path)
project.namespace.update!(path: build(:namespace).path)
expect(File).not_to exist(old_export_path)
end
end
context 'deleting the namespace' do
it 'removes the export file' do
expect(File).to exist(old_export_path)
project.namespace.destroy
expect(File).not_to exist(old_export_path)
end
end
end
describe 'legacy storage' do
let(:project) { create(:project, :legacy_storage) }
it_behaves_like 'handling project exports on namespace change'
end
describe 'hashed storage' do
let(:project) { create(:project) }
it_behaves_like 'handling project exports on namespace change'
end
def setup_export_project
visit edit_project_path(project)
expect(page).to have_content('Export project')
find(:link, 'Export project').send_keys(:return)
visit edit_project_path(project)
expect(page).to have_content('Download export')
end
end
...@@ -132,7 +132,6 @@ describe Gitlab::Cleanup::ProjectUploads do ...@@ -132,7 +132,6 @@ describe Gitlab::Cleanup::ProjectUploads do
let!(:path) { File.join(FileUploader.root, orphaned.model.full_path, orphaned.path) } let!(:path) { File.join(FileUploader.root, orphaned.model.full_path, orphaned.path) }
before do before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader) stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.dirname(path)) FileUtils.mkdir_p(File.dirname(path))
...@@ -156,7 +155,6 @@ describe Gitlab::Cleanup::ProjectUploads do ...@@ -156,7 +155,6 @@ describe Gitlab::Cleanup::ProjectUploads do
let!(:new_path) { File.join(FileUploader.root, '-', 'project-lost-found', 'wrong', orphaned.path) } let!(:new_path) { File.join(FileUploader.root, '-', 'project-lost-found', 'wrong', orphaned.path) }
before do before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader) stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.dirname(path)) FileUtils.mkdir_p(File.dirname(path))
......
require 'spec_helper'
describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
let!(:service) { described_class.new }
let!(:project) { create(:project, :with_object_export) }
let(:shared) { project.import_export_shared }
let!(:user) { create(:user) }
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
stub_feature_flags(import_export_object_storage: true)
end
it 'returns if project exported file is not found' do
allow(project).to receive(:export_project_object_exists?).and_return(false)
expect(service).not_to receive(:strategy_execute)
service.execute(user, project)
end
it 'creates a lock file in the export dir' do
allow(service).to receive(:delete_after_export_lock)
service.execute(user, project)
expect(lock_path_exist?).to be_truthy
end
context 'when the method succeeds' do
it 'removes the lock file' do
service.execute(user, project)
expect(lock_path_exist?).to be_falsey
end
end
context 'when the method fails' do
before do
allow(service).to receive(:strategy_execute).and_call_original
end
context 'when validation fails' do
before do
allow(service).to receive(:invalid?).and_return(true)
end
it 'does not create the lock file' do
expect(service).not_to receive(:create_or_update_after_export_lock)
service.execute(user, project)
end
it 'does not execute main logic' do
expect(service).not_to receive(:strategy_execute)
service.execute(user, project)
end
it 'logs validation errors in shared context' do
expect(service).to receive(:log_validation_errors)
service.execute(user, project)
end
end
context 'when an exception is raised' do
it 'removes the lock' do
expect { service.execute(user, project) }.to raise_error(NotImplementedError)
expect(lock_path_exist?).to be_falsey
end
end
end
end
describe '#log_validation_errors' do
it 'add the message to the shared context' do
errors = %w(test_message test_message2)
allow(service).to receive(:invalid?).and_return(true)
allow(service.errors).to receive(:full_messages).and_return(errors)
expect(shared).to receive(:add_error_message).twice.and_call_original
service.execute(user, project)
expect(shared.errors).to eq errors
end
end
describe '#to_json' do
it 'adds the current strategy class to the serialized attributes' do
params = { param1: 1 }
result = params.merge(klass: described_class.to_s).to_json
expect(described_class.new(params).to_json).to eq result
end
end
def lock_path_exist?
File.exist?(described_class.lock_file_path(project))
end
end
...@@ -9,11 +9,10 @@ describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do ...@@ -9,11 +9,10 @@ describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
describe '#execute' do describe '#execute' do
before do before do
allow(service).to receive(:strategy_execute) allow(service).to receive(:strategy_execute)
stub_feature_flags(import_export_object_storage: false)
end end
it 'returns if project exported file is not found' do it 'returns if project exported file is not found' do
allow(project).to receive(:export_project_path).and_return(nil) allow(project).to receive(:export_file_exists?).and_return(false)
expect(service).not_to receive(:strategy_execute) expect(service).not_to receive(:strategy_execute)
......
...@@ -24,34 +24,13 @@ describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do ...@@ -24,34 +24,13 @@ describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
end end
describe '#execute' do describe '#execute' do
context 'without object storage' do
before do
stub_feature_flags(import_export_object_storage: false)
end
it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
expect(project).to receive(:remove_exported_project_file)
strategy.execute(user, project)
end
end
context 'with object storage' do
before do
stub_feature_flags(import_export_object_storage: true)
end
it 'removes the exported project file after the upload' do it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file) allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error) allow(strategy).to receive(:handle_response_error)
expect(project).to receive(:remove_exported_project_file) expect(project).to receive(:remove_exports)
strategy.execute(user, project) strategy.execute(user, project)
end end
end end
end
end end
...@@ -8,8 +8,7 @@ describe Gitlab::ImportExport::AvatarSaver do ...@@ -8,8 +8,7 @@ describe Gitlab::ImportExport::AvatarSaver do
before do before do
FileUtils.mkdir_p("#{shared.export_path}/avatar/") FileUtils.mkdir_p("#{shared.export_path}/avatar/")
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:export_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
end end
after do after do
...@@ -19,7 +18,7 @@ describe Gitlab::ImportExport::AvatarSaver do ...@@ -19,7 +18,7 @@ describe Gitlab::ImportExport::AvatarSaver do
it 'saves a project avatar' do it 'saves a project avatar' do
described_class.new(project: project_with_avatar, shared: shared).save described_class.new(project: project_with_avatar, shared: shared).save
expect(File).to exist("#{shared.export_path}/avatar/dk.png") expect(File).to exist(Dir["#{shared.export_path}/avatar/**/dk.png"].first)
end end
it 'is fine not to have an avatar' do it 'is fine not to have an avatar' do
......
require 'spec_helper'
describe Gitlab::ImportExport::FileImporter do
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
let(:valid_file) { "#{shared.export_path}/valid.json" }
let(:symlink_file) { "#{shared.export_path}/invalid.json" }
let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
let(:subfolder_symlink_file) { "#{shared.export_path}/subfolder/invalid.json" }
let(:evil_symlink_file) { "#{shared.export_path}/.\nevil" }
before do
stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
allow(SecureRandom).to receive(:hex).and_return('abcd')
setup_files
end
after do
FileUtils.rm_rf(storage_path)
end
context 'normal run' do
before do
described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
expect(File.exist?(symlink_file)).to be false
end
it 'removes hidden symlinks in root folder' do
expect(File.exist?(hidden_symlink_file)).to be false
end
it 'removes evil symlinks in root folder' do
expect(File.exist?(evil_symlink_file)).to be false
end
it 'removes symlinks in subfolders' do
expect(File.exist?(subfolder_symlink_file)).to be false
end
it 'does not remove a valid file' do
expect(File.exist?(valid_file)).to be true
end
it 'creates the file in the right subfolder' do
expect(shared.export_path).to include('test/abcd')
end
end
context 'error' do
before do
allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
described_class.import(project: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
expect(File.exist?(symlink_file)).to be false
end
it 'removes hidden symlinks in root folder' do
expect(File.exist?(hidden_symlink_file)).to be false
end
it 'removes symlinks in subfolders' do
expect(File.exist?(subfolder_symlink_file)).to be false
end
it 'does not remove a valid file' do
expect(File.exist?(valid_file)).to be true
end
end
def setup_files
FileUtils.mkdir_p("#{shared.export_path}/subfolder/")
FileUtils.touch(valid_file)
FileUtils.ln_s(valid_file, symlink_file)
FileUtils.ln_s(valid_file, subfolder_symlink_file)
FileUtils.ln_s(valid_file, hidden_symlink_file)
FileUtils.ln_s(valid_file, evil_symlink_file)
end
end
...@@ -2,7 +2,7 @@ require 'spec_helper' ...@@ -2,7 +2,7 @@ require 'spec_helper'
describe Gitlab::ImportExport::FileImporter do describe Gitlab::ImportExport::FileImporter do
let(:shared) { Gitlab::ImportExport::Shared.new(nil) } let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
let(:export_path) { "#{Dir.tmpdir}/file_importer_spec" } let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
let(:valid_file) { "#{shared.export_path}/valid.json" } let(:valid_file) { "#{shared.export_path}/valid.json" }
let(:symlink_file) { "#{shared.export_path}/invalid.json" } let(:symlink_file) { "#{shared.export_path}/invalid.json" }
let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" } let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
...@@ -11,7 +11,9 @@ describe Gitlab::ImportExport::FileImporter do ...@@ -11,7 +11,9 @@ describe Gitlab::ImportExport::FileImporter do
before do before do
stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0) stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true) allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test') allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
allow(SecureRandom).to receive(:hex).and_return('abcd') allow(SecureRandom).to receive(:hex).and_return('abcd')
...@@ -19,12 +21,12 @@ describe Gitlab::ImportExport::FileImporter do ...@@ -19,12 +21,12 @@ describe Gitlab::ImportExport::FileImporter do
end end
after do after do
FileUtils.rm_rf(export_path) FileUtils.rm_rf(storage_path)
end end
context 'normal run' do context 'normal run' do
before do before do
described_class.import(project: nil, archive_file: '', shared: shared) described_class.import(project: build(:project), archive_file: '', shared: shared)
end end
it 'removes symlinks in root folder' do it 'removes symlinks in root folder' do
...@@ -55,7 +57,7 @@ describe Gitlab::ImportExport::FileImporter do ...@@ -55,7 +57,7 @@ describe Gitlab::ImportExport::FileImporter do
context 'error' do context 'error' do
before do before do
allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError) allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
described_class.import(project: nil, archive_file: '', shared: shared) described_class.import(project: build(:project), archive_file: '', shared: shared)
end end
it 'removes symlinks in root folder' do it 'removes symlinks in root folder' do
......
require 'spec_helper'
describe Gitlab::ImportExport::Importer do
let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
let(:project) { create(:project) }
let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(shared.export_path)
ImportExportUpload.create(project: project, import_file: import_file)
end
after do
FileUtils.rm_rf(test_path)
end
describe '#execute' do
it 'succeeds' do
importer.execute
expect(shared.errors).to be_empty
end
it 'extracts the archive' do
expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
importer.execute
end
it 'checks the version' do
expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
importer.execute
end
context 'all restores are executed' do
[
Gitlab::ImportExport::AvatarRestorer,
Gitlab::ImportExport::RepoRestorer,
Gitlab::ImportExport::WikiRestorer,
Gitlab::ImportExport::UploadsRestorer,
Gitlab::ImportExport::LfsRestorer,
Gitlab::ImportExport::StatisticsRestorer
].each do |restorer|
it "calls the #{restorer}" do
fake_restorer = double(restorer.to_s)
expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
importer.execute
end
end
it 'restores the ProjectTree' do
expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
importer.execute
end
it 'removes the import file' do
expect(importer).to receive(:remove_import_file).and_call_original
importer.execute
expect(project.import_export_upload.import_file&.file).to be_nil
end
end
context 'when project successfully restored' do
let!(:existing_project) { create(:project, namespace: user.namespace) }
let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') }
before do
restorers = double(:restorers, all?: true)
allow(subject).to receive(:import_file).and_return(true)
allow(subject).to receive(:check_version!).and_return(true)
allow(subject).to receive(:restorers).and_return(restorers)
allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
end
context 'when import_data' do
context 'has original_path' do
it 'overwrites existing project' do
expect_any_instance_of(::Projects::OverwriteProjectService).to receive(:execute).with(existing_project)
subject.execute
end
end
context 'has not original_path' do
before do
allow(project).to receive(:import_data).and_return(double(data: {}))
end
it 'does not call the overwrite service' do
expect_any_instance_of(::Projects::OverwriteProjectService).not_to receive(:execute).with(existing_project)
subject.execute
end
end
end
end
end
end
...@@ -4,16 +4,18 @@ describe Gitlab::ImportExport::Importer do ...@@ -4,16 +4,18 @@ describe Gitlab::ImportExport::Importer do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" } let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared } let(:shared) { project.import_export_shared }
let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) } let(:project) { create(:project) }
let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
subject(:importer) { described_class.new(project) } subject(:importer) { described_class.new(project) }
before do before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file) allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(shared.export_path) FileUtils.mkdir_p(shared.export_path)
FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path) ImportExportUpload.create(project: project, import_file: import_file)
end end
after do after do
...@@ -64,6 +66,14 @@ describe Gitlab::ImportExport::Importer do ...@@ -64,6 +66,14 @@ describe Gitlab::ImportExport::Importer do
importer.execute importer.execute
end end
it 'removes the import file' do
expect(importer).to receive(:remove_import_file).and_call_original
importer.execute
expect(project.import_export_upload.import_file&.file).to be_nil
end
it 'sets the correct visibility_level when visibility level is a string' do it 'sets the correct visibility_level when visibility level is a string' do
project.create_or_update_import_data( project.create_or_update_import_data(
data: { override_params: { visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s } } data: { override_params: { visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s } }
...@@ -85,7 +95,6 @@ describe Gitlab::ImportExport::Importer do ...@@ -85,7 +95,6 @@ describe Gitlab::ImportExport::Importer do
allow(subject).to receive(:import_file).and_return(true) allow(subject).to receive(:import_file).and_return(true)
allow(subject).to receive(:check_version!).and_return(true) allow(subject).to receive(:check_version!).and_return(true)
allow(subject).to receive(:restorers).and_return(restorers) allow(subject).to receive(:restorers).and_return(restorers)
allow(restorers).to receive(:all?).and_return(true)
allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path })) allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
end end
......
...@@ -18,20 +18,7 @@ describe Gitlab::ImportExport::Saver do ...@@ -18,20 +18,7 @@ describe Gitlab::ImportExport::Saver do
FileUtils.rm_rf(export_path) FileUtils.rm_rf(export_path)
end end
context 'local archive' do
it 'saves the repo to disk' do
stub_feature_flags(import_export_object_storage: false)
subject.save
expect(shared.errors).to be_empty
expect(Dir.empty?(shared.archive_path)).to be false
end
end
context 'object storage' do
it 'saves the repo using object storage' do it 'saves the repo using object storage' do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(ImportExportUploader) stub_uploads_object_storage(ImportExportUploader)
subject.save subject.save
...@@ -39,5 +26,4 @@ describe Gitlab::ImportExport::Saver do ...@@ -39,5 +26,4 @@ describe Gitlab::ImportExport::Saver do
expect(ImportExportUpload.find_by(project: project).export_file.url) expect(ImportExportUpload.find_by(project: project).export_file.url)
.to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*]) .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
end end
end
end end
...@@ -4,6 +4,7 @@ describe Gitlab::ImportExport::UploadsManager do ...@@ -4,6 +4,7 @@ describe Gitlab::ImportExport::UploadsManager do
let(:shared) { project.import_export_shared } let(:shared) { project.import_export_shared }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:project) { create(:project) } let(:project) { create(:project) }
let(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) }
let(:exported_file_path) { "#{shared.export_path}/uploads/#{upload.secret}/#{File.basename(upload.path)}" } let(:exported_file_path) { "#{shared.export_path}/uploads/#{upload.secret}/#{File.basename(upload.path)}" }
subject(:manager) { described_class.new(project: project, shared: shared) } subject(:manager) { described_class.new(project: project, shared: shared) }
...@@ -69,31 +70,10 @@ describe Gitlab::ImportExport::UploadsManager do ...@@ -69,31 +70,10 @@ describe Gitlab::ImportExport::UploadsManager do
end end
end end
end end
context 'using object storage' do
let!(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) }
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
end
it 'saves the file' do
fake_uri = double
expect(fake_uri).to receive(:open).and_return(StringIO.new('File content'))
expect(URI).to receive(:parse).and_return(fake_uri)
manager.save
expect(File.read(exported_file_path)).to eq('File content')
end
end end
describe '#restore' do describe '#restore' do
context 'using object storage' do
before do before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader) stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038')) FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038'))
...@@ -103,10 +83,7 @@ describe Gitlab::ImportExport::UploadsManager do ...@@ -103,10 +83,7 @@ describe Gitlab::ImportExport::UploadsManager do
it 'restores the file' do it 'restores the file' do
manager.restore manager.restore
expect(project.uploads.size).to eq(1) expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
expect(project.uploads.first.build_uploader.filename).to eq('dummy.txt')
end
end
end end
end end
end end
...@@ -8,7 +8,7 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -8,7 +8,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
before do before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/random')) FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/random'))
FileUtils.touch(File.join(shared.export_path, 'uploads/random', "dummy.txt")) FileUtils.touch(File.join(shared.export_path, 'uploads/random', 'dummy.txt'))
end end
after do after do
...@@ -27,9 +27,7 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -27,9 +27,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
it 'copies the uploads to the project path' do it 'copies the uploads to the project path' do
subject.restore subject.restore
uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) } expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
expect(uploads).to include('dummy.txt')
end end
end end
...@@ -45,9 +43,7 @@ describe Gitlab::ImportExport::UploadsRestorer do ...@@ -45,9 +43,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
it 'copies the uploads to the project path' do it 'copies the uploads to the project path' do
subject.restore subject.restore
uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) } expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt')
expect(uploads).to include('dummy.txt')
end end
end end
end end
......
...@@ -7,7 +7,6 @@ describe Gitlab::ImportExport::UploadsSaver do ...@@ -7,7 +7,6 @@ describe Gitlab::ImportExport::UploadsSaver do
let(:shared) { project.import_export_shared } let(:shared) { project.import_export_shared }
before do before do
stub_feature_flags(import_export_object_storage: false)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
end end
......
...@@ -394,12 +394,6 @@ describe Namespace do ...@@ -394,12 +394,6 @@ describe Namespace do
child.destroy child.destroy
end end
end end
it 'removes the exports folder' do
expect(namespace).to receive(:remove_exports!)
namespace.destroy
end
end end
context 'hashed storage' do context 'hashed storage' do
...@@ -414,12 +408,6 @@ describe Namespace do ...@@ -414,12 +408,6 @@ describe Namespace do
expect(File.exist?(deleted_path_in_dir)).to be(false) expect(File.exist?(deleted_path_in_dir)).to be(false)
end end
it 'removes the exports folder' do
expect(namespace).to receive(:remove_exports!)
namespace.destroy
end
end end
end end
...@@ -706,26 +694,6 @@ describe Namespace do ...@@ -706,26 +694,6 @@ describe Namespace do
end end
end end
describe '#remove_exports' do
let(:legacy_project) { create(:project, :with_export, :legacy_storage, namespace: namespace) }
let(:hashed_project) { create(:project, :with_export, namespace: namespace) }
let(:export_path) { Dir.mktmpdir('namespace_remove_exports_spec') }
let(:legacy_export) { legacy_project.export_project_path }
let(:hashed_export) { hashed_project.export_project_path }
it 'removes exports for legacy and hashed projects' do
allow(Gitlab::ImportExport).to receive(:storage_path) { export_path }
expect(File.exist?(legacy_export)).to be_truthy
expect(File.exist?(hashed_export)).to be_truthy
namespace.remove_exports!
expect(File.exist?(legacy_export)).to be_falsy
expect(File.exist?(hashed_export)).to be_falsy
end
end
describe '#full_path_was' do describe '#full_path_was' do
context 'when the group has no parent' do context 'when the group has no parent' do
it 'should return the path was' do it 'should return the path was' do
......
...@@ -2854,73 +2854,12 @@ describe Project do ...@@ -2854,73 +2854,12 @@ describe Project do
end end
describe '#remove_export' do describe '#remove_export' do
let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
let(:project) { create(:project, :with_export) } let(:project) { create(:project, :with_export) }
before do it 'removes the export' do
stub_feature_flags(import_export_object_storage: false)
end
it 'removes the exports directory for the project' do
expect(File.exist?(project.export_path)).to be_truthy
allow(FileUtils).to receive(:rm_rf).and_call_original
expect(FileUtils).to receive(:rm_rf).with(project.export_path).and_call_original
project.remove_exports project.remove_exports
expect(File.exist?(project.export_path)).to be_falsy expect(project.export_file_exists?).to be_falsey
end
it 'is a no-op on legacy projects when there is no namespace' do
export_path = legacy_project.export_path
legacy_project.namespace.delete
legacy_project.reload
expect(FileUtils).not_to receive(:rm_rf).with(export_path)
legacy_project.remove_exports
expect(File.exist?(export_path)).to be_truthy
end
it 'runs on hashed storage projects when there is no namespace' do
export_path = project.export_path
project.namespace.delete
legacy_project.reload
allow(FileUtils).to receive(:rm_rf).and_call_original
expect(FileUtils).to receive(:rm_rf).with(export_path).and_call_original
project.remove_exports
expect(File.exist?(export_path)).to be_falsy
end
it 'is run when the project is destroyed' do
expect(project).to receive(:remove_exports).and_call_original
project.destroy
end
end
describe '#remove_exported_project_file' do
let(:project) { create(:project, :with_export) }
it 'removes the exported project file' do
stub_feature_flags(import_export_object_storage: false)
exported_file = project.export_project_path
expect(File.exist?(exported_file)).to be_truthy
allow(FileUtils).to receive(:rm_rf).and_call_original
expect(FileUtils).to receive(:rm_rf).with(exported_file).and_call_original
project.remove_exported_project_file
expect(File.exist?(exported_file)).to be_falsy
end end
end end
......
...@@ -4,8 +4,8 @@ describe API::ProjectExport do ...@@ -4,8 +4,8 @@ describe API::ProjectExport do
set(:project) { create(:project) } set(:project) { create(:project) }
set(:project_none) { create(:project) } set(:project_none) { create(:project) }
set(:project_started) { create(:project) } set(:project_started) { create(:project) }
set(:project_finished) { create(:project) } let(:project_finished) { create(:project, :with_export) }
set(:project_after_export) { create(:project) } let(:project_after_export) { create(:project, :with_export) }
set(:user) { create(:user) } set(:user) { create(:user) }
set(:admin) { create(:admin) } set(:admin) { create(:admin) }
...@@ -29,13 +29,7 @@ describe API::ProjectExport do ...@@ -29,13 +29,7 @@ describe API::ProjectExport do
# simulate exporting work directory # simulate exporting work directory
FileUtils.mkdir_p File.join(project_started.export_path, 'securerandom-hex') FileUtils.mkdir_p File.join(project_started.export_path, 'securerandom-hex')
# simulate exported
FileUtils.mkdir_p project_finished.export_path
FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz')
# simulate in after export action # simulate in after export action
FileUtils.mkdir_p project_after_export.export_path
FileUtils.touch File.join(project_after_export.export_path, '_export.tar.gz')
FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export) FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export)
end end
...@@ -191,14 +185,11 @@ describe API::ProjectExport do ...@@ -191,14 +185,11 @@ describe API::ProjectExport do
context 'when upload complete' do context 'when upload complete' do
before do before do
FileUtils.rm_rf(project_after_export.export_path) project_after_export.remove_exports
if project_after_export.export_project_object_exists?
upload = project_after_export.import_export_upload
upload.remove_export_file!
upload.save
end end
it 'has removed the export' do
expect(project_after_export.export_file_exists?).to be_falsey
end end
it_behaves_like '404 response' do it_behaves_like '404 response' do
...@@ -273,14 +264,14 @@ describe API::ProjectExport do ...@@ -273,14 +264,14 @@ describe API::ProjectExport do
before do before do
stub_uploads_object_storage(ImportExportUploader) stub_uploads_object_storage(ImportExportUploader)
[project, project_finished, project_after_export].each do |p| project.add_maintainer(user)
p.add_maintainer(user) project_finished.add_maintainer(user)
project_after_export.add_maintainer(user)
upload = ImportExportUpload.new(project: p) upload = ImportExportUpload.new(project: project)
upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz") upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
upload.save! upload.save!
end end
end
it_behaves_like 'get project download by strategy' it_behaves_like 'get project download by strategy'
end end
......
...@@ -7,7 +7,6 @@ describe API::ProjectImport do ...@@ -7,7 +7,6 @@ describe API::ProjectImport do
let(:namespace) { create(:group) } let(:namespace) { create(:group) }
before do before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader) stub_uploads_object_storage(FileUploader)
namespace.add_owner(user) namespace.add_owner(user)
......
...@@ -52,7 +52,7 @@ module ExportFileHelper ...@@ -52,7 +52,7 @@ module ExportFileHelper
# Expands the compressed file for an exported project into +tmpdir+ # Expands the compressed file for an exported project into +tmpdir+
def in_directory_with_expanded_export(project) def in_directory_with_expanded_export(project)
Dir.mktmpdir do |tmpdir| Dir.mktmpdir do |tmpdir|
export_file = project.export_project_path export_file = project.export_file.path
_output, exit_status = Gitlab::Popen.popen(%W{tar -zxf #{export_file} -C #{tmpdir}}) _output, exit_status = Gitlab::Popen.popen(%W{tar -zxf #{export_file} -C #{tmpdir}})
yield(exit_status, tmpdir) yield(exit_status, tmpdir)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment