Commit a790276d authored by Aleksei Lipniagov's avatar Aleksei Lipniagov

Bump WH version, use accelerated Import via API

- Switch to the new WH version including the /authorize route
for the Project Import uploads via API
- Remove the code which supports both accelerated and non-accelerated
uploads for the API import

UI Import is handled separately, and it is not related to this.
parent c146461e
---
title: Enable Workhorse upload acceleration for Project Import uploads via API
merge_request: 26914
author:
type: performance
......@@ -4,41 +4,40 @@ require 'spec_helper'
describe API::ProjectImport do
include ExternalAuthorizationServiceHelpers
include WorkhorseHelpers
let(:export_path) { "#{Dir.tmpdir}/project_export_spec" }
let(:user) { create(:user) }
let(:file) { File.join('spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
let(:namespace) { create(:group) }
let(:file) { File.join('spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
let(:file_name) { 'project_export.tar.gz' }
let(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') }
let(:workhorse_headers) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token } }
let(:file_upload) { fixture_file_upload(file) }
before do
allow_next_instance_of(Gitlab::ImportExport) do |instance|
allow(instance).to receive(:storage_path).and_return(export_path)
end
enable_external_authorization_service_check
stub_licensed_features(external_authorization_service_api_management: true)
namespace.add_owner(user)
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
describe 'POST /projects/import' do
let(:override_params) { { 'external_authorization_classification_label' => 'Hello world' } }
before do
enable_external_authorization_service_check
stub_licensed_features(external_authorization_service_api_management: true)
let(:params) do
{
path: 'test-import',
namespace: namespace.id,
override_params: override_params
}
end
let(:override_params) { { 'external_authorization_classification_label' => 'Hello world' } }
subject do
Sidekiq::Testing.inline! do
post api('/projects/import', user),
params: {
path: 'test-import',
file: fixture_file_upload(file),
namespace: namespace.id,
override_params: override_params
}
upload_archive(file_upload, workhorse_headers, params)
end
end
......@@ -62,4 +61,15 @@ describe API::ProjectImport do
end
end
end
def upload_archive(file, headers = {}, params = {})
workhorse_finalize(
api("/projects/import", user),
method: :post,
file_key: :file,
params: params.merge(file: file_upload),
headers: headers,
send_rewritten_field: true
)
end
end
......@@ -4,11 +4,12 @@ module API
module Helpers
module FileUploadHelpers
def file_is_valid?
params[:file] && params[:file]['tempfile'].respond_to?(:read)
filename = params[:file]&.original_filename
filename && ImportExportUploader::EXTENSION_WHITELIST.include?(File.extname(filename).delete('.'))
end
def validate_file!
render_api_error!('Uploaded file is invalid', 400) unless file_is_valid?
render_api_error!({ error: _('You need to upload a GitLab project export archive (ending in .gz).') }, 422) unless file_is_valid?
end
end
end
......
......@@ -21,10 +21,6 @@ module API
def rate_limiter
::Gitlab::ApplicationRateLimiter
end
def with_workhorse_upload_acceleration?
request.headers[Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER].present?
end
end
before do
......@@ -46,11 +42,7 @@ module API
params do
requires :path, type: String, desc: 'The new project path and name'
# TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
# and mark WH fields as required (instead of optional) after the WH version including
# https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/459
# is deployed and GITLAB_WORKHORSE_VERSION is updated accordingly.
requires :file, types: [::API::Validations::Types::WorkhorseFile, File], desc: 'The project export file to be imported' # rubocop:disable Scalability/FileUploads
requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The project export file to be imported'
optional :name, type: String, desc: 'The name of the project to be imported. Defaults to the path of the project if not provided.'
optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace."
optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it'
......@@ -75,7 +67,7 @@ module API
success Entities::ProjectImportStatus
end
post 'import' do
require_gitlab_workhorse! if with_workhorse_upload_acceleration?
require_gitlab_workhorse!
key = "project_import".to_sym
......@@ -87,27 +79,19 @@ module API
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42437')
validate_file!
namespace = if import_params[:namespace]
find_namespace!(import_params[:namespace])
else
current_user.namespace
end
# TODO: remove the condition after the WH version including
# https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/459
# is deployed and GITLAB_WORKHORSE_VERSION is updated accordingly.
file = if with_workhorse_upload_acceleration?
import_params[:file] || bad_request!('Unable to process project import file')
else
validate_file!
import_params[:file]['tempfile']
end
project_params = {
path: import_params[:path],
namespace_id: namespace.id,
name: import_params[:name],
file: file,
file: import_params[:file],
overwrite: import_params[:overwrite]
}
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment