Commit 6bf2c12d authored by George Koltsov's avatar George Koltsov Committed by Ash McKenzie

Add Group Import API endpoint

- Add POST '/groups/import' API endpoint
- Triggers Group Import functionality similar to Project Import
- Imports a group structure along with it's relations
parent f503bfec
---
title: Add Group Import API endpoint & update Group Import/Export documentation
merge_request: 20353
author:
type: added
# Group Import/Export API
> Introduced in GitLab 12.8 as an experimental feature. May change in future releases.
Group Import/Export functionality allows to export group structure and import it at a new location.
Used in combination with [Project Import/Export](project_import_export.md) it allows you to preserve connections with group level relations
(e.g. a connection between a project issue and group epic).
Group Export includes:
1. Group Milestones
1. Group Boards
1. Group Labels
1. Group Badges
1. Group Members
1. Sub-groups (each sub-group includes all data above)
## Schedule new export
Start a new group export.
```text
POST /groups/:id/export
```
| Attribute | Type | Required | Description |
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | ID of the groupd owned by the authenticated user |
```shell
curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/export
```
```json
{
"message": "202 Accepted"
}
```
## Export download
Download the finished export.
```text
GET /groups/:id/export/download
```
| Attribute | Type | Required | Description |
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | ID of the group owned by the authenticated user |
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" --remote-header-name --remote-name https://gitlab.example.com/api/v4/groups/1/export/download
```
```shell
ls *export.tar.gz
2020-12-05_22-11-148_namespace_export.tar.gz
```
Time spent on exporting a group may vary depending on a size of the group. Export download endpoint will return exported archive once it is available. 404 is returned otherwise.
## Import a file
```text
POST /groups/import
```
| Attribute | Type | Required | Description |
| --------- | -------------- | -------- | ---------------------------------------- |
| `name` | string | yes | The name of the group to be imported |
| `path` | string | yes | Name and path for new group |
| `file` | string | yes | The file to be uploaded |
| `parent_id` | integer | no | ID of a parent group that the group will be imported into. Defaults to the current user's namespace if not provided. |
To upload a file from your file system, use the `--form` argument. This causes
cURL to post data using the header `Content-Type: multipart/form-data`.
The `file=` parameter must point to a file on your file system and be preceded
by `@`. For example:
```shell
curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "name=imported-group" --form "path=imported-group" --form "file=@/path/to/file" https://gitlab.example.com/api/v4/groups/import
```
......@@ -875,3 +875,7 @@ And to switch pages add:
## Group badges
Read more in the [Group Badges](group_badges.md) documentation.
## Group Import/Export
Read more in the [Group Import/Export](group_import_export.md) documentation.
......@@ -130,6 +130,7 @@ module API
mount ::API::GroupBoards
mount ::API::GroupClusters
mount ::API::GroupExport
mount ::API::GroupImport
mount ::API::GroupLabels
mount ::API::GroupMilestones
mount ::API::Groups
......
# frozen_string_literal: true
module API
class GroupImport < Grape::API
MAXIMUM_FILE_SIZE = 50.megabytes.freeze
helpers do
def authorize_create_group!
parent_group = find_group!(params[:parent_id]) if params[:parent_id].present?
if parent_group
authorize! :create_subgroup, parent_group
else
authorize! :create_group
end
end
end
resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Workhorse authorize the group import upload' do
detail 'This feature was introduced in GitLab 12.8'
end
post 'import/authorize' do
require_gitlab_workhorse!
Gitlab::Workhorse.verify_api_request!(headers)
status 200
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
ImportExportUploader.workhorse_authorize(has_length: false, maximum_size: MAXIMUM_FILE_SIZE)
end
desc 'Create a new group import' do
detail 'This feature was introduced in GitLab 12.8'
success Entities::Group
end
params do
requires :path, type: String, desc: 'Group path'
requires :name, type: String, desc: 'Group name'
optional :parent_id, type: Integer, desc: "The ID of the parent group that the group will be imported into. Defaults to the current user's namespace."
optional 'file.path', type: String, desc: 'Path to locally stored body (generated by Workhorse)'
optional 'file.name', type: String, desc: 'Real filename as send in Content-Disposition (generated by Workhorse)'
optional 'file.type', type: String, desc: 'Real content type as send in Content-Type (generated by Workhorse)'
optional 'file.size', type: Integer, desc: 'Real size of file (generated by Workhorse)'
optional 'file.md5', type: String, desc: 'MD5 checksum of the file (generated by Workhorse)'
optional 'file.sha1', type: String, desc: 'SHA1 checksum of the file (generated by Workhorse)'
optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)'
end
post 'import' do
authorize_create_group!
require_gitlab_workhorse!
uploaded_file = UploadedFile.from_params(params, :file, ImportExportUploader.workhorse_local_upload_path)
bad_request!('Unable to process group import file') unless uploaded_file
group_params = {
path: params[:path],
name: params[:name],
parent_id: params[:parent_id],
import_export_upload: ImportExportUpload.new(import_file: uploaded_file)
}
group = ::Groups::CreateService.new(current_user, group_params).execute
if group.persisted?
GroupImportWorker.perform_async(current_user.id, group.id)
accepted!
else
render_api_error!("Failed to save group #{group.errors.messages}", 400)
end
end
end
end
end
# frozen_string_literal: true
module API
module Helpers
module FileUploadHelpers
def file_is_valid?
params[:file] && params[:file]['tempfile'].respond_to?(:read)
end
def validate_file!
render_api_error!('Uploaded file is invalid', 400) unless file_is_valid?
end
end
end
end
......@@ -5,20 +5,13 @@ module API
include PaginationParams
helpers Helpers::ProjectsHelpers
helpers Helpers::FileUploadHelpers
helpers do
def import_params
declared_params(include_missing: false)
end
def file_is_valid?
import_params[:file] && import_params[:file]['tempfile'].respond_to?(:read)
end
def validate_file!
render_api_error!('The file is invalid', 400) unless file_is_valid?
end
def throttled?(key, scope)
rate_limiter.throttled?(key, scope: scope)
end
......
# frozen_string_literal: true
require 'spec_helper'
describe API::GroupImport do
include WorkhorseHelpers
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let(:path) { '/groups/import' }
let(:file) { File.join('spec', 'fixtures', 'group_export.tar.gz') }
let(:export_path) { "#{Dir.tmpdir}/group_export_spec" }
let(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') }
let(:workhorse_header) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token } }
before do
allow_next_instance_of(Gitlab::ImportExport) do |import_export|
expect(import_export).to receive(:storage_path).and_return(export_path)
end
stub_uploads_object_storage(ImportExportUploader)
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
describe 'POST /groups/import' do
let(:file_upload) { fixture_file_upload(file) }
let(:params) do
{
path: 'test-import-group',
name: 'test-import-group',
file: fixture_file_upload(file)
}
end
subject { post api('/groups/import', user), params: params, headers: workhorse_header }
shared_examples 'when all params are correct' do
context 'when user is authorized to create new group' do
it 'creates new group and accepts request' do
subject
expect(response).to have_gitlab_http_status(202)
end
context 'when importing to a parent group' do
before do
group.add_owner(user)
end
it 'creates new group and accepts request' do
params[:parent_id] = group.id
subject
expect(response).to have_gitlab_http_status(202)
expect(group.children.count).to eq(1)
end
context 'when parent group is invalid' do
it 'returns 404 and does not create new group' do
params[:parent_id] = 99999
expect { subject }.not_to change { Group.count }
expect(response).to have_gitlab_http_status(404)
expect(json_response['message']).to eq('404 Group Not Found')
end
context 'when user is not an owner of parent group' do
it 'returns 403 Forbidden HTTP status' do
params[:parent_id] = create(:group).id
subject
expect(response).to have_gitlab_http_status(403)
expect(json_response['message']).to eq('403 Forbidden')
end
end
end
end
context 'when group creation failed' do
before do
allow_next_instance_of(Group) do |group|
allow(group).to receive(:persisted?).and_return(false)
end
end
it 'returns 400 HTTP status' do
subject
expect(response).to have_gitlab_http_status(400)
end
end
end
context 'when user is not authorized to create new group' do
let(:user) { create(:user, can_create_group: false) }
it 'forbids the request' do
subject
expect(response).to have_gitlab_http_status(403)
end
end
end
shared_examples 'when some params are missing' do
context 'when required params are missing' do
shared_examples 'missing parameter' do |params, error_message|
it 'returns 400 HTTP status' do
params[:file] = file_upload
expect do
post api('/groups/import', user), params: params, headers: workhorse_header
end.not_to change { Group.count }.from(1)
expect(response).to have_gitlab_http_status(400)
expect(json_response['error']).to eq(error_message)
end
end
include_examples 'missing parameter', { name: 'test' }, 'path is missing'
include_examples 'missing parameter', { path: 'test' }, 'name is missing'
end
end
context 'with object storage disabled' do
before do
stub_uploads_object_storage(ImportExportUploader, enabled: false)
end
context 'without a file from workhorse' do
it 'rejects the request' do
subject
expect(response).to have_gitlab_http_status(:bad_request)
end
end
context 'without a workhorse header' do
it 'rejects request without a workhorse header' do
post api('/groups/import', user), params: params
expect(response).to have_gitlab_http_status(:forbidden)
end
end
context 'when params from workhorse are correct' do
let(:params) do
{
path: 'test-import-group',
name: 'test-import-group',
'file.path' => file_upload.path,
'file.name' => file_upload.original_filename
}
end
include_examples 'when all params are correct'
include_examples 'when some params are missing'
end
it "doesn't attempt to migrate file to object storage" do
expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
subject
end
end
context 'with object storage enabled' do
before do
stub_uploads_object_storage(ImportExportUploader, enabled: true)
allow(ImportExportUploader).to receive(:workhorse_upload_path).and_return('/')
end
context 'with direct upload enabled' do
let(:file_name) { 'group_export.tar.gz' }
let!(:fog_connection) do
stub_uploads_object_storage(ImportExportUploader, direct_upload: true)
end
let(:tmp_object) do
fog_connection.directories.new(key: 'uploads').files.create(
key: "tmp/uploads/#{file_name}",
body: file_upload
)
end
let(:fog_file) { fog_to_uploaded_file(tmp_object) }
let(:params) do
{
path: 'test-import-group',
name: 'test-import-group',
file: fog_file,
'file.remote_id' => file_name,
'file.size' => fog_file.size
}
end
it 'accepts the request and stores the file' do
expect { subject }.to change { Group.count }.by(1)
expect(response).to have_gitlab_http_status(:accepted)
end
include_examples 'when all params are correct'
include_examples 'when some params are missing'
end
end
end
describe 'POST /groups/import/authorize' do
subject { post api('/groups/import/authorize', user), headers: workhorse_header }
it 'authorizes importing group with workhorse header' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
end
it 'rejects requests that bypassed gitlab-workhorse' do
workhorse_header.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
context 'when using remote storage' do
context 'when direct upload is enabled' do
before do
stub_uploads_object_storage(ImportExportUploader, enabled: true, direct_upload: true)
end
it 'responds with status 200, location of file remote store and object details' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
expect(json_response).not_to have_key('TempPath')
expect(json_response['RemoteObject']).to have_key('ID')
expect(json_response['RemoteObject']).to have_key('GetURL')
expect(json_response['RemoteObject']).to have_key('StoreURL')
expect(json_response['RemoteObject']).to have_key('DeleteURL')
expect(json_response['RemoteObject']).to have_key('MultipartUpload')
end
end
context 'when direct upload is disabled' do
before do
stub_uploads_object_storage(ImportExportUploader, enabled: true, direct_upload: false)
end
it 'handles as a local file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
expect(json_response['TempPath']).to eq(ImportExportUploader.workhorse_local_upload_path)
expect(json_response['RemoteObject']).to be_nil
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment