Commit 8664dd3a authored by Imre Farkas's avatar Imre Farkas

Merge branch 'georgekoltsov/bulk-imports-migration-history' into 'master'

Add Bulk Imports API to view user initiated imports

See merge request gitlab-org/gitlab!64335
parents a7041e34 c10058e6
# frozen_string_literal: true
module BulkImports
class EntitiesFinder
def initialize(user:, bulk_import: nil, status: nil)
@user = user
@bulk_import = bulk_import
@status = status
end
def execute
::BulkImports::Entity
.preload(:failures) # rubocop: disable CodeReuse/ActiveRecord
.by_user_id(user.id)
.then(&method(:filter_by_bulk_import))
.then(&method(:filter_by_status))
end
private
attr_reader :user, :bulk_import, :status
def filter_by_bulk_import(entities)
return entities unless bulk_import
entities.where(bulk_import_id: bulk_import.id) # rubocop: disable CodeReuse/ActiveRecord
end
def filter_by_status(entities)
return entities unless ::BulkImports::Entity.all_human_statuses.include?(status)
entities.with_status(status)
end
end
end
# frozen_string_literal: true
module BulkImports
class ImportsFinder
def initialize(user:, status: nil)
@user = user
@status = status
end
def execute
filter_by_status(user.bulk_imports)
end
private
attr_reader :user, :status
def filter_by_status(imports)
return imports unless BulkImport.all_human_statuses.include?(status)
imports.with_status(status)
end
end
end
......@@ -33,4 +33,8 @@ class BulkImport < ApplicationRecord
transition any => :failed
end
end
def self.all_human_statuses
state_machine.states.map(&:human_name)
end
end
......@@ -48,6 +48,8 @@ class BulkImports::Entity < ApplicationRecord
enum source_type: { group_entity: 0, project_entity: 1 }
scope :by_user_id, ->(user_id) { joins(:bulk_import).where(bulk_imports: { user_id: user_id }) }
state_machine :status, initial: :created do
state :created, value: 0
state :started, value: 1
......@@ -68,6 +70,10 @@ class BulkImports::Entity < ApplicationRecord
end
end
def self.all_human_statuses
state_machine.states.map(&:human_name)
end
def encoded_source_full_path
ERB::Util.url_encode(source_full_path)
end
......
# frozen_string_literal: true
class AddIndexToBulkImportEntitiesOnBulkImportIdAndStatus < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
NEW_INDEX_NAME = 'index_bulk_import_entities_on_bulk_import_id_and_status'
OLD_INDEX_NAME = 'index_bulk_import_entities_on_bulk_import_id'
def up
add_concurrent_index :bulk_import_entities, [:bulk_import_id, :status], name: NEW_INDEX_NAME
remove_concurrent_index_by_name :bulk_import_entities, name: OLD_INDEX_NAME
end
def down
add_concurrent_index :bulk_import_entities, :bulk_import_id, name: OLD_INDEX_NAME
remove_concurrent_index_by_name :bulk_import_entities, name: NEW_INDEX_NAME
end
end
cba36a2e8bedd70f8ccaca47517314d0a3c75a9b8d90715a29919247aa686835
\ No newline at end of file
......@@ -22822,7 +22822,7 @@ CREATE INDEX index_broadcast_message_on_ends_at_and_broadcast_type_and_id ON bro
CREATE INDEX index_bulk_import_configurations_on_bulk_import_id ON bulk_import_configurations USING btree (bulk_import_id);
CREATE INDEX index_bulk_import_entities_on_bulk_import_id ON bulk_import_entities USING btree (bulk_import_id);
CREATE INDEX index_bulk_import_entities_on_bulk_import_id_and_status ON bulk_import_entities USING btree (bulk_import_id, status);
CREATE INDEX index_bulk_import_entities_on_namespace_id ON bulk_import_entities USING btree (namespace_id);
---
stage: Manage
group: Import
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# GitLab Migrations (Bulk Imports) API
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64335) in GitLab 14.1.
With the GitLab Migrations API, you can view the progress of migrations initiated with
[GitLab Group Migration](../user/group/import/index.md).
## List all GitLab migrations
```plaintext
GET /bulk_imports
```
| Attribute | Type | Required | Description |
|:-----------|:--------|:---------|:---------------------------------------|
| `per_page` | integer | no | Number of records to return per page. |
| `page` | integer | no | Page to retrieve. |
| `status` | string | no | Import status. |
The status can be one of the following:
- `created`
- `started`
- `finished`
- `failed`
```shell
curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/bulk_imports?per_page=2&page=1"
```
```json
[
{
"id": 1,
"status": "finished",
"source_type": "gitlab",
"created_at": "2021-06-18T09:45:55.358Z",
"updated_at": "2021-06-18T09:46:27.003Z"
},
{
"id": 2,
"status": "started",
"source_type": "gitlab",
"created_at": "2021-06-18T09:47:36.581Z",
"updated_at": "2021-06-18T09:47:58.286Z"
}
]
```
## List all GitLab migrations' entities
```plaintext
GET /bulk_imports/entities
```
| Attribute | Type | Required | Description |
|:-----------|:--------|:---------|:---------------------------------------|
| `per_page` | integer | no | Number of records to return per page. |
| `page` | integer | no | Page to retrieve. |
| `status` | string | no | Import status. |
The status can be one of the following:
- `created`
- `started`
- `finished`
- `failed`
```shell
curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/bulk_imports/entities?per_page=2&page=1&status=started"
```
```json
[
{
"id": 1,
"bulk_import_id": 1,
"status": "finished",
"source_full_path": "source_group",
"destination_name": "destination_name",
"destination_namespace": "destination_path",
"parent_id": null,
"namespace_id": 1,
"project_id": null,
"created_at": "2021-06-18T09:47:37.390Z",
"updated_at": "2021-06-18T09:47:51.867Z",
"failures": []
},
{
"id": 2,
"bulk_import_id": 2,
"status": "failed",
"source_full_path": "another_group",
"destination_name": "another_name",
"destination_namespace": "another_namespace",
"parent_id": null,
"namespace_id": null,
"project_id": null,
"created_at": "2021-06-24T10:40:20.110Z",
"updated_at": "2021-06-24T10:40:46.590Z",
"failures": [
{
"pipeline_class": "BulkImports::Groups::Pipelines::GroupPipeline",
"pipeline_step": "extractor",
"exception_class": "Exception",
"correlation_id_value": "dfcf583058ed4508e4c7c617bd7f0edd",
"created_at": "2021-06-24T10:40:46.495Z"
}
]
}
]
```
## Get GitLab migration details
```plaintext
GET /bulk_imports/:id
```
```shell
curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/bulk_imports/1"
```
```json
{
"id": 1,
"status": "finished",
"source_type": "gitlab",
"created_at": "2021-06-18T09:45:55.358Z",
"updated_at": "2021-06-18T09:46:27.003Z"
}
```
## List GitLab migration entities
```plaintext
GET /bulk_imports/:id/entities
```
| Attribute | Type | Required | Description |
|:-----------|:--------|:---------|:---------------------------------------|
| `per_page` | integer | no | Number of records to return per page. |
| `page` | integer | no | Page to retrieve. |
| `status` | string | no | Import status. |
The status can be one of the following:
- `created`
- `started`
- `finished`
- `failed`
```shell
curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/bulk_imports/1/entities?per_page=2&page=1&status=finished"
```
```json
[
{
"id": 1,
"status": "finished",
"source_type": "gitlab",
"created_at": "2021-06-18T09:45:55.358Z",
"updated_at": "2021-06-18T09:46:27.003Z"
}
]
```
## Get GitLab migration entity details
```plaintext
GET /bulk_imports/:id/entities/:entity_id
```
```shell
curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/bulk_imports/1/entities/2"
```
```json
{
"id": 1,
"status": "finished",
"source_type": "gitlab",
"created_at": "2021-06-18T09:45:55.358Z",
"updated_at": "2021-06-18T09:46:27.003Z"
}
```
......@@ -152,6 +152,7 @@ module API
mount ::API::Boards
mount ::API::Branches
mount ::API::BroadcastMessages
mount ::API::BulkImports
mount ::API::Ci::Pipelines
mount ::API::Ci::PipelineSchedules
mount ::API::Ci::Runner
......
# frozen_string_literal: true
module API
class BulkImports < ::API::Base
include PaginationParams
feature_category :importers
helpers do
def bulk_imports
@bulk_imports ||= ::BulkImports::ImportsFinder.new(user: current_user, status: params[:status]).execute
end
def bulk_import
@bulk_import ||= bulk_imports.find(params[:import_id])
end
def bulk_import_entities
@bulk_import_entities ||= ::BulkImports::EntitiesFinder.new(user: current_user, bulk_import: bulk_import, status: params[:status]).execute
end
def bulk_import_entity
@bulk_import_entity ||= bulk_import_entities.find(params[:entity_id])
end
end
before { authenticate! }
resource :bulk_imports do
desc 'List all GitLab Migrations' do
detail 'This feature was introduced in GitLab 14.1.'
end
params do
use :pagination
optional :status, type: String, values: BulkImport.all_human_statuses,
desc: 'Return GitLab Migrations with specified status'
end
get do
present paginate(bulk_imports), with: Entities::BulkImport
end
desc "List all GitLab Migrations' entities" do
detail 'This feature was introduced in GitLab 14.1.'
end
params do
use :pagination
optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses,
desc: "Return all GitLab Migrations' entities with specified status"
end
get :entities do
entities = ::BulkImports::EntitiesFinder.new(user: current_user, status: params[:status]).execute
present paginate(entities), with: Entities::BulkImports::Entity
end
desc 'Get GitLab Migration details' do
detail 'This feature was introduced in GitLab 14.1.'
end
params do
requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration"
end
get ':import_id' do
present bulk_import, with: Entities::BulkImport
end
desc "List GitLab Migration entities" do
detail 'This feature was introduced in GitLab 14.1.'
end
params do
requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration"
optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses,
desc: 'Return import entities with specified status'
use :pagination
end
get ':import_id/entities' do
present paginate(bulk_import_entities), with: Entities::BulkImports::Entity
end
desc 'Get GitLab Migration entity details' do
detail 'This feature was introduced in GitLab 14.1.'
end
params do
requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration"
requires :entity_id, type: Integer, desc: "The ID of GitLab Migration entity"
end
get ':import_id/entities/:entity_id' do
present bulk_import_entity, with: Entities::BulkImports::Entity
end
end
end
end
# frozen_string_literal: true
module API
module Entities
class BulkImport < Grape::Entity
expose :id
expose :status_name, as: :status
expose :source_type
expose :created_at
expose :updated_at
end
end
end
# frozen_string_literal: true
module API
module Entities
module BulkImports
class Entity < Grape::Entity
expose :id
expose :bulk_import_id
expose :status_name, as: :status
expose :source_full_path
expose :destination_name
expose :destination_namespace
expose :parent_id
expose :namespace_id
expose :project_id
expose :created_at
expose :updated_at
expose :failures, using: EntityFailure
end
end
end
end
# frozen_string_literal: true
module API
module Entities
module BulkImports
class EntityFailure < Grape::Entity
expose :pipeline_class
expose :pipeline_step
expose :exception_class
expose :correlation_id_value
expose :created_at
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::EntitiesFinder do
let_it_be(:user) { create(:user) }
let_it_be(:user_import_1) { create(:bulk_import, user: user) }
let_it_be(:started_entity_1) { create(:bulk_import_entity, :started, bulk_import: user_import_1) }
let_it_be(:finished_entity_1) { create(:bulk_import_entity, :finished, bulk_import: user_import_1) }
let_it_be(:failed_entity_1) { create(:bulk_import_entity, :failed, bulk_import: user_import_1) }
let_it_be(:user_import_2) { create(:bulk_import, user: user) }
let_it_be(:started_entity_2) { create(:bulk_import_entity, :started, bulk_import: user_import_2) }
let_it_be(:finished_entity_2) { create(:bulk_import_entity, :finished, bulk_import: user_import_2) }
let_it_be(:failed_entity_2) { create(:bulk_import_entity, :failed, bulk_import: user_import_2) }
let_it_be(:not_user_import) { create(:bulk_import) }
let_it_be(:started_entity_3) { create(:bulk_import_entity, :started, bulk_import: not_user_import) }
let_it_be(:finished_entity_3) { create(:bulk_import_entity, :finished, bulk_import: not_user_import) }
let_it_be(:failed_entity_3) { create(:bulk_import_entity, :failed, bulk_import: not_user_import) }
subject { described_class.new(user: user) }
describe '#execute' do
it 'returns a list of import entities associated with user' do
expect(subject.execute)
.to contain_exactly(
started_entity_1, finished_entity_1, failed_entity_1,
started_entity_2, finished_entity_2, failed_entity_2
)
end
context 'when bulk import is specified' do
subject { described_class.new(user: user, bulk_import: user_import_1) }
it 'returns a list of import entities filtered by bulk import' do
expect(subject.execute)
.to contain_exactly(
started_entity_1, finished_entity_1, failed_entity_1
)
end
context 'when specified import is not associated with user' do
subject { described_class.new(user: user, bulk_import: not_user_import) }
it 'does not return entities' do
expect(subject.execute).to be_empty
end
end
end
context 'when status is specified' do
subject { described_class.new(user: user, status: 'failed') }
it 'returns a list of import entities filtered by status' do
expect(subject.execute)
.to contain_exactly(
failed_entity_1, failed_entity_2
)
end
context 'when invalid status is specified' do
subject { described_class.new(user: user, status: 'invalid') }
it 'does not filter entities by status' do
expect(subject.execute)
.to contain_exactly(
started_entity_1, finished_entity_1, failed_entity_1,
started_entity_2, finished_entity_2, failed_entity_2
)
end
end
end
context 'when bulk import and status are specified' do
subject { described_class.new(user: user, bulk_import: user_import_2, status: 'finished') }
it 'returns matched import entities' do
expect(subject.execute).to contain_exactly(finished_entity_2)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::ImportsFinder do
let_it_be(:user) { create(:user) }
let_it_be(:started_import) { create(:bulk_import, :started, user: user) }
let_it_be(:finished_import) { create(:bulk_import, :finished, user: user) }
let_it_be(:not_user_import) { create(:bulk_import) }
subject { described_class.new(user: user) }
describe '#execute' do
it 'returns a list of imports associated with user' do
expect(subject.execute).to contain_exactly(started_import, finished_import)
end
context 'when status is specified' do
subject { described_class.new(user: user, status: 'started') }
it 'returns a list of import entities filtered by status' do
expect(subject.execute).to contain_exactly(started_import)
end
context 'when invalid status is specified' do
subject { described_class.new(user: user, status: 'invalid') }
it 'does not filter entities by status' do
expect(subject.execute).to contain_exactly(started_import, finished_import)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::Entities::BulkImport do
let_it_be(:import) { create(:bulk_import) }
subject { described_class.new(import).as_json }
it 'has the correct attributes' do
expect(subject).to include(
:id,
:status,
:source_type,
:created_at,
:updated_at
)
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::Entities::BulkImports::EntityFailure do
let_it_be(:failure) { create(:bulk_import_failure) }
subject { described_class.new(failure).as_json }
it 'has the correct attributes' do
expect(subject).to include(
:pipeline_class,
:pipeline_step,
:exception_class,
:correlation_id_value,
:created_at
)
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::Entities::BulkImports::Entity do
let_it_be(:entity) { create(:bulk_import_entity) }
subject { described_class.new(entity).as_json }
it 'has the correct attributes' do
expect(subject).to include(
:id,
:bulk_import_id,
:status,
:source_full_path,
:destination_name,
:destination_namespace,
:parent_id,
:namespace_id,
:project_id,
:created_at,
:updated_at,
:failures
)
end
end
......@@ -15,4 +15,10 @@ RSpec.describe BulkImport, type: :model do
it { is_expected.to define_enum_for(:source_type).with_values(%i[gitlab]) }
end
describe '.all_human_statuses' do
it 'returns all human readable entity statuses' do
expect(described_class.all_human_statuses).to contain_exactly('created', 'started', 'finished', 'failed')
end
end
end
......@@ -134,4 +134,24 @@ RSpec.describe BulkImports::Entity, type: :model do
expect(entity.encoded_source_full_path).to eq(expected)
end
end
describe 'scopes' do
describe '.by_user_id' do
it 'returns entities associated with specified user' do
user = create(:user)
import = create(:bulk_import, user: user)
entity_1 = create(:bulk_import_entity, bulk_import: import)
entity_2 = create(:bulk_import_entity, bulk_import: import)
create(:bulk_import_entity)
expect(described_class.by_user_id(user.id)).to contain_exactly(entity_1, entity_2)
end
end
end
describe '.all_human_statuses' do
it 'returns all human readable entity statuses' do
expect(described_class.all_human_statuses).to contain_exactly('created', 'started', 'finished', 'failed')
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::BulkImports do
let_it_be(:user) { create(:user) }
let_it_be(:import_1) { create(:bulk_import, user: user) }
let_it_be(:import_2) { create(:bulk_import, user: user) }
let_it_be(:entity_1) { create(:bulk_import_entity, bulk_import: import_1) }
let_it_be(:entity_2) { create(:bulk_import_entity, bulk_import: import_1) }
let_it_be(:entity_3) { create(:bulk_import_entity, bulk_import: import_2) }
let_it_be(:failure_3) { create(:bulk_import_failure, entity: entity_3) }
describe 'GET /bulk_imports' do
it 'returns a list of bulk imports authored by the user' do
get api('/bulk_imports', user)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response.pluck('id')).to contain_exactly(import_1.id, import_2.id)
end
end
describe 'GET /bulk_imports/entities' do
it 'returns a list of all import entities authored by the user' do
get api('/bulk_imports/entities', user)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response.pluck('id')).to contain_exactly(entity_1.id, entity_2.id, entity_3.id)
end
end
describe 'GET /bulk_imports/:id' do
it 'returns specified bulk import' do
get api("/bulk_imports/#{import_1.id}", user)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['id']).to eq(import_1.id)
end
end
describe 'GET /bulk_imports/:id/entities' do
it 'returns specified bulk import entities with failures' do
get api("/bulk_imports/#{import_2.id}/entities", user)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response.pluck('id')).to contain_exactly(entity_3.id)
expect(json_response.first['failures'].first['exception_class']).to eq(failure_3.exception_class)
end
end
describe 'GET /bulk_imports/:id/entities/:entity_id' do
it 'returns specified bulk import entity' do
get api("/bulk_imports/#{import_1.id}/entities/#{entity_2.id}", user)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['id']).to eq(entity_2.id)
end
end
context 'when user is unauthenticated' do
it 'returns 401' do
get api('/bulk_imports', nil)
expect(response).to have_gitlab_http_status(:unauthorized)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment