Skip to content

G
gitlab-ce
Commit b18aa67e authored by Natalia Tepluhina's avatar Natalia Tepluhina
Browse files
Options

Merge branch 'xanf-simplify-import-entities-graphql' into 'master' 

Simplify management of local state in import_groups app

See merge request gitlab-org/gitlab!60615
parents fef65601 9dc3c797
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 422 additions and 268 deletions
app/assets/javascripts/import_entities/import_groups/components/import_table_row.vue
View file @ b18aa67e
......@@ -72,11 +72,11 @@ export default {
},
isAlreadyImported() {
return this.group.status !== STATUSES.NONE;
return this.group.progress.status !== STATUSES.NONE;
},
isFinished() {
return this.group.status === STATUSES.FINISHED;
return this.group.progress.status === STATUSES.FINISHED;
},
fullPath() {
......@@ -165,7 +165,7 @@ export default {
</div>
</td>
<td class="gl-p-4 gl-white-space-nowrap">
<import-status :status="group.status" />
<import-status :status="group.progress.status" />
</td>
<td class="gl-p-4">
<gl-button
......
app/assets/javascripts/import_entities/import_groups/graphql/client_factory.js
View file @ b18aa67e
......@@ -4,40 +4,82 @@ import axios from '~/lib/utils/axios_utils';
import { parseIntPagination, normalizeHeaders } from '~/lib/utils/common_utils';
import { s__ } from '~/locale';
import { STATUSES } from '../../constants';
import bulkImportSourceGroupItemFragment from './fragments/bulk_import_source_group_item.fragment.graphql';
import setImportProgressMutation from './mutations/set_import_progress.mutation.graphql';
import updateImportStatusMutation from './mutations/update_import_status.mutation.graphql';
import availableNamespacesQuery from './queries/available_namespaces.query.graphql';
import bulkImportSourceGroupQuery from './queries/bulk_import_source_group.query.graphql';
import { SourceGroupsManager } from './services/source_groups_manager';
import { StatusPoller } from './services/status_poller';
import typeDefs from './typedefs.graphql';
export const clientTypenames = {
BulkImportSourceGroupConnection: 'ClientBulkImportSourceGroupConnection',
BulkImportSourceGroup: 'ClientBulkImportSourceGroup',
AvailableNamespace: 'ClientAvailableNamespace',
BulkImportPageInfo: 'ClientBulkImportPageInfo',
BulkImportTarget: 'ClientBulkImportTarget',
BulkImportProgress: 'ClientBulkImportProgress',
};
export function createResolvers({ endpoints, sourceUrl, GroupsManager = SourceGroupsManager }) {
let statusPoller;
function makeGroup(data) {
const result = {
__typename: clientTypenames.BulkImportSourceGroup,
...data,
};
const NESTED_OBJECT_FIELDS = {
import_target: clientTypenames.BulkImportTarget,
progress: clientTypenames.BulkImportProgress,
};
let sourceGroupManager;
const getGroupsManager = (client) => {
if (!sourceGroupManager) {
sourceGroupManager = new GroupsManager({ client, sourceUrl });
Object.entries(NESTED_OBJECT_FIELDS).forEach(([field, type]) => {
if (!data[field]) {
return;
}
return sourceGroupManager;
};
result[field] = {
__typename: type,
...data[field],
};
});
return result;
}
const localProgressId = (id) => `not-started-${id}`;
export function createResolvers({ endpoints, sourceUrl, GroupsManager = SourceGroupsManager }) {
const groupsManager = new GroupsManager({
sourceUrl,
});
let statusPoller;
return {
Query: {
async bulkImportSourceGroup(_, { id }, { client, getCacheKey }) {
return client.readFragment({
fragment: bulkImportSourceGroupItemFragment,
fragmentName: 'BulkImportSourceGroupItem',
id: getCacheKey({
__typename: clientTypenames.BulkImportSourceGroup,
id,
}),
});
},
async bulkImportSourceGroups(_, vars, { client }) {
if (!statusPoller) {
statusPoller = new StatusPoller({
groupManager: getGroupsManager(client),
updateImportStatus: ({ id, status_name: status }) =>
client.mutate({
mutation: updateImportStatusMutation,
variables: { id, status },
}),
pollPath: endpoints.jobs,
});
statusPoller.startPolling();
}
const groupsManager = getGroupsManager(client);
return Promise.all([
axios.get(endpoints.status, {
params: {
......@@ -59,19 +101,20 @@ export function createResolvers({ endpoints, sourceUrl, GroupsManager = SourceGr
return {
__typename: clientTypenames.BulkImportSourceGroupConnection,
nodes: data.importable_data.map((group) => {
const cachedImportState = groupsManager.getImportStateFromStorageByGroupId(
group.id,
);
const { jobId, importState: cachedImportState } =
groupsManager.getImportStateFromStorageByGroupId(group.id) ?? {};
return {
__typename: clientTypenames.BulkImportSourceGroup,
return makeGroup({
...group,
status: cachedImportState?.status ?? STATUSES.NONE,
progress: {
id: jobId ?? localProgressId(group.id),
status: cachedImportState?.status ?? STATUSES.NONE,
},
import_target: cachedImportState?.importTarget ?? {
new_name: group.full_path,
target_namespace: availableNamespaces[0]?.full_path ?? '',
},
};
});
}),
pageInfo: {
__typename: clientTypenames.BulkImportPageInfo,
......@@ -91,26 +134,65 @@ export function createResolvers({ endpoints, sourceUrl, GroupsManager = SourceGr
),
},
Mutation: {
setTargetNamespace(_, { targetNamespace, sourceGroupId }, { client }) {
getGroupsManager(client).updateById(sourceGroupId, (sourceGroup) => {
// eslint-disable-next-line no-param-reassign
sourceGroup.import_target.target_namespace = targetNamespace;
setTargetNamespace: (_, { targetNamespace, sourceGroupId }) =>
makeGroup({
id: sourceGroupId,
import_target: {
target_namespace: targetNamespace,
},
}),
setNewName: (_, { newName, sourceGroupId }) =>
makeGroup({
id: sourceGroupId,
import_target: {
new_name: newName,
},
}),
async setImportProgress(_, { sourceGroupId, status, jobId }) {
if (jobId) {
groupsManager.saveImportState(jobId, { status });
}
return makeGroup({
id: sourceGroupId,
progress: {
id: jobId ?? localProgressId(sourceGroupId),
status,
},
});
},
setNewName(_, { newName, sourceGroupId }, { client }) {
getGroupsManager(client).updateById(sourceGroupId, (sourceGroup) => {
// eslint-disable-next-line no-param-reassign
sourceGroup.import_target.new_name = newName;
});
async updateImportStatus(_, { id, status }) {
groupsManager.saveImportState(id, { status });
return {
__typename: clientTypenames.BulkImportProgress,
id,
status,
};
},
async importGroup(_, { sourceGroupId }, { client }) {
const groupManager = getGroupsManager(client);
const group = groupManager.findById(sourceGroupId);
groupManager.setImportStatus(group, STATUSES.SCHEDULING);
try {
const response = await axios.post(endpoints.createBulkImport, {
const {
data: { bulkImportSourceGroup: group },
} = await client.query({
query: bulkImportSourceGroupQuery,
variables: { id: sourceGroupId },
});
const GROUP_BEING_SCHEDULED = makeGroup({
id: sourceGroupId,
progress: {
id: localProgressId(sourceGroupId),
status: STATUSES.SCHEDULING,
},
});
const defaultErrorMessage = s__('BulkImport|Importing the group failed');
axios
.post(endpoints.createBulkImport, {
bulk_import: [
{
source_type: 'group_entity',
......@@ -119,18 +201,38 @@ export function createResolvers({ endpoints, sourceUrl, GroupsManager = SourceGr
destination_name: group.import_target.new_name,
},
],
});
groupManager.startImport({ group, importId: response.data.id });
} catch (e) {
const message = e?.response?.data?.error ?? s__('BulkImport|Importing the group failed');
createFlash({ message });
groupManager.setImportStatus(group, STATUSES.NONE);
throw e;
}
})
.then(({ data: { id: jobId } }) => {
groupsManager.saveImportState(jobId, {
id: group.id,
importTarget: group.import_target,
status: STATUSES.CREATED,
});
return { status: STATUSES.CREATED, jobId };
})
.catch((e) => {
const message = e?.response?.data?.error ?? defaultErrorMessage;
createFlash({ message });
return { status: STATUSES.NONE };
})
.then((newStatus) =>
client.mutate({
mutation: setImportProgressMutation,
variables: { sourceGroupId, ...newStatus },
}),
)
.catch(() => createFlash({ message: defaultErrorMessage }));
return GROUP_BEING_SCHEDULED;
},
},
};
}
export const createApolloClient = ({ sourceUrl, endpoints }) =>
createDefaultClient(createResolvers({ sourceUrl, endpoints }), { assumeImmutableResults: true });
createDefaultClient(
createResolvers({ sourceUrl, endpoints }),
{ assumeImmutableResults: true },
typeDefs,
);
app/assets/javascripts/import_entities/import_groups/graphql/fragments/bulk_import_source_group_item.fragment.graphql
View file @ b18aa67e
#import "./bulk_import_source_group_progress.fragment.graphql"
fragment BulkImportSourceGroupItem on ClientBulkImportSourceGroup {
id
web_url
full_path
full_name
status
import_target
progress {
...BulkImportSourceGroupProgress
}
import_target {
target_namespace
new_name
}
}
app/assets/javascripts/import_entities/import_groups/graphql/mutations/import_group.mutation.graphql
View file @ b18aa67e
mutation importGroup($sourceGroupId: String!) {
importGroup(sourceGroupId: $sourceGroupId) @client
importGroup(sourceGroupId: $sourceGroupId) @client {
id
progress {
id
status
}
}
}
app/assets/javascripts/import_entities/import_groups/graphql/mutations/set_import_progress.mutation.graphql 0 → 100644
View file @ b18aa67e
mutation setImportProgress($status: String!, $sourceGroupId: String!, $jobId: String) {
setImportProgress(status: $status, sourceGroupId: $sourceGroupId, jobId: $jobId) @client {
id
progress {
id
status
}
}
}
app/assets/javascripts/import_entities/import_groups/graphql/mutations/set_new_name.mutation.graphql
View file @ b18aa67e
mutation setNewName($newName: String!, $sourceGroupId: String!) {
setNewName(newName: $newName, sourceGroupId: $sourceGroupId) @client
setNewName(newName: $newName, sourceGroupId: $sourceGroupId) @client {
id
import_target {
new_name
}
}
}
app/assets/javascripts/import_entities/import_groups/graphql/mutations/set_target_namespace.mutation.graphql
View file @ b18aa67e
mutation setTargetNamespace($targetNamespace: String!, $sourceGroupId: String!) {
setTargetNamespace(targetNamespace: $targetNamespace, sourceGroupId: $sourceGroupId) @client
setTargetNamespace(targetNamespace: $targetNamespace, sourceGroupId: $sourceGroupId) @client {
id
import_target {
target_namespace
}
}
}
app/assets/javascripts/import_entities/import_groups/graphql/mutations/update_import_status.mutation.graphql 0 → 100644
View file @ b18aa67e
mutation updateImportStatus($status: String!, $id: String!) {
updateImportStatus(status: $status, id: $id) @client {
id
status
}
}
app/assets/javascripts/import_entities/import_groups/graphql/queries/bulk_import_source_group.query.graphql 0 → 100644
View file @ b18aa67e
#import "../fragments/bulk_import_source_group_item.fragment.graphql"
query bulkImportSourceGroup($id: ID!) {
bulkImportSourceGroup(id: $id) @client {
...BulkImportSourceGroupItem
}
}
app/assets/javascripts/import_entities/import_groups/graphql/services/source_groups_manager.js
View file @ b18aa67e
import { defaultDataIdFromObject } from 'apollo-cache-inmemory';
import produce from 'immer';
import { debounce, merge } from 'lodash';
import { STATUSES } from '../../../constants';
import ImportSourceGroupFragment from '../fragments/bulk_import_source_group_item.fragment.graphql';
function extractTypeConditionFromFragment(fragment) {
return fragment.definitions[0]?.typeCondition.name.value;
}
function generateGroupId(id) {
return defaultDataIdFromObject({
__typename: extractTypeConditionFromFragment(ImportSourceGroupFragment),
id,
});
}
export const KEY = 'gl-bulk-imports-import-state';
export const DEBOUNCE_INTERVAL = 200;
export class SourceGroupsManager {
constructor({ client, sourceUrl, storage = window.localStorage }) {
this.client = client;
constructor({ sourceUrl, storage = window.localStorage }) {
this.sourceUrl = sourceUrl;
this.storage = storage;
......@@ -35,45 +19,30 @@ export class SourceGroupsManager {
}
}
findById(id) {
const cacheId = generateGroupId(id);
return this.client.readFragment({ fragment: ImportSourceGroupFragment, id: cacheId });
}
update(group, fn) {
this.client.writeFragment({
fragment: ImportSourceGroupFragment,
id: generateGroupId(group.id),
data: produce(group, fn),
});
}
saveImportState(importId, group) {
const key = this.getStorageKey(importId);
const oldState = this.importStates[key] ?? {};
updateById(id, fn) {
const group = this.findById(id);
this.update(group, fn);
}
if (!oldState.id && !group.id) {
return;
}
saveImportState(importId, group) {
this.importStates[this.getStorageKey(importId)] = {
id: group.id,
importTarget: group.import_target,
this.importStates[key] = {
...oldState,
...group,
status: group.status,
};
this.saveImportStatesToStorage();
}
getImportStateFromStorage(importId) {
return this.importStates[this.getStorageKey(importId)];
}
getImportStateFromStorageByGroupId(groupId) {
const PREFIX = this.getStorageKey('');
const [, importState] =
const [jobId, importState] =
Object.entries(this.importStates).find(
([key, group]) => key.startsWith(PREFIX) && group.id === groupId,
) ?? [];
return importState;
return { jobId, importState };
}
getStorageKey(importId) {
......@@ -91,34 +60,4 @@ export class SourceGroupsManager {
// empty catch intentional: storage might be unavailable or full
}
}, DEBOUNCE_INTERVAL);
startImport({ group, importId }) {
this.setImportStatus(group, STATUSES.CREATED);
this.saveImportState(importId, group);
}
setImportStatus(group, status) {
this.update(group, (sourceGroup) => {
// eslint-disable-next-line no-param-reassign
sourceGroup.status = status;
});
}
setImportStatusByImportId(importId, status) {
const importState = this.getImportStateFromStorage(importId);
if (!importState) {
return;
}
if (importState.status !== status) {
importState.status = status;
}
const group = this.findById(importState.id);
if (group?.id) {
this.setImportStatus(group, status);
}
this.saveImportStatesToStorage();
}
}
app/assets/javascripts/import_entities/import_groups/graphql/services/status_poller.js
View file @ b18aa67e
......@@ -5,13 +5,15 @@ import Poll from '~/lib/utils/poll';
import { s__ } from '~/locale';
export class StatusPoller {
constructor({ groupManager, pollPath }) {
constructor({ updateImportStatus, pollPath }) {
this.eTagPoll = new Poll({
resource: {
fetchJobs: () => axios.get(pollPath),
},
method: 'fetchJobs',
successCallback: ({ data }) => this.updateImportsStatuses(data),
successCallback: ({ data: statuses }) => {
statuses.forEach((status) => updateImportStatus(status));
},
errorCallback: () =>
createFlash({
message: s__('BulkImport|Update of import statuses with realtime changes failed'),
......@@ -25,17 +27,9 @@ export class StatusPoller {
this.eTagPoll.stop();
}
});
this.groupManager = groupManager;
}
startPolling() {
this.eTagPoll.makeRequest();
}
async updateImportsStatuses(importStatuses) {
importStatuses.forEach(({ id, status_name: statusName }) => {
this.groupManager.setImportStatusByImportId(id, statusName);
});
}
}
app/assets/javascripts/import_entities/import_groups/graphql/typedefs.graphql 0 → 100644
View file @ b18aa67e
type ClientBulkImportAvailableNamespace {
id: ID!
full_path: String!
}
type ClientBulkImportTarget {
target_namespace: String!
new_name: String!
}
type ClientBulkImportSourceGroupConnection {
nodes: [ClientBulkImportSourceGroup!]!
pageInfo: ClientBulkImportPageInfo!
}
type ClientBulkImportProgress {
id: ID
status: String!
}
type ClientBulkImportSourceGroup {
id: ID!
web_url: String!
full_path: String!
full_name: String!
progress: ClientBulkImportProgress!
import_target: ClientBulkImportTarget!
}
type ClientBulkImportPageInfo {
page: Int!
perPage: Int!
total: Int!
totalPages: Int!
}
extend type Query {
bulkImportSourceGroup(id: ID!): ClientBulkImportSourceGroup
bulkImportSourceGroups(
page: Int!
perPage: Int!
filter: String!
): ClientBulkImportSourceGroupConnection!
availableNamespaces: [ClientBulkImportAvailableNamespace!]!
}
extend type Mutation {
setNewName(newName: String, sourceGroupId: ID!): ClientTargetNamespace!
setTargetNamespace(targetNamespace: String, sourceGroupId: ID!): ClientTargetNamespace!
importGroup(id: ID!): ClientBulkImportSourceGroup!
setImportProgress(id: ID, status: String!): ClientBulkImportSourceGroup!
updateImportProgress(id: ID, status: String!): ClientBulkImportProgress
}
spec/frontend/import_entities/import_groups/components/import_table_row_spec.js
View file @ b18aa67e
......@@ -19,7 +19,7 @@ const getFakeGroup = (status) => ({
new_name: 'group1',
},
id: 1,
status,
progress: { status },
});
const EXISTING_GROUP_TARGET_NAMESPACE = 'existing-group';
......
spec/frontend/import_entities/import_groups/graphql/fixtures.js
View file @ b18aa67e
......@@ -10,7 +10,10 @@ export const generateFakeEntry = ({ id, status, ...rest }) => ({
new_name: `group${id}`,
},
id,
status,
progress: {
id: `test-${id}`,
status,
},
...rest,
});
......
spec/frontend/import_entities/import_groups/graphql/services/source_groups_manager_spec.js
View file @ b18aa67e
import { defaultDataIdFromObject } from 'apollo-cache-inmemory';
import { clientTypenames } from '~/import_entities/import_groups/graphql/client_factory';
import ImportSourceGroupFragment from '~/import_entities/import_groups/graphql/fragments/bulk_import_source_group_item.fragment.graphql';
import {
KEY,
SourceGroupsManager,
......@@ -10,42 +7,29 @@ const FAKE_SOURCE_URL = 'http://demo.host';
describe('SourceGroupsManager', () => {
let manager;
let client;
let storage;
const getFakeGroup = () => ({
__typename: clientTypenames.BulkImportSourceGroup,
id: 5,
});
beforeEach(() => {
client = {
readFragment: jest.fn(),
writeFragment: jest.fn(),
};
storage = {
getItem: jest.fn(),
setItem: jest.fn(),
};
manager = new SourceGroupsManager({ client, storage, sourceUrl: FAKE_SOURCE_URL });
manager = new SourceGroupsManager({ storage, sourceUrl: FAKE_SOURCE_URL });
});
describe('storage management', () => {
const IMPORT_ID = 1;
const IMPORT_TARGET = { destination_name: 'demo', destination_namespace: 'foo' };
const STATUS = 'FAKE_STATUS';
const FAKE_GROUP = { id: 1, import_target: IMPORT_TARGET, status: STATUS };
const FAKE_GROUP = { id: 1, importTarget: IMPORT_TARGET, status: STATUS };
it('loads state from storage on creation', () => {
expect(storage.getItem).toHaveBeenCalledWith(KEY);
});
it('saves to storage when import is starting', () => {
manager.startImport({
importId: IMPORT_ID,
group: FAKE_GROUP,
});
it('saves to storage when saveImportState is called', () => {
manager.saveImportState(IMPORT_ID, FAKE_GROUP);
const storedObject = JSON.parse(storage.setItem.mock.calls[0][1]);
expect(Object.values(storedObject)[0]).toStrictEqual({
id: FAKE_GROUP.id,
......@@ -54,15 +38,12 @@ describe('SourceGroupsManager', () => {
});
});
it('saves to storage when import status is updated', () => {
it('updates storage when previous state is available', () => {
const CHANGED_STATUS = 'changed';
manager.startImport({
importId: IMPORT_ID,
group: FAKE_GROUP,
});
manager.saveImportState(IMPORT_ID, FAKE_GROUP);
manager.setImportStatusByImportId(IMPORT_ID, CHANGED_STATUS);
manager.saveImportState(IMPORT_ID, { status: CHANGED_STATUS });
const storedObject = JSON.parse(storage.setItem.mock.calls[1][1]);
expect(Object.values(storedObject)[0]).toStrictEqual({
id: FAKE_GROUP.id,
......@@ -71,63 +52,4 @@ describe('SourceGroupsManager', () => {
});
});
});
it('finds item by group id', () => {
const ID = 5;
const FAKE_GROUP = getFakeGroup();
client.readFragment.mockReturnValue(FAKE_GROUP);
const group = manager.findById(ID);
expect(group).toBe(FAKE_GROUP);
expect(client.readFragment).toHaveBeenCalledWith({
fragment: ImportSourceGroupFragment,
id: defaultDataIdFromObject(getFakeGroup()),
});
});
it('updates group with provided function', () => {
const UPDATED_GROUP = {};
const fn = jest.fn().mockReturnValue(UPDATED_GROUP);
manager.update(getFakeGroup(), fn);
expect(client.writeFragment).toHaveBeenCalledWith({
fragment: ImportSourceGroupFragment,
id: defaultDataIdFromObject(getFakeGroup()),
data: UPDATED_GROUP,
});
});
it('updates group by id with provided function', () => {
const UPDATED_GROUP = {};
const fn = jest.fn().mockReturnValue(UPDATED_GROUP);
client.readFragment.mockReturnValue(getFakeGroup());
manager.updateById(getFakeGroup().id, fn);
expect(client.readFragment).toHaveBeenCalledWith({
fragment: ImportSourceGroupFragment,
id: defaultDataIdFromObject(getFakeGroup()),
});
expect(client.writeFragment).toHaveBeenCalledWith({
fragment: ImportSourceGroupFragment,
id: defaultDataIdFromObject(getFakeGroup()),
data: UPDATED_GROUP,
});
});
it('sets import status when group is provided', () => {
client.readFragment.mockReturnValue(getFakeGroup());
const NEW_STATUS = 'NEW_STATUS';
manager.setImportStatus(getFakeGroup(), NEW_STATUS);
expect(client.writeFragment).toHaveBeenCalledWith({
fragment: ImportSourceGroupFragment,
id: defaultDataIdFromObject(getFakeGroup()),
data: {
...getFakeGroup(),
status: NEW_STATUS,
},
});
});
});
spec/frontend/import_entities/import_groups/graphql/services/status_poller_spec.js
View file @ b18aa67e
......@@ -21,17 +21,15 @@ const FAKE_POLL_PATH = '/fake/poll/path';
describe('Bulk import status poller', () => {
let poller;
let mockAdapter;
let groupManager;
let updateImportStatus;
const getPollHistory = () => mockAdapter.history.get.filter((x) => x.url === FAKE_POLL_PATH);
beforeEach(() => {
mockAdapter = new MockAdapter(axios);
mockAdapter.onGet(FAKE_POLL_PATH).reply(200, {});
groupManager = {
setImportStatusByImportId: jest.fn(),
};
poller = new StatusPoller({ groupManager, pollPath: FAKE_POLL_PATH });
updateImportStatus = jest.fn();
poller = new StatusPoller({ updateImportStatus, pollPath: FAKE_POLL_PATH });
});
it('creates poller with proper config', () => {
......@@ -96,9 +94,9 @@ describe('Bulk import status poller', () => {
it('when success response arrives updates relevant group status', () => {
const FAKE_ID = 5;
const [[pollConfig]] = Poll.mock.calls;
const FAKE_RESPONSE = { id: FAKE_ID, status_name: STATUSES.FINISHED };
pollConfig.successCallback({ data: [FAKE_RESPONSE] });
pollConfig.successCallback({ data: [{ id: FAKE_ID, status_name: STATUSES.FINISHED }] });
expect(groupManager.setImportStatusByImportId).toHaveBeenCalledWith(FAKE_ID, STATUSES.FINISHED);
expect(updateImportStatus).toHaveBeenCalledWith(FAKE_RESPONSE);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7