Commit c6d58ae4 authored by Arturo Herrero's avatar Arturo Herrero Committed by Kerri Miller

Propagate group-level integrations

In https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40717 and
https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42128, we
propagate instance-level integrations to groups.

This propagates group-level integrations to the lower levels using the
same approach of batching and Sidekiq queues.

This is the backend behavior when saving an instance-level or
group-level integration:
- Update inherited integrations (find all integrations inheriting from
  the instance-level or group-level integration that we are updating).
- For instance-level:
  - Create integration for all projects without integration
  - Create integration for all groups without integration
- For group-level:
  - Create integration for projects belonging to the group without
    integration (new).
  - Create integration for groups belonging to the group without
    integration (new).
parent 353c8b72
......@@ -7,8 +7,13 @@ module Admin
def propagate
update_inherited_integrations
create_integration_for_groups_without_integration if Feature.enabled?(:group_level_integrations)
create_integration_for_projects_without_integration
if integration.instance?
create_integration_for_groups_without_integration if Feature.enabled?(:group_level_integrations)
create_integration_for_projects_without_integration
else
create_integration_for_groups_without_integration_belonging_to_group
create_integration_for_projects_without_integration_belonging_to_group
end
end
private
......@@ -28,5 +33,19 @@ module Admin
PropagateIntegrationGroupWorker.perform_async(integration.id, min_id, max_id)
end
end
def create_integration_for_groups_without_integration_belonging_to_group
integration.group.descendants.without_integration(integration).each_batch(of: BATCH_SIZE) do |groups|
min_id, max_id = groups.pick("MIN(namespaces.id), MAX(namespaces.id)")
PropagateIntegrationGroupWorker.perform_async(integration.id, min_id, max_id)
end
end
def create_integration_for_projects_without_integration_belonging_to_group
Project.without_integration(integration).in_namespace(integration.group.self_and_descendants).each_batch(of: BATCH_SIZE) do |projects|
min_id, max_id = projects.pick("MIN(projects.id), MAX(projects.id)")
PropagateIntegrationProjectWorker.perform_async(integration.id, min_id, max_id)
end
end
end
end
......@@ -11,7 +11,13 @@ class PropagateIntegrationGroupWorker
integration = Service.find_by_id(integration_id)
return unless integration
batch = Group.where(id: min_id..max_id).without_integration(integration)
batch = if integration.instance?
Group.where(id: min_id..max_id).without_integration(integration)
else
integration.group.descendants.where(id: min_id..max_id).without_integration(integration)
end
return if batch.empty?
BulkCreateIntegrationService.new(integration, batch, 'group').execute
end
......
......@@ -12,6 +12,9 @@ class PropagateIntegrationProjectWorker
return unless integration
batch = Project.where(id: min_id..max_id).without_integration(integration)
batch = batch.in_namespace(integration.group.self_and_descendants) if integration.group_id
return if batch.empty?
BulkCreateIntegrationService.new(integration, batch, 'project').execute
end
......
......@@ -3,10 +3,10 @@
require 'spec_helper'
RSpec.describe Integration do
let!(:project_1) { create(:project) }
let!(:project_2) { create(:project) }
let!(:project_3) { create(:project) }
let(:instance_integration) { create(:jira_service, :instance) }
let_it_be(:project_1) { create(:project) }
let_it_be(:project_2) { create(:project) }
let_it_be(:project_3) { create(:project) }
let_it_be(:instance_integration) { create(:jira_service, :instance) }
before do
create(:jira_service, project: project_1, inherit_from_id: instance_integration.id)
......
......@@ -10,56 +10,16 @@ RSpec.describe Admin::PropagateIntegrationService do
stub_jira_service_test
end
let(:group) { create(:group) }
let_it_be(:project) { create(:project) }
let!(:instance_integration) do
JiraService.create!(
instance: true,
active: true,
push_events: true,
url: 'http://update-jira.instance.com',
username: 'user',
password: 'secret'
)
let_it_be(:instance_integration) { create(:jira_service, :instance) }
let_it_be(:not_inherited_integration) { create(:jira_service, project: project) }
let_it_be(:inherited_integration) do
create(:jira_service, project: create(:project), inherit_from_id: instance_integration.id)
end
let!(:inherited_integration) do
JiraService.create!(
project: create(:project),
inherit_from_id: instance_integration.id,
instance: false,
active: true,
push_events: false,
url: 'http://jira.instance.com',
username: 'user',
password: 'secret'
)
end
let!(:not_inherited_integration) do
JiraService.create!(
project: project,
inherit_from_id: nil,
instance: false,
active: true,
push_events: false,
url: 'http://jira.instance.com',
username: 'user',
password: 'secret'
)
end
let!(:different_type_inherited_integration) do
BambooService.create!(
project: project,
inherit_from_id: instance_integration.id,
instance: false,
active: true,
push_events: false,
bamboo_url: 'http://gitlab.com',
username: 'mic',
password: 'password',
build_key: 'build'
)
let_it_be(:different_type_inherited_integration) do
create(:redmine_service, project: project, inherit_from_id: instance_integration.id)
end
context 'with inherited integration' do
......@@ -74,7 +34,7 @@ RSpec.describe Admin::PropagateIntegrationService do
end
context 'with a project without integration' do
let!(:another_project) { create(:project) }
let(:another_project) { create(:project) }
it 'calls to PropagateIntegrationProjectWorker' do
expect(PropagateIntegrationProjectWorker).to receive(:perform_async)
......@@ -85,8 +45,6 @@ RSpec.describe Admin::PropagateIntegrationService do
end
context 'with a group without integration' do
let!(:group) { create(:group) }
it 'calls to PropagateIntegrationProjectWorker' do
expect(PropagateIntegrationGroupWorker).to receive(:perform_async)
.with(instance_integration.id, group.id, group.id)
......@@ -94,5 +52,31 @@ RSpec.describe Admin::PropagateIntegrationService do
described_class.propagate(instance_integration)
end
end
context 'for a group-level integration' do
let(:group_integration) { create(:jira_service, group: group, project: nil) }
context 'with a project without integration' do
let(:another_project) { create(:project, group: group) }
it 'calls to PropagateIntegrationProjectWorker' do
expect(PropagateIntegrationProjectWorker).to receive(:perform_async)
.with(group_integration.id, another_project.id, another_project.id)
described_class.propagate(group_integration)
end
end
context 'with a group without integration' do
let(:subgroup) { create(:group, parent: group) }
it 'calls to PropagateIntegrationGroupWorker' do
expect(PropagateIntegrationGroupWorker).to receive(:perform_async)
.with(group_integration.id, subgroup.id, subgroup.id)
described_class.propagate(group_integration)
end
end
end
end
end
......@@ -4,33 +4,40 @@ require 'spec_helper'
RSpec.describe PropagateIntegrationGroupWorker do
describe '#perform' do
let_it_be(:group1) { create(:group) }
let_it_be(:group2) { create(:group) }
let_it_be(:group) { create(:group) }
let_it_be(:another_group) { create(:group) }
let_it_be(:subgroup1) { create(:group, parent: group) }
let_it_be(:subgroup2) { create(:group, parent: group) }
let_it_be(:integration) { create(:redmine_service, :instance) }
before do
allow(BulkCreateIntegrationService).to receive(:new)
.with(integration, match_array([group1, group2]), 'group')
.and_return(double(execute: nil))
end
let(:job_args) { [integration.id, group.id, subgroup2.id] }
it_behaves_like 'an idempotent worker' do
let(:job_args) { [integration.id, group1.id, group2.id] }
it 'calls to BulkCreateIntegrationService' do
expect(BulkCreateIntegrationService).to receive(:new)
.with(integration, match_array([group1, group2]), 'group')
.with(integration, match_array([group, another_group, subgroup1, subgroup2]), 'group').twice
.and_return(double(execute: nil))
subject
end
context 'with a group integration' do
let_it_be(:integration) { create(:redmine_service, group: group, project: nil) }
it 'calls to BulkCreateIntegrationService' do
expect(BulkCreateIntegrationService).to receive(:new)
.with(integration, match_array([subgroup1, subgroup2]), 'group').twice
.and_return(double(execute: nil))
subject
end
end
end
context 'with an invalid integration id' do
it 'returns without failure' do
expect(BulkCreateIntegrationService).not_to receive(:new)
subject.perform(0, group1.id, group2.id)
subject.perform(0, 1, 100)
end
end
end
......
......@@ -9,18 +9,12 @@ RSpec.describe PropagateIntegrationInheritWorker do
let_it_be(:integration2) { create(:bugzilla_service, inherit_from_id: integration.id) }
let_it_be(:integration3) { create(:redmine_service) }
before do
allow(BulkUpdateIntegrationService).to receive(:new)
.with(integration, match_array(integration1))
.and_return(double(execute: nil))
end
it_behaves_like 'an idempotent worker' do
let(:job_args) { [integration.id, integration1.id, integration3.id] }
it 'calls to BulkCreateIntegrationService' do
expect(BulkUpdateIntegrationService).to receive(:new)
.with(integration, match_array(integration1))
.with(integration, match_array(integration1)).twice
.and_return(double(execute: nil))
subject
......
......@@ -4,33 +4,40 @@ require 'spec_helper'
RSpec.describe PropagateIntegrationProjectWorker do
describe '#perform' do
let_it_be(:group) { create(:group) }
let_it_be(:project1) { create(:project) }
let_it_be(:project2) { create(:project) }
let_it_be(:project2) { create(:project, group: group) }
let_it_be(:project3) { create(:project, group: group) }
let_it_be(:integration) { create(:redmine_service, :instance) }
before do
allow(BulkCreateIntegrationService).to receive(:new)
.with(integration, match_array([project1, project2]), 'project')
.and_return(double(execute: nil))
end
let(:job_args) { [integration.id, project1.id, project3.id] }
it_behaves_like 'an idempotent worker' do
let(:job_args) { [integration.id, project1.id, project2.id] }
it 'calls to BulkCreateIntegrationService' do
expect(BulkCreateIntegrationService).to receive(:new)
.with(integration, match_array([project1, project2]), 'project')
.with(integration, match_array([project1, project2, project3]), 'project').twice
.and_return(double(execute: nil))
subject
end
context 'with a group integration' do
let_it_be(:integration) { create(:redmine_service, group: group, project: nil) }
it 'calls to BulkCreateIntegrationService' do
expect(BulkCreateIntegrationService).to receive(:new)
.with(integration, match_array([project2, project3]), 'project').twice
.and_return(double(execute: nil))
subject
end
end
end
context 'with an invalid integration id' do
it 'returns without failure' do
expect(BulkCreateIntegrationService).not_to receive(:new)
subject.perform(0, project1.id, project2.id)
subject.perform(0, 1, 100)
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment