Propagate group-level integrations

In https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40717 and
https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42128, we
propagate instance-level integrations to groups.

This propagates group-level integrations to the lower levels using the
same approach of batching and Sidekiq queues.

This is the backend behavior when saving an instance-level or
group-level integration:
- Update inherited integrations (find all integrations inheriting from
  the instance-level or group-level integration that we are updating).
- For instance-level:
  - Create integration for all projects without integration
  - Create integration for all groups without integration
- For group-level:
  - Create integration for projects belonging to the group without
    integration (new).
  - Create integration for groups belonging to the group without
    integration (new).

app/services/admin/propagate_integration_service.rb

@@ -7,8 +7,13 @@ module Admin
     def propagate
       update_inherited_integrations
 
+      if integration.instance?
         create_integration_for_groups_without_integration if Feature.enabled?(:group_level_integrations)
         create_integration_for_projects_without_integration
+      else
+        create_integration_for_groups_without_integration_belonging_to_group
+        create_integration_for_projects_without_integration_belonging_to_group
+      end
     end
 
     private
@@ -28,5 +33,19 @@ module Admin
         PropagateIntegrationGroupWorker.perform_async(integration.id, min_id, max_id)
       end
     end
+
+    def create_integration_for_groups_without_integration_belonging_to_group
+      integration.group.descendants.without_integration(integration).each_batch(of: BATCH_SIZE) do |groups|
+        min_id, max_id = groups.pick("MIN(namespaces.id), MAX(namespaces.id)")
+        PropagateIntegrationGroupWorker.perform_async(integration.id, min_id, max_id)
+      end
+    end
+
+    def create_integration_for_projects_without_integration_belonging_to_group
+      Project.without_integration(integration).in_namespace(integration.group.self_and_descendants).each_batch(of: BATCH_SIZE) do |projects|
+        min_id, max_id = projects.pick("MIN(projects.id), MAX(projects.id)")
+        PropagateIntegrationProjectWorker.perform_async(integration.id, min_id, max_id)
+      end
+    end
   end
 end

app/workers/propagate_integration_group_worker.rb

@@ -11,7 +11,13 @@ class PropagateIntegrationGroupWorker
     integration = Service.find_by_id(integration_id)
     return unless integration
 
-    batch = Group.where(id: min_id..max_id).without_integration(integration)
+    batch = if integration.instance?
+              Group.where(id: min_id..max_id).without_integration(integration)
+            else
+              integration.group.descendants.where(id: min_id..max_id).without_integration(integration)
+            end
+
+    return if batch.empty?
 
     BulkCreateIntegrationService.new(integration, batch, 'group').execute
   end

app/workers/propagate_integration_project_worker.rb

@@ -12,6 +12,9 @@ class PropagateIntegrationProjectWorker
     return unless integration
 
     batch = Project.where(id: min_id..max_id).without_integration(integration)
+    batch = batch.in_namespace(integration.group.self_and_descendants) if integration.group_id
+
+    return if batch.empty?
 
     BulkCreateIntegrationService.new(integration, batch, 'project').execute
   end

spec/models/integration_spec.rb

@@ -3,10 +3,10 @@
 require 'spec_helper'
 
 RSpec.describe Integration do
-  let!(:project_1) { create(:project) }
-  let!(:project_2) { create(:project) }
-  let!(:project_3) { create(:project) }
-  let(:instance_integration) { create(:jira_service, :instance) }
+  let_it_be(:project_1) { create(:project) }
+  let_it_be(:project_2) { create(:project) }
+  let_it_be(:project_3) { create(:project) }
+  let_it_be(:instance_integration) { create(:jira_service, :instance) }
 
   before do
     create(:jira_service, project: project_1, inherit_from_id: instance_integration.id)

spec/services/admin/propagate_integration_service_spec.rb

@@ -10,56 +10,16 @@ RSpec.describe Admin::PropagateIntegrationService do
       stub_jira_service_test
     end
 
-    let_it_be(:project) { create(:project) }
-    let!(:instance_integration) do
-      JiraService.create!(
-        instance: true,
-        active: true,
-        push_events: true,
-        url: 'http://update-jira.instance.com',
-        username: 'user',
-        password: 'secret'
-      )
-    end
-
-    let!(:inherited_integration) do
-      JiraService.create!(
-        project: create(:project),
-        inherit_from_id: instance_integration.id,
-        instance: false,
-        active: true,
-        push_events: false,
-        url: 'http://jira.instance.com',
-        username: 'user',
-        password: 'secret'
-      )
-    end
+    let(:group) { create(:group) }
 
-    let!(:not_inherited_integration) do
-      JiraService.create!(
-        project: project,
-        inherit_from_id: nil,
-        instance: false,
-        active: true,
-        push_events: false,
-        url: 'http://jira.instance.com',
-        username: 'user',
-        password: 'secret'
-      )
+    let_it_be(:project) { create(:project) }
+    let_it_be(:instance_integration) { create(:jira_service, :instance) }
+    let_it_be(:not_inherited_integration) { create(:jira_service, project: project) }
+    let_it_be(:inherited_integration) do
+      create(:jira_service, project: create(:project), inherit_from_id: instance_integration.id)
     end
-
-    let!(:different_type_inherited_integration) do
-      BambooService.create!(
-        project: project,
-        inherit_from_id: instance_integration.id,
-        instance: false,
-        active: true,
-        push_events: false,
-        bamboo_url: 'http://gitlab.com',
-        username: 'mic',
-        password: 'password',
-        build_key: 'build'
-      )
+    let_it_be(:different_type_inherited_integration) do
+      create(:redmine_service, project: project, inherit_from_id: instance_integration.id)
     end
 
     context 'with inherited integration' do
@@ -74,7 +34,7 @@ RSpec.describe Admin::PropagateIntegrationService do
     end
 
     context 'with a project without integration' do
-      let!(:another_project) { create(:project) }
+      let(:another_project) { create(:project) }
 
       it 'calls to PropagateIntegrationProjectWorker' do
         expect(PropagateIntegrationProjectWorker).to receive(:perform_async)
@@ -85,8 +45,6 @@ RSpec.describe Admin::PropagateIntegrationService do
     end
 
     context 'with a group without integration' do
-      let!(:group) { create(:group) }
-
       it 'calls to PropagateIntegrationProjectWorker' do
         expect(PropagateIntegrationGroupWorker).to receive(:perform_async)
           .with(instance_integration.id, group.id, group.id)
@@ -94,5 +52,31 @@ RSpec.describe Admin::PropagateIntegrationService do
         described_class.propagate(instance_integration)
       end
     end
+
+    context 'for a group-level integration' do
+      let(:group_integration) { create(:jira_service, group: group, project: nil) }
+
+      context 'with a project without integration' do
+        let(:another_project) { create(:project, group: group) }
+
+        it 'calls to PropagateIntegrationProjectWorker' do
+          expect(PropagateIntegrationProjectWorker).to receive(:perform_async)
+            .with(group_integration.id, another_project.id, another_project.id)
+
+          described_class.propagate(group_integration)
+        end
+      end
+
+      context 'with a group without integration' do
+        let(:subgroup) { create(:group, parent: group) }
+
+        it 'calls to PropagateIntegrationGroupWorker' do
+          expect(PropagateIntegrationGroupWorker).to receive(:perform_async)
+            .with(group_integration.id, subgroup.id, subgroup.id)
+
+          described_class.propagate(group_integration)
+        end
+      end
+    end
   end
 end

spec/workers/propagate_integration_group_worker_spec.rb

@@ -4,33 +4,40 @@ require 'spec_helper'
 
 RSpec.describe PropagateIntegrationGroupWorker do
   describe '#perform' do
-    let_it_be(:group1) { create(:group) }
-    let_it_be(:group2) { create(:group) }
+    let_it_be(:group) { create(:group) }
+    let_it_be(:another_group) { create(:group) }
+    let_it_be(:subgroup1) { create(:group, parent: group) }
+    let_it_be(:subgroup2) { create(:group, parent: group) }
     let_it_be(:integration) { create(:redmine_service, :instance) }
+    let(:job_args) { [integration.id, group.id, subgroup2.id] }
 
-    before do
-      allow(BulkCreateIntegrationService).to receive(:new)
-        .with(integration, match_array([group1, group2]), 'group')
+    it_behaves_like 'an idempotent worker' do
+      it 'calls to BulkCreateIntegrationService' do
+        expect(BulkCreateIntegrationService).to receive(:new)
+          .with(integration, match_array([group, another_group, subgroup1, subgroup2]), 'group').twice
           .and_return(double(execute: nil))
+
+        subject
       end
 
-    it_behaves_like 'an idempotent worker' do
-      let(:job_args) { [integration.id, group1.id, group2.id] }
+      context 'with a group integration' do
+        let_it_be(:integration) { create(:redmine_service, group: group, project: nil) }
 
         it 'calls to BulkCreateIntegrationService' do
           expect(BulkCreateIntegrationService).to receive(:new)
-          .with(integration, match_array([group1, group2]), 'group')
+            .with(integration, match_array([subgroup1, subgroup2]), 'group').twice
             .and_return(double(execute: nil))
 
           subject
         end
       end
+    end
 
     context 'with an invalid integration id' do
       it 'returns without failure' do
         expect(BulkCreateIntegrationService).not_to receive(:new)
 
-        subject.perform(0, group1.id, group2.id)
+        subject.perform(0, 1, 100)
       end
     end
   end

spec/workers/propagate_integration_inherit_worker_spec.rb

@@ -9,18 +9,12 @@ RSpec.describe PropagateIntegrationInheritWorker do
     let_it_be(:integration2) { create(:bugzilla_service, inherit_from_id: integration.id) }
     let_it_be(:integration3) { create(:redmine_service) }
 
-    before do
-      allow(BulkUpdateIntegrationService).to receive(:new)
-        .with(integration, match_array(integration1))
-        .and_return(double(execute: nil))
-    end
-
     it_behaves_like 'an idempotent worker' do
       let(:job_args) { [integration.id, integration1.id, integration3.id] }
 
       it 'calls to BulkCreateIntegrationService' do
         expect(BulkUpdateIntegrationService).to receive(:new)
-          .with(integration, match_array(integration1))
+          .with(integration, match_array(integration1)).twice
           .and_return(double(execute: nil))
 
         subject

spec/workers/propagate_integration_project_worker_spec.rb

@@ -4,33 +4,40 @@ require 'spec_helper'
 
 RSpec.describe PropagateIntegrationProjectWorker do
   describe '#perform' do
+    let_it_be(:group) { create(:group) }
     let_it_be(:project1) { create(:project) }
-    let_it_be(:project2) { create(:project) }
+    let_it_be(:project2) { create(:project, group: group) }
+    let_it_be(:project3) { create(:project, group: group) }
     let_it_be(:integration) { create(:redmine_service, :instance) }
+    let(:job_args) { [integration.id, project1.id, project3.id] }
 
-    before do
-      allow(BulkCreateIntegrationService).to receive(:new)
-        .with(integration, match_array([project1, project2]), 'project')
+    it_behaves_like 'an idempotent worker' do
+      it 'calls to BulkCreateIntegrationService' do
+        expect(BulkCreateIntegrationService).to receive(:new)
+          .with(integration, match_array([project1, project2, project3]), 'project').twice
           .and_return(double(execute: nil))
+
+        subject
       end
 
-    it_behaves_like 'an idempotent worker' do
-      let(:job_args) { [integration.id, project1.id, project2.id] }
+      context 'with a group integration' do
+        let_it_be(:integration) { create(:redmine_service, group: group, project: nil) }
 
         it 'calls to BulkCreateIntegrationService' do
           expect(BulkCreateIntegrationService).to receive(:new)
-          .with(integration, match_array([project1, project2]), 'project')
+            .with(integration, match_array([project2, project3]), 'project').twice
             .and_return(double(execute: nil))
 
           subject
         end
       end
+    end
 
     context 'with an invalid integration id' do
       it 'returns without failure' do
         expect(BulkCreateIntegrationService).not_to receive(:new)
 
-        subject.perform(0, project1.id, project2.id)
+        subject.perform(0, 1, 100)
       end
     end
   end