Sync recent MRs in a namespace with Jira Connect

When a new namespace is subscribed to Jira Connect
it creates jobs to sync the most recent MRs
that reference a Jira issue.

app/models/merge_request.rb

@@ -304,6 +304,8 @@ class MergeRequest < ApplicationRecord
     includes(:metrics)
   end
 
+  scope :with_jira_issue_keys, -> { where('title ~ :regex OR merge_requests.description ~ :regex', regex: Gitlab::Regex.jira_issue_key_regex.source) }
+
   after_save :keep_around_commit, unless: :importing?
 
   alias_attribute :project, :target_project

app/services/jira_connect_subscriptions/create_service.rb

@@ -3,6 +3,8 @@
 module JiraConnectSubscriptions
   class CreateService < ::JiraConnectSubscriptions::BaseService
     include Gitlab::Utils::StrongMemoize
+    MERGE_REQUEST_SYNC_BATCH_SIZE = 20
+    MERGE_REQUEST_SYNC_BATCH_delay = 1.minute.freeze
 
     def execute
       unless namespace && can?(current_user, :create_jira_connect_subscription, namespace)
@@ -18,6 +20,8 @@ module JiraConnectSubscriptions
       subscription = JiraConnectSubscription.new(installation: jira_connect_installation, namespace: namespace)
 
       if subscription.save
+        schedule_sync_project_jobs
+
         success
       else
         error(subscription.errors.full_messages.join(', '), 422)
@@ -29,5 +33,18 @@ module JiraConnectSubscriptions
         Namespace.find_by_full_path(params[:namespace_path])
       end
     end
+
+    def schedule_sync_project_jobs
+      return unless Feature.enabled?(:jira_connect_full_namespace_sync)
+
+      namespace.all_projects.each_batch(of: MERGE_REQUEST_SYNC_BATCH_SIZE) do |projects, index|
+        JiraConnect::SyncProjectWorker.bulk_perform_in_with_contexts(
+          index * MERGE_REQUEST_SYNC_BATCH_delay,
+          projects,
+          arguments_proc: -> (project) { [project.id, Atlassian::JiraConnect::Client.generate_update_sequence_id] },
+          context_proc: -> (project) { { project: project } }
+        )
+      end
+    end
   end
 end

app/workers/all_queues.yml

@@ -827,6 +827,14 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: jira_connect:jira_connect_sync_project
+  :feature_category: :integrations
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: jira_importer:jira_import_advance_stage
   :feature_category: :importers
   :has_external_dependencies:

app/workers/jira_connect/sync_project_worker.rb

+# frozen_string_literal: true
+
+module JiraConnect
+  class SyncProjectWorker
+    include ApplicationWorker
+
+    queue_namespace :jira_connect
+    feature_category :integrations
+    idempotent!
+    worker_has_external_dependencies!
+
+    MERGE_REQUEST_LIMIT = 400
+
+    def perform(project_id, update_sequence_id)
+      project = Project.find_by_id(project_id)
+
+      return if project.nil?
+
+      JiraConnect::SyncService.new(project).execute(merge_requests: merge_requests_to_sync(project), update_sequence_id: update_sequence_id)
+    end
+
+    private
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def merge_requests_to_sync(project)
+      project.merge_requests.with_jira_issue_keys.preload(:author).limit(MERGE_REQUEST_LIMIT).order(id: :desc)
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+  end
+end

changelogs/unreleased/jira_connect_sync_namespace_worker.yml

+---
+title: Jira Connect automatically synchronizes up to 400 existing merge requests per project when a namespace is connected.
+merge_request: 43880
+author:
+type: added

config/feature_flags/development/jira_connect_full_namespace_sync.yml

+---
+name: jira_connect_full_namespace_sync
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43880
+rollout_issue_url: 
+type: development
+group: group::ecosystem
+default_enabled: false

db/post_migrate/20201106082723_add_merge_request_jira_reference_indexes.rb

+# frozen_string_literal: true
+
+class AddMergeRequestJiraReferenceIndexes < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  DESCRIPTION_INDEX_NAME = 'index_merge_requests_on_target_project_id_iid_jira_description'
+  TITLE_INDEX_NAME = 'index_merge_requests_on_target_project_id_and_iid_jira_title'
+
+  JIRA_KEY_REGEX = '[A-Z][A-Z_0-9]+-\d+'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(
+      :merge_requests,
+      [:target_project_id, :iid],
+      name: TITLE_INDEX_NAME,
+      using: :btree,
+      where: "(merge_requests.title)::text ~ '#{JIRA_KEY_REGEX}'::text"
+    )
+
+    add_concurrent_index(
+      :merge_requests,
+      [:target_project_id, :iid],
+      name: DESCRIPTION_INDEX_NAME,
+      using: :btree,
+      where: "(merge_requests.description)::text ~ '#{JIRA_KEY_REGEX}'::text"
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name(
+      :merge_requests,
+      TITLE_INDEX_NAME
+    )
+
+    remove_concurrent_index_by_name(
+      :merge_requests,
+      DESCRIPTION_INDEX_NAME
+    )
+  end
+end

db/schema_migrations/20201106082723

+a2dc0d31af6834adf6634f6051d7d451fc48d31492d96efe57547c3e9d61a64d
\ No newline at end of file

db/structure.sql

@@ -21144,8 +21144,12 @@ CREATE UNIQUE INDEX index_merge_requests_on_target_project_id_and_iid ON merge_r
 
 CREATE INDEX index_merge_requests_on_target_project_id_and_iid_and_state_id ON merge_requests USING btree (target_project_id, iid, state_id);
 
+CREATE INDEX index_merge_requests_on_target_project_id_and_iid_jira_title ON merge_requests USING btree (target_project_id, iid) WHERE ((title)::text ~ '[A-Z][A-Z_0-9]+-\d+'::text);
+
 CREATE INDEX index_merge_requests_on_target_project_id_and_target_branch ON merge_requests USING btree (target_project_id, target_branch) WHERE ((state_id = 1) AND (merge_when_pipeline_succeeds = true));
 
+CREATE INDEX index_merge_requests_on_target_project_id_iid_jira_description ON merge_requests USING btree (target_project_id, iid) WHERE (description ~ '[A-Z][A-Z_0-9]+-\d+'::text);
+
 CREATE INDEX index_merge_requests_on_title ON merge_requests USING btree (title);
 
 CREATE INDEX index_merge_requests_on_title_trigram ON merge_requests USING gin (title gin_trgm_ops);

spec/models/merge_request_spec.rb

@@ -79,6 +79,18 @@ RSpec.describe MergeRequest, factory_default: :keep do
     end
   end
 
+  describe '.with_jira_issue_keys' do
+    let_it_be(:mr_with_jira_title) { create(:merge_request, :unique_branches, title: 'Fix TEST-123') }
+    let_it_be(:mr_with_jira_description) { create(:merge_request, :unique_branches, description: 'this closes TEST-321') }
+    let_it_be(:mr_without_jira_reference) { create(:merge_request, :unique_branches) }
+
+    subject { described_class.with_jira_issue_keys }
+
+    it { is_expected.to contain_exactly(mr_with_jira_title, mr_with_jira_description) }
+
+    it { is_expected.not_to include(mr_without_jira_reference) }
+  end
+
   describe '#squash_in_progress?' do
     let(:repo_path) do
       Gitlab::GitalyClient::StorageSettings.allow_disk_access do

spec/services/jira_connect_subscriptions/create_service_spec.rb

@@ -32,6 +32,36 @@ RSpec.describe JiraConnectSubscriptions::CreateService do
     it 'returns success' do
       expect(subject[:status]).to eq(:success)
     end
+
+    context 'namespace has projects' do
+      let!(:project_1) { create(:project, group: group) }
+      let!(:project_2) { create(:project, group: group) }
+
+      before do
+        stub_const("#{described_class}::MERGE_REQUEST_SYNC_BATCH_SIZE", 1)
+      end
+
+      it 'starts workers to sync projects in batches with delay' do
+        allow(Atlassian::JiraConnect::Client).to receive(:generate_update_sequence_id).and_return(123)
+
+        expect(JiraConnect::SyncProjectWorker).to receive(:bulk_perform_in).with(1.minute, [[project_1.id, 123]])
+        expect(JiraConnect::SyncProjectWorker).to receive(:bulk_perform_in).with(2.minutes, [[project_2.id, 123]])
+
+        subject
+      end
+
+      context 'when the jira_connect_full_namespace_sync feature flag is disabled' do
+        before do
+          stub_feature_flags(jira_connect_full_namespace_sync: false)
+        end
+
+        specify do
+          expect(JiraConnect::SyncProjectWorker).not_to receive(:bulk_perform_in_with_contexts)
+
+          subject
+        end
+      end
+    end
   end
 
   context 'when path is invalid' do

spec/workers/jira_connect/sync_project_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe JiraConnect::SyncProjectWorker, factory_default: :keep do
+  describe '#perform' do
+    let_it_be(:project) { create_default(:project) }
+    let!(:mr_with_jira_title) { create(:merge_request, :unique_branches, title: 'TEST-123') }
+    let!(:mr_with_jira_description) { create(:merge_request, :unique_branches, description: 'TEST-323') }
+    let!(:mr_with_other_title) { create(:merge_request, :unique_branches) }
+    let!(:jira_subscription) { create(:jira_connect_subscription, namespace: project.namespace) }
+
+    let(:jira_connect_sync_service) { JiraConnect::SyncService.new(project) }
+    let(:job_args) { [project.id, update_sequence_id] }
+    let(:update_sequence_id) { 1 }
+
+    before do
+      stub_request(:post, 'https://sample.atlassian.net/rest/devinfo/0.10/bulk').to_return(status: 200, body: '', headers: {})
+
+      jira_connect_sync_service
+      allow(JiraConnect::SyncService).to receive(:new) { jira_connect_sync_service }
+    end
+
+    context 'when the project is not found' do
+      it 'does not raise an error' do
+        expect { described_class.new.perform('non_existing_record_id', update_sequence_id) }.not_to raise_error
+      end
+    end
+
+    it 'avoids N+1 database queries' do
+      control_count = ActiveRecord::QueryRecorder.new { described_class.new.perform(project.id, update_sequence_id) }.count
+
+      create(:merge_request, :unique_branches, title: 'TEST-123')
+
+      expect { described_class.new.perform(project.id, update_sequence_id) }.not_to exceed_query_limit(control_count)
+    end
+
+    it_behaves_like 'an idempotent worker' do
+      let(:request_url) { 'https://sample.atlassian.net/rest/devinfo/0.10/bulk' }
+      let(:request_body) do
+        {
+          repositories: [
+            Atlassian::JiraConnect::Serializers::RepositoryEntity.represent(
+              project,
+              merge_requests: [mr_with_jira_description, mr_with_jira_title],
+              update_sequence_id: update_sequence_id
+            )
+          ]
+        }.to_json
+      end
+
+      it 'sends the request with custom update_sequence_id' do
+        expect(Atlassian::JiraConnect::Client).to receive(:post)
+          .exactly(IdempotentWorkerHelper::WORKER_EXEC_TIMES).times
+          .with(URI(request_url), headers: anything, body: request_body)
+
+        subject
+      end
+
+      context 'when the number of merge requests to sync is higher than the limit' do
+        let!(:most_recent_merge_request) { create(:merge_request, :unique_branches, description: 'TEST-323', title: 'TEST-123') }
+
+        before do
+          stub_const("#{described_class}::MERGE_REQUEST_LIMIT", 1)
+        end
+
+        it 'syncs only the most recent merge requests within the limit' do
+          expect(jira_connect_sync_service).to receive(:execute)
+            .exactly(IdempotentWorkerHelper::WORKER_EXEC_TIMES).times
+            .with(merge_requests: [most_recent_merge_request], update_sequence_id: update_sequence_id)
+
+          subject
+        end
+      end
+    end
+  end
+end