Merge branch '212948-jira-importer-labels' into 'master'

Map labels from Jira to labels in GitLab

See merge request gitlab-org/gitlab!29970

changelogs/unreleased/212948-jira-importer-labels.yml

+---
+title: Map labels from Jira to labels in GitLab
+merge_request: 29970
+author:
+type: added

lib/gitlab/jira_import/handle_labels_service.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    class HandleLabelsService
+      def initialize(project, jira_labels)
+        @project = project
+        @jira_labels = jira_labels
+      end
+
+      def execute
+        return if jira_labels.blank?
+
+        existing_labels = LabelsFinder.new(nil, project: project, title: jira_labels)
+          .execute(skip_authorization: true).select(:id, :name)
+        new_labels = create_missing_labels(existing_labels)
+
+        label_ids = existing_labels.map(&:id)
+        label_ids += new_labels if new_labels.present?
+        label_ids
+      end
+
+      private
+
+      attr_reader :project, :jira_labels
+
+      def create_missing_labels(existing_labels)
+        labels_to_create = jira_labels - existing_labels.map(&:name)
+        return if labels_to_create.empty?
+
+        new_labels_hash = labels_to_create.map do |title|
+          { project_id: project.id, title: title, type: 'ProjectLabel' }
+        end
+
+        Label.insert_all(new_labels_hash).rows.flatten
+      end
+    end
+  end
+end

lib/gitlab/jira_import/issue_serializer.rb

@@ -21,7 +21,8 @@ module Gitlab
           state_id: map_status(jira_issue.status.statusCategory),
           updated_at: jira_issue.updated,
           created_at: jira_issue.created,
-          author_id: project.creator_id # TODO: map actual author: https://gitlab.com/gitlab-org/gitlab/-/issues/210580
+          author_id: project.creator_id, # TODO: map actual author: https://gitlab.com/gitlab-org/gitlab/-/issues/210580
+          label_ids: label_ids
         }
       end
 
@@ -49,6 +50,15 @@ module Gitlab
           Issuable::STATE_ID_MAP[:opened]
         end
       end
+
+      # We already create labels in Gitlab::JiraImport::LabelsImporter stage but
+      # there is a possibility it may fail or
+      # new labels were created on the Jira in the meantime
+      def label_ids
+        return if jira_issue.fields['labels'].blank?
+
+        Gitlab::JiraImport::HandleLabelsService.new(project, jira_issue.fields['labels']).execute
+      end
     end
   end
 end

lib/gitlab/jira_import/labels_importer.rb

@@ -5,6 +5,8 @@ module Gitlab
     class LabelsImporter < BaseImporter
       attr_reader :job_waiter
 
+      MAX_LABELS = 500
+
       def initialize(project)
         super
         @job_waiter = JobWaiter.new
@@ -25,9 +27,29 @@ module Gitlab
       end
 
       def import_jira_labels
-        # todo: import jira labels, see https://gitlab.com/gitlab-org/gitlab/-/issues/212651
+        start_at = 0
+        loop do
+          break if process_jira_page(start_at)
+
+          start_at += MAX_LABELS
+        end
+
         job_waiter
       end
+
+      def process_jira_page(start_at)
+        request = "/rest/api/2/label?maxResults=#{MAX_LABELS}&startAt=#{start_at}"
+        response = JSON.parse(client.get(request))
+
+        return true if response['values'].blank?
+        return true unless response.key?('isLast')
+
+        Gitlab::JiraImport::HandleLabelsService.new(project, response['values']).execute
+
+        response['isLast']
+      rescue => e
+        Gitlab::ErrorTracking.track_exception(e, project_id: project.id, request: request)
+      end
     end
   end
 end

lib/gitlab/jira_import/metadata_collector.rb

@@ -13,7 +13,6 @@ module Gitlab
       def execute
         add_field(%w(issuetype name), 'Issue type')
         add_field(%w(priority name), 'Priority')
-        add_labels
         add_field('environment', 'Environment')
         add_field('duedate', 'Due date')
         add_parent
@@ -33,12 +32,6 @@ module Gitlab
         metadata << "- #{field_label}: #{value}"
       end
 
-      def add_labels
-        return if fields['labels'].blank? || !fields['labels'].is_a?(Array)
-
-        metadata << "- Labels: #{fields['labels'].join(', ')}"
-      end
-
       def add_parent
         parent_issue_key = fields.dig('parent', 'key')
 

spec/lib/gitlab/jira_import/handle_labels_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::JiraImport::HandleLabelsService do
+  describe '#execute' do
+    let_it_be(:group)   { create(:group) }
+    let_it_be(:project) { create(:project, group: group) }
+
+    let_it_be(:project_label)       { create(:label, project: project, title: 'bug') }
+    let_it_be(:other_project_label) { create(:label, title: 'feature') }
+    let_it_be(:group_label)         { create(:group_label, group: group, title: 'dev') }
+    let(:jira_labels)               { %w(bug feature dev group::new) }
+
+    subject { described_class.new(project, jira_labels).execute }
+
+    context 'when some provided jira labels are missing' do
+      def created_labels
+        project.labels.reorder(id: :desc).first(2)
+      end
+
+      it 'creates the missing labels on the project level' do
+        expect { subject }.to change { Label.count }.from(3).to(5)
+
+        expect(created_labels.map(&:title)).to match_array(%w(feature group::new))
+      end
+
+      it 'returns the id of all labels matching the title' do
+        expect(subject).to match_array([project_label.id, group_label.id] + created_labels.map(&:id))
+      end
+    end
+
+    context 'when no provided jira labels are missing' do
+      let(:jira_labels) { %w(bug dev) }
+
+      it 'does not create any new labels' do
+        expect { subject }.not_to change { Label.count }.from(3)
+      end
+
+      it 'returns the id of all labels matching the title' do
+        expect(subject).to match_array([project_label.id, group_label.id])
+      end
+    end
+
+    context 'when no labels are provided' do
+      let(:jira_labels) { [] }
+
+      it 'does not create any new labels' do
+        expect { subject }.not_to change { Label.count }.from(3)
+      end
+    end
+  end
+end

spec/lib/gitlab/jira_import/issue_serializer_spec.rb

@@ -4,7 +4,11 @@ require 'spec_helper'
 
 describe Gitlab::JiraImport::IssueSerializer do
   describe '#execute' do
-    let_it_be(:project) { create(:project) }
+    let_it_be(:group) { create(:group) }
+    let_it_be(:project) { create(:project, group: group) }
+    let_it_be(:project_label) { create(:label, project: project, title: 'bug') }
+    let_it_be(:other_project_label) { create(:label, project: project, title: 'feature') }
+    let_it_be(:group_label) { create(:group_label, group: group, title: 'dev') }
 
     let(:iid) { 5 }
     let(:key) { 'PROJECT-5' }
@@ -19,11 +23,13 @@ describe Gitlab::JiraImport::IssueSerializer do
       { 'key' => 'FOO-2', 'id' => '1050', 'fields' => { 'summary' => 'parent issue FOO' } }
     end
     let(:priority_field) { { 'name' => 'Medium' } }
+    let(:labels_field) { %w(bug dev backend frontend) }
 
     let(:fields) do
       {
         'parent' => parent_field,
-        'priority' => priority_field
+        'priority' => priority_field,
+        'labels' => labels_field
       }
     end
 
@@ -73,9 +79,33 @@ describe Gitlab::JiraImport::IssueSerializer do
           state_id: 1,
           updated_at: updated_at,
           created_at: created_at,
-          author_id: project.creator_id
+          author_id: project.creator_id,
+          label_ids: [project_label.id, group_label.id] + Label.reorder(id: :asc).last(2).pluck(:id)
         )
       end
+
+      it 'creates a hash for valid issue' do
+        expect(Issue.new(subject)).to be_valid
+      end
+
+      it 'creates all missing labels (on project level)' do
+        expect { subject }.to change { Label.count }.from(3).to(5)
+
+        expect(Label.find_by(title: 'frontend').project).to eq(project)
+        expect(Label.find_by(title: 'backend').project).to eq(project)
+      end
+
+      context 'when there are no new labels' do
+        let(:labels_field) { %w(bug dev) }
+
+        it 'assigns the labels to the Issue hash' do
+          expect(subject[:label_ids]).to match_array([project_label.id, group_label.id])
+        end
+
+        it 'does not create new labels' do
+          expect { subject }.not_to change { Label.count }.from(3)
+        end
+      end
     end
 
     context 'with done status' do

spec/lib/gitlab/jira_import/labels_importer_spec.rb

@@ -3,18 +3,23 @@
 require 'spec_helper'
 
 describe Gitlab::JiraImport::LabelsImporter do
-  let_it_be(:user) { create(:user) }
-  let_it_be(:project) { create(:project) }
+  let_it_be(:user)         { create(:user) }
+  let_it_be(:group)        { create(:group) }
+  let_it_be(:project)      { create(:project, group: group) }
   let_it_be(:jira_service) { create(:jira_service, project: project) }
 
   subject { described_class.new(project).execute }
 
   before do
     stub_feature_flags(jira_issue_import: true)
+    stub_const('Gitlab::JiraImport::LabelsImporter::MAX_LABELS', 2)
+
+    WebMock.stub_request(:get, 'https://jira.example.com/rest/api/2/serverInfo')
+      .to_return(body: { url: 'http://url' }.to_json )
   end
 
   describe '#execute', :clean_gitlab_redis_cache do
-    context 'when label is missing from jira import' do
+    context 'when jira import label is missing from jira import' do
       let_it_be(:no_label_jira_import) { create(:jira_import_state, label: nil, project: project) }
 
       it 'raises error' do
@@ -22,16 +27,71 @@ describe Gitlab::JiraImport::LabelsImporter do
       end
     end
 
-    context 'when label exists' do
-      let_it_be(:label) { create(:label) }
+    context 'when jira import label exists' do
+      let_it_be(:label)                  { create(:label) }
       let_it_be(:jira_import_with_label) { create(:jira_import_state, label: label, project: project) }
+      let_it_be(:issue_label)            { create(:label, project: project, title: 'bug') }
+
+      let(:jira_labels_1) { { "maxResults" => 2, "startAt" => 0, "total" => 3, "isLast" => false, "values" => %w(backend bug) } }
+      let(:jira_labels_2) { { "maxResults" => 2, "startAt" => 2, "total" => 3, "isLast" => true, "values" => %w(feature) } }
+
+      before do
+        WebMock.stub_request(:get, 'https://jira.example.com/rest/api/2/label?maxResults=2&startAt=0')
+          .to_return(body: jira_labels_1.to_json )
+        WebMock.stub_request(:get, 'https://jira.example.com/rest/api/2/label?maxResults=2&startAt=2')
+          .to_return(body: jira_labels_2.to_json )
+      end
+
+      context 'when labels are returned from jira' do
+        it 'caches import label' do
+          expect(Gitlab::Cache::Import::Caching.read(Gitlab::JiraImport.import_label_cache_key(project.id))).to be nil
+
+          subject
+
+          expect(Gitlab::JiraImport.get_import_label_id(project.id).to_i).to eq(label.id)
+        end
+
+        it 'calls Gitlab::JiraImport::HandleLabelsService' do
+          expect(Gitlab::JiraImport::HandleLabelsService).to receive(:new).with(project, %w(backend bug)).and_return(double(execute: [1, 2]))
+          expect(Gitlab::JiraImport::HandleLabelsService).to receive(:new).with(project, %w(feature)).and_return(double(execute: [3]))
+
+          subject
+        end
+      end
+
+      context 'when there are no labels to be handled' do
+        shared_examples 'no labels handling' do
+          it 'does not call Gitlab::JiraImport::HandleLabelsService' do
+            expect(Gitlab::JiraImport::HandleLabelsService).not_to receive(:new)
+
+            subject
+          end
+        end
+
+        let(:jira_labels) { { "maxResults" => 2, "startAt" => 0, "total" => 3, "values" => [] } }
+
+        before do
+          WebMock.stub_request(:get, 'https://jira.example.com/rest/api/2/label?maxResults=2&startAt=0')
+            .to_return(body: jira_labels.to_json )
+        end
+
+        context 'when the labels field is empty' do
+          let(:jira_labels) { { "maxResults" => 2, "startAt" => 0, "isLast" => true, "total" => 3, "values" => [] } }
+
+          it_behaves_like 'no labels handling'
+        end
+
+        context 'when the labels field is missing' do
+          let(:jira_labels) { { "maxResults" => 2, "startAt" => 0, "isLast" => true, "total" => 3 } }
 
-      it 'caches import label' do
-        expect(Gitlab::Cache::Import::Caching.read(Gitlab::JiraImport.import_label_cache_key(project.id))).to be nil
+          it_behaves_like 'no labels handling'
+        end
 
-        subject
+        context 'when the isLast argument is missing' do
+          let(:jira_labels) { { "maxResults" => 2, "startAt" => 0, "total" => 3, "values" => %w(bug dev) } }
 
-        expect(Gitlab::JiraImport.get_import_label_id(project.id).to_i).to eq(label.id)
+          it_behaves_like 'no labels handling'
+        end
       end
     end
   end

spec/lib/gitlab/jira_import/metadata_collector_spec.rb

@@ -9,7 +9,6 @@ describe Gitlab::JiraImport::MetadataCollector do
     let(:description) { 'basic description' }
     let(:created_at) { '2020-01-01 20:00:00' }
     let(:updated_at) { '2020-01-10 20:00:00' }
-    let(:assignee) { double(displayName: 'Solver') }
     let(:jira_status) { 'new' }
 
     let(:parent_field) do
@@ -18,7 +17,6 @@ describe Gitlab::JiraImport::MetadataCollector do
     let(:issue_type_field) { { 'name' => 'Task' } }
     let(:fix_versions_field) { [{ 'name' => '1.0' }, { 'name' => '1.1' }] }
     let(:priority_field) { { 'name' => 'Medium' } }
-    let(:labels_field) { %w(bug backend) }
     let(:environment_field) { 'staging' }
     let(:duedate_field) { '2020-03-01' }
 
@@ -28,7 +26,6 @@ describe Gitlab::JiraImport::MetadataCollector do
         'issuetype' => issue_type_field,
         'fixVersions' => fix_versions_field,
         'priority' => priority_field,
-        'labels' => labels_field,
         'environment' => environment_field,
         'duedate' => duedate_field
       }
@@ -41,8 +38,6 @@ describe Gitlab::JiraImport::MetadataCollector do
         description: description,
         created: created_at,
         updated: updated_at,
-        assignee: assignee,
-        reporter: double(displayName: 'Reporter'),
         status: double(statusCategory: { 'key' => jira_status }),
         fields: fields
       )
@@ -59,7 +54,6 @@ describe Gitlab::JiraImport::MetadataCollector do
 
           - Issue type: Task
           - Priority: Medium
-          - Labels: bug, backend
           - Environment: staging
           - Due date: 2020-03-01
           - Parent issue: [FOO-2] parent issue FOO
@@ -71,11 +65,9 @@ describe Gitlab::JiraImport::MetadataCollector do
     end
 
     context 'when some fields are in incorrect format' do
-      let(:assignee) { nil }
       let(:parent_field) { nil }
       let(:fix_versions_field) { [] }
       let(:priority_field) { nil }
-      let(:labels_field) { [] }
       let(:environment_field) { nil }
       let(:duedate_field) { nil }
 
@@ -112,22 +104,6 @@ describe Gitlab::JiraImport::MetadataCollector do
         end
       end
 
-      context 'when a labels field is not an array' do
-        let(:labels_field) { { 'first' => 'bug' } }
-
-        it 'skips the labels' do
-          expected_result = <<~MD
-            ---
-
-            **Issue metadata**
-
-            - Issue type: Task
-          MD
-
-          expect(subject.strip).to eq(expected_result.strip)
-        end
-      end
-
       context 'when a parent field has incorrectly formatted summary' do
         let(:parent_field) do
           { 'key' => 'FOO-2', 'id' => '1050', 'other_field' => { 'summary' => 'parent issue FOO' } }
@@ -167,10 +143,8 @@ describe Gitlab::JiraImport::MetadataCollector do
     end
 
     context 'when some metadata fields are missing' do
-      let(:assignee) { nil }
       let(:parent_field) { nil }
       let(:fix_versions_field) { [] }
-      let(:labels_field) { [] }
       let(:environment_field) { nil }
 
       it 'skips the missing fields' do
@@ -189,12 +163,10 @@ describe Gitlab::JiraImport::MetadataCollector do
     end
 
     context 'when all metadata fields are missing' do
-      let(:assignee) { nil }
       let(:parent_field) { nil }
       let(:issue_type_field) { nil }
       let(:fix_versions_field) { [] }
       let(:priority_field) { nil }
-      let(:labels_field) { [] }
       let(:environment_field) { nil }
       let(:duedate_field) { nil }
 

spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb

@@ -37,6 +37,9 @@ describe Gitlab::JiraImport::Stage::ImportLabelsWorker do
 
         before do
           jira_import.start!
+
+          WebMock.stub_request(:get, 'https://jira.example.com/rest/api/2/label?maxResults=500&startAt=0')
+            .to_return(body: {}.to_json )
         end
 
         it_behaves_like 'advance to next stage', :issues