Merge branch 'sarnold-fix-prometheus-issue-creation' into 'master'

Create issue automatically from Prometheus alert

Closes #231497

See merge request gitlab-org/gitlab!37884

app/services/alert_management/process_prometheus_alert_service.rb

@@ -34,6 +34,8 @@ module AlertManagement
       else
         create_alert_management_alert
       end
+
+      process_incident_alert
     end
 
     def reset_alert_management_alert_status
@@ -47,16 +49,17 @@ module AlertManagement
     end
 
     def create_alert_management_alert
-      am_alert = AlertManagement::Alert.new(am_alert_params.merge(ended_at: nil))
-      if am_alert.save
-        am_alert.execute_services
+      new_alert = AlertManagement::Alert.new(am_alert_params.merge(ended_at: nil))
+      if new_alert.save
+        new_alert.execute_services
+        @am_alert = new_alert
         return
       end
 
       logger.warn(
         message: 'Unable to create AlertManagement::Alert',
         project_id: project.id,
-        alert_errors: am_alert.errors.messages
+        alert_errors: new_alert.errors.messages
       )
     end
 
@@ -89,12 +92,21 @@ module AlertManagement
       SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed?
     end
 
+    def process_incident_alert
+      return unless am_alert
+      return if am_alert.issue
+
+      IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, am_alert.id)
+    end
+
     def logger
       @logger ||= Gitlab::AppLogger
     end
 
     def am_alert
-      @am_alert ||= AlertManagement::Alert.not_resolved.for_fingerprint(project, gitlab_fingerprint).first
+      strong_memoize(:am_alert) do
+        AlertManagement::Alert.not_resolved.for_fingerprint(project, gitlab_fingerprint).first
+      end
     end
 
     def bad_request

app/workers/incident_management/process_alert_worker.rb

@@ -30,7 +30,11 @@ module IncidentManagement
     end
 
     def parsed_payload(alert)
-      Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project)
+      if alert.prometheus?
+        alert.payload
+      else
+        Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project)
+      end
     end
 
     def create_issue_for(alert)

changelogs/unreleased/sarnold-fix-prometheus-issue-creation.yml

+---
+title: Fix automatic issue creation via Prometheus alerts
+merge_request: 37884
+author:
+type: fixed

spec/factories/alert_management/alerts.rb

@@ -101,6 +101,16 @@ FactoryBot.define do
 
     trait :prometheus do
       monitoring_tool { Gitlab::AlertManagement::AlertParams::MONITORING_TOOLS[:prometheus] }
+      payload do
+        {
+          annotations: {
+            title: 'This is a prometheus error',
+            summary: 'Summary of the error',
+            description: 'Description of the error'
+          },
+          startsAt: started_at
+        }.with_indifferent_access
+      end
     end
 
     trait :all_fields do

spec/services/alert_management/process_prometheus_alert_service_spec.rb

@@ -83,6 +83,15 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
         context 'when alert does not exist' do
           context 'when alert can be created' do
             it_behaves_like 'creates an alert management alert'
+
+            it 'processes the incident alert' do
+              expect(IncidentManagement::ProcessAlertWorker)
+                .to receive(:perform_async)
+                .with(nil, nil, kind_of(Integer))
+                .once
+
+              expect(subject).to be_success
+            end
           end
 
           context 'when alert cannot be created' do
@@ -102,6 +111,13 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
 
               execute
             end
+
+            it 'does not create incident issue' do
+              expect(IncidentManagement::ProcessAlertWorker)
+                .not_to receive(:perform_async)
+
+              expect(subject).to be_success
+            end
           end
 
           it { is_expected.to be_success }

spec/workers/incident_management/process_alert_worker_spec.rb

@@ -16,66 +16,73 @@ RSpec.describe IncidentManagement::ProcessAlertWorker do
     subject { described_class.new.perform(nil, nil, alert.id) }
 
     before do
+      allow(Gitlab::AppLogger).to receive(:warn).and_call_original
+
       allow(IncidentManagement::CreateIssueService)
         .to receive(:new).with(alert.project, parsed_payload)
         .and_call_original
     end
 
-    it 'creates an issue' do
-      expect(IncidentManagement::CreateIssueService)
-        .to receive(:new).with(alert.project, parsed_payload)
+    shared_examples 'creates issue successfully' do
+      it 'creates an issue' do
+        expect(IncidentManagement::CreateIssueService)
+          .to receive(:new).with(alert.project, parsed_payload)
 
-      expect { subject }.to change { Issue.count }.by(1)
-    end
+        expect { subject }.to change { Issue.count }.by(1)
+      end
 
-    context 'with invalid alert' do
-      let(:invalid_alert_id) { non_existing_record_id }
+      it 'updates AlertManagement::Alert#issue_id' do
+        subject
 
-      subject { described_class.new.perform(nil, nil, invalid_alert_id) }
+        expect(alert.reload.issue_id).to eq(created_issue.id)
+      end
 
-      it 'does not create issues' do
-        expect(IncidentManagement::CreateIssueService).not_to receive(:new)
+      it 'does not write a warning to log' do
+        subject
 
-        expect { subject }.not_to change { Issue.count }
+        expect(Gitlab::AppLogger).not_to have_received(:warn)
       end
     end
 
     context 'with valid alert' do
-      before do
-        allow(Gitlab::AppLogger).to receive(:warn).and_call_original
-      end
+      it_behaves_like 'creates issue successfully'
 
-      context 'when alert can be updated' do
-        it 'updates AlertManagement::Alert#issue_id' do
-          subject
+      context 'when alert cannot be updated' do
+        let_it_be(:alert) { create(:alert_management_alert, :with_validation_errors, project: project, payload: payload) }
 
-          expect(alert.reload.issue_id).to eq(created_issue.id)
+        it 'updates AlertManagement::Alert#issue_id' do
+          expect { subject }.not_to change { alert.reload.issue_id }
         end
 
-        it 'does not write a warning to log' do
+        it 'logs a warning' do
           subject
 
-          expect(Gitlab::AppLogger).not_to have_received(:warn)
+          expect(Gitlab::AppLogger).to have_received(:warn).with(
+            message: 'Cannot link an Issue with Alert',
+            issue_id: created_issue.id,
+            alert_id: alert.id,
+            alert_errors: { hosts: ['hosts array is over 255 chars'] }
+          )
         end
+      end
 
-        context 'when alert cannot be updated' do
-          let_it_be(:alert) { create(:alert_management_alert, :with_validation_errors, project: project, payload: payload) }
+      context 'prometheus alert' do
+        let_it_be(:alert) { create(:alert_management_alert, :prometheus, project: project, started_at: started_at) }
+        let_it_be(:parsed_payload) { alert.payload }
 
-          it 'updates AlertManagement::Alert#issue_id' do
-            expect { subject }.not_to change { alert.reload.issue_id }
-          end
+        it_behaves_like 'creates issue successfully'
+      end
+    end
 
-          it 'logs a warning' do
-            subject
+    context 'with invalid alert' do
+      let(:invalid_alert_id) { non_existing_record_id }
 
-            expect(Gitlab::AppLogger).to have_received(:warn).with(
-              message: 'Cannot link an Issue with Alert',
-              issue_id: created_issue.id,
-              alert_id: alert.id,
-              alert_errors: { hosts: ['hosts array is over 255 chars'] }
-            )
-          end
-        end
+      subject { described_class.new.perform(nil, nil, invalid_alert_id) }
+
+      it 'does not create issues' do
+        expect(IncidentManagement::CreateIssueService).not_to receive(:new)
+
+        expect { subject }.not_to change { Issue.count }
       end
     end
   end