Merge branch '7495-listen-for-resolved-prometheus-alerts' into 'master'

Resolve "Listen for resolved Prometheus alerts"

Closes #7495

See merge request gitlab-org/gitlab-ee!7382

ee/app/controllers/projects/prometheus/alerts_controller.rb

+# frozen_string_literal: true
+
 module Projects
   module Prometheus
     class AlertsController < Projects::ApplicationController
@@ -22,9 +24,13 @@ module Projects
       end
 
       def notify
-        NotificationService.new.async.prometheus_alerts_fired(project, params["alerts"])
+        notify = Projects::Prometheus::Alerts::NotifyService.new(project, current_user, params)
 
-        head :ok
+        if notify.execute
+          head :ok
+        else
+          head :unprocessable_entity
+        end
       end
 
       def create

ee/app/services/clusters/applications/prometheus_update_service.rb

@@ -96,7 +96,7 @@ module Clusters
             "webhook_configs" => [
               {
                 "url" => notify_url,
-                "send_resolved" => false
+                "send_resolved" => true
               }
             ]
           }

ee/app/services/projects/prometheus/alerts/notify_service.rb

+# frozen_string_literal: true
+
+module Projects
+  module Prometheus
+    module Alerts
+      class NotifyService < BaseService
+        def execute
+          return false unless valid?
+
+          notification_service.async.prometheus_alerts_fired(project, firings) if firings.any?
+
+          true
+        end
+
+        private
+
+        def firings
+          @firings ||= alerts_by_status('firing')
+        end
+
+        def alerts_by_status(status)
+          params['alerts'].select { |alert| alert['status'] == status }
+        end
+
+        def valid?
+          params['version'] == '4'
+        end
+      end
+    end
+  end
+end

ee/changelogs/unreleased/7495-listen-for-resolved-prometheus-alerts.yml

+---
+title: Listen for resolved Prometheus alerts
+merge_request: 7382
+author:
+type: changed

ee/spec/controllers/projects/prometheus/alerts_controller_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe Projects::Prometheus::AlertsController do
@@ -83,31 +85,28 @@ describe Projects::Prometheus::AlertsController do
   end
 
   describe 'POST #notify' do
-    it 'sends a notification' do
-      alert = create(:prometheus_alert, project: project, environment: environment, prometheus_metric: metric)
-      notification_service = spy
+    let(:notify_service) { spy }
+    let(:payload) { {} }
 
-      alert_params = {
-        "alert" => alert.title,
-        "expr" => "#{alert.query} #{alert.computed_operator} #{alert.threshold}",
-        "for" => "5m",
-        "labels" => {
-          "gitlab" => "hook",
-          "gitlab_alert_id" => alert.prometheus_metric_id
-        }
-      }
+    before do
+      allow(Projects::Prometheus::Alerts::NotifyService).to receive(:new).and_return(notify_service)
+    end
 
-      allow(NotificationService).to receive(:new).and_return(notification_service)
-      expect(notification_service).to receive_message_chain(:async, :prometheus_alerts_fired).with(project, [alert_params])
+    it 'sends a notification for firing alerts only' do
+      expect(notify_service).to receive(:execute).and_return(true)
 
-      if Gitlab.rails5?
-        post :notify, params: project_params(alerts: [alert.to_param]), as: :json
-      else
-        post :notify, project_params(alerts: [alert]), as: :json
-      end
+      post :notify, project_params(payload), as: :json
 
       expect(response).to have_gitlab_http_status(200)
     end
+
+    it 'renders unprocessable entity if notification fails' do
+      expect(notify_service).to receive(:execute).and_return(false)
+
+      post :notify, project_params, as: :json
+
+      expect(response).to have_gitlab_http_status(422)
+    end
   end
 
   describe 'POST #create' do

ee/spec/models/prometheus_alert_spec.rb

+# frozen_string_literal: true
+
 require 'spec_helper'
 
 describe PrometheusAlert do

ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::Prometheus::Alerts::NotifyService do
+  let(:user) { create(:user) }
+  let(:project) { create(:project) }
+  let(:service) { described_class.new(project, user, payload) }
+
+  context 'with valid payload' do
+    let(:alert_firing) { create(:prometheus_alert, project: project) }
+    let(:alert_resolved) { create(:prometheus_alert, project: project) }
+    let(:notification_service) { spy }
+    let(:payload) { payload_for(firing: [alert_firing], resolved: [alert_resolved]) }
+    let(:payload_alert_firing) { payload['alerts'].first }
+
+    before do
+      allow(NotificationService).to receive(:new).and_return(notification_service)
+    end
+
+    it 'sends a notification for firing alerts only' do
+      expect(notification_service)
+        .to receive_message_chain(:async, :prometheus_alerts_fired)
+        .with(project, [payload_alert_firing])
+
+      expect(service.execute).to eq(true)
+    end
+  end
+
+  context 'with invalid payload' do
+    let(:payload) { {} }
+
+    it 'returns false without `version`' do
+      expect(service.execute).to eq(false)
+    end
+
+    it 'returns false if `version` is not 4' do
+      payload['version'] = '5'
+
+      expect(service.execute).to eq(false)
+    end
+  end
+
+  private
+
+  def payload_for(firing: [], resolved: [])
+    status = firing.any? ? 'firing' : 'resolved'
+    alerts = firing + resolved
+    alert_name = alerts.first.title
+    prometheus_metric_id = alerts.first.prometheus_metric_id.to_s
+
+    alerts_map = \
+      firing.map { |alert| map_alert_payload('firing', alert) } +
+      resolved.map { |alert| map_alert_payload('resolved', alert) }
+
+    # See https://prometheus.io/docs/alerting/configuration/#%3Cwebhook_config%3E
+    {
+      'version' => '4',
+      'receiver' => 'gitlab',
+      'status' => status,
+      'alerts' => alerts_map,
+      'groupLabels' => {
+        'alertname' => alert_name
+      },
+      'commonLabels' => {
+        'alertname' => alert_name,
+        'gitlab' => 'hook',
+        'gitlab_alert_id' => prometheus_metric_id
+      },
+      'commonAnnotations' => {},
+      'externalURL' => '',
+      'groupKey' => "{}:{alertname=\'#{alert_name}\'}"
+    }
+  end
+
+  def map_alert_payload(status, alert)
+    {
+      'status' => status,
+      'labels' => {
+        'alertname' => alert.title,
+        'gitlab' => 'hook',
+        'gitlab_alert_id' => alert.prometheus_metric_id.to_s
+      },
+      'annotations' => {},
+      'startsAt' => '2018-09-24T08:57:31.095725221Z',
+      'endsAt' => '0001-01-01T00:00:00Z',
+      'generatorURL' => 'http://prometheus-prometheus-server-URL'
+    }
+  end
+end