Add support for embedding all prometheus alerts

Adds functionality to incidenment management issue creation
to automatically include a metrics embed in issues created
for alerts which come from manually configured prometheus
instances.

app/presenters/projects/prometheus/alert_presenter.rb

@@ -3,7 +3,7 @@
 module Projects
   module Prometheus
     class AlertPresenter < Gitlab::View::Presenter::Delegated
-      RESERVED_ANNOTATIONS = %w(gitlab_incident_markdown title).freeze
+      RESERVED_ANNOTATIONS = %w(gitlab_incident_markdown gitlab_y_label title).freeze
       GENERIC_ALERT_SUMMARY_ANNOTATIONS = %w(monitoring_tool service hosts).freeze
       MARKDOWN_LINE_BREAK = "  \n".freeze
       INCIDENT_LABEL_NAME = IncidentManagement::CreateIssueService::INCIDENT_LABEL[:title].freeze

doc/user/incident_management/index.md

@@ -138,6 +138,4 @@ Incident Management features can be easily enabled & disabled via the Project se
 
 #### Auto-creation
 
-GitLab Issues can automatically be created as a result of an Alert notification. An Issue created this way will contain error information to help you further debug the error.
-
-For [GitLab-managed alerting rules](../project/integrations/prometheus.md#setting-up-alerts-for-prometheus-metrics-ultimate), the issue will include an embedded chart for the query corresponding to the alert. The chart will show an hour of data surrounding the starting point of the incident, 30 minutes before and after.
+You can automatically create GitLab issues from an Alert notification. Issues created this way contain error information to help you debug the error. Appropriately configured alerts include an [embedded chart](../project/integrations/prometheus.md#embedding-metrics-based-on-alerts-in-incident-issues) for the query corresponding to the alert.

doc/user/project/integrations/prometheus.md

@@ -802,6 +802,22 @@ It is also possible to embed either the default dashboard metrics or individual
 
 ![Embedded Metrics in issue templates](img/embed_metrics_issue_template.png)
 
+### Embedding metrics based on alerts in incident issues
+
+For [GitLab-managed alerting rules](#setting-up-alerts-for-prometheus-metrics-ultimate), the issue will include an embedded chart for the query corresponding to the alert. The chart displays an hour of data surrounding the starting point of the incident, 30 minutes before and after.
+
+For [manually configured Prometheus instances](#manual-configuration-of-prometheus), a chart corresponding to the query can be included if these requirements are met:
+
+- The alert corresponds to an environment managed through GitLab.
+- The alert corresponds to a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries).
+- The alert contains the required attributes listed in the chart below.
+
+| Attributes | Required | Description |
+| ---------- | -------- | ----------- |
+| `annotations/gitlab_environment_name` | Yes | Name of the GitLab-managed environment corresponding to the alert |
+| One of `annotations/title`, `annotations/summary`, `labels/alertname` | Yes | Will be used as the chart title |
+| `annotations/gitlab_y_label` | No | Will be used as the chart's y-axis label |
+
 ### Embedding Cluster Health Charts **(ULTIMATE)**
 
 > [Introduced](<https://gitlab.com/gitlab-org/gitlab/issues/40997>) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9.

ee/app/presenters/ee/projects/prometheus/alert_presenter.rb

@@ -10,22 +10,63 @@ module EE
 
         override :metric_embed_for_alert
         def metric_embed_for_alert
+          url = embed_url_for_gitlab_alert || embed_url_for_self_managed_alert
+
+          "\n[](#{url})" if url
+        end
+
+        private
+
+        def embed_url_for_gitlab_alert
           return unless gitlab_alert
 
-          time = starts_at ? Time.rfc3339(starts_at) : Time.current
-          url = metrics_dashboard_project_prometheus_alert_url(
+          metrics_dashboard_project_prometheus_alert_url(
             project,
             gitlab_alert.prometheus_metric_id,
             environment_id: environment.id,
+            **alert_embed_window_params(embed_time)
+          )
+        end
+
+        def embed_url_for_self_managed_alert
+          return unless environment && full_query && title
+
+          metrics_dashboard_project_environment_url(
+            project,
+            environment,
+            embed_json: dashboard_for_self_managed_alert.to_json,
+            **alert_embed_window_params(embed_time)
+          )
+        end
+
+        def embed_time
+          starts_at ? Time.rfc3339(starts_at) : Time.current
+        end
+
+        def alert_embed_window_params(time)
+          {
             start: format_embed_timestamp(time - METRIC_TIME_WINDOW),
             end: format_embed_timestamp(time + METRIC_TIME_WINDOW)
-          )
+          }
+        end
 
-          "\n[](#{url})"
+        def format_embed_timestamp(time)
+          time.utc.strftime('%FT%TZ')
         end
 
-        def format_embed_timestamp(timestamp)
-          timestamp.utc.strftime('%FT%TZ')
+        def dashboard_for_self_managed_alert
+          {
+            panel_groups: [{
+              panels: [{
+                type: 'line-graph',
+                title: title,
+                y_label: y_label,
+                metrics: [{
+                  query_range: full_query
+                }]
+              }]
+            }]
+          }
         end
       end
     end

ee/changelogs/unreleased/sy-auto-embed-alert.yml

+---
+title: Automatically embed metrics in issues for alerts from manually configured Prometheus instances
+merge_request: 28622
+author:
+type: added

ee/spec/presenters/ee/projects/prometheus/alert_presenter_spec.rb

@@ -8,18 +8,105 @@ describe Projects::Prometheus::AlertPresenter do
   let(:presenter) { described_class.new(alert) }
   let(:payload) { {} }
   let(:alert) { create(:alerting_alert, project: project, payload: payload) }
+  let(:markdown_line_break) { '  ' }
+  let(:starts_at) { '2018-03-12T09:06:00Z' }
 
   describe '#issue_summary_markdown' do
-    let(:markdown_line_break) { '  ' }
+    shared_examples_for 'markdown with metrics embed' do
+      let(:expected_markdown) do
+        <<~MARKDOWN.chomp
+        #### Summary
+
+        **Start time:** #{presenter.starts_at}#{markdown_line_break}
+        **full_query:** `avg(metric) > 1.0`
+
+        [](#{url})
+        MARKDOWN
+      end
+
+      context 'without a starting time available' do
+        around do |example|
+          Timecop.freeze(starts_at) { example.run }
+        end
+
+        it { is_expected.to eq(expected_markdown) }
+      end
+
+      context 'with a starting time available' do
+        before do
+          payload['startsAt'] = starts_at
+        end
+
+        it { is_expected.to eq(expected_markdown) }
+      end
+    end
 
     subject { presenter.issue_summary_markdown }
 
-    context 'with gitlab alert' do
+    context 'for gitlab-managed prometheus alerts' do
       let(:gitlab_alert) { create(:prometheus_alert, project: project) }
       let(:metric_id) { gitlab_alert.prometheus_metric_id }
       let(:env_id) { gitlab_alert.environment_id }
-      let(:starts_at) { '2018-03-12T09:06:00Z' }
 
+      before do
+        payload['labels'] = { 'gitlab_alert_id' => metric_id }
+      end
+
+      let(:url) { "http://localhost/#{project.full_path}/prometheus/alerts/#{metric_id}/metrics_dashboard?end=2018-03-12T09%3A36%3A00Z&environment_id=#{env_id}&start=2018-03-12T08%3A36%3A00Z" }
+
+      it_behaves_like 'markdown with metrics embed'
+    end
+
+    context 'for alerts from a self-managed prometheus' do
+      let!(:environment) { create(:environment, project: project, name: 'production') }
+      let(:url) { "http://localhost/#{project.full_path}/-/environments/#{environment.id}/metrics_dashboard?embed_json=#{CGI.escape(embed_content.to_json)}&end=2018-03-12T09%3A36%3A00Z&start=2018-03-12T08%3A36%3A00Z" }
+
+      let(:title) { 'title' }
+      let(:y_label) { 'y_label' }
+      let(:query) { 'avg(metric) > 1.0' }
+      let(:embed_content) do
+        {
+          panel_groups: [{
+            panels: [{
+              type: 'line-graph',
+              title: title,
+              y_label: y_label,
+              metrics: [{ query_range: query }]
+            }]
+          }]
+        }
+      end
+
+      before do
+        # Setup embed time range
+        payload['startsAt'] = starts_at
+
+        # Setup query
+        payload['generatorURL'] = "http://host?g0.expr=#{CGI.escape(query)}"
+
+        # Setup environment
+        payload['labels'] ||= {}
+        payload['labels']['gitlab_environment_name'] = 'production'
+
+        # Setup chart title & axis labels
+        payload['annotations'] ||= {}
+        payload['annotations']['title'] = 'title'
+        payload['annotations']['gitlab_y_label'] = 'y_label'
+      end
+
+      it_behaves_like 'markdown with metrics embed'
+
+      context 'without y_label' do
+        let(:y_label) { title }
+
+        before do
+          payload['annotations'].delete('gitlab_y_label')
+        end
+
+        it_behaves_like 'markdown with metrics embed'
+      end
+
+      context 'when not enough information is present for an embed' do
         let(:expected_markdown) do
           <<~MARKDOWN.chomp
           #### Summary
@@ -27,29 +114,42 @@ describe Projects::Prometheus::AlertPresenter do
           **Start time:** #{presenter.starts_at}#{markdown_line_break}
           **full_query:** `avg(metric) > 1.0`
 
-        [](http://localhost/#{project.full_path}/prometheus/alerts/#{metric_id}/metrics_dashboard?end=2018-03-12T09%3A36%3A00Z&environment_id=#{env_id}&start=2018-03-12T08%3A36%3A00Z)
           MARKDOWN
         end
 
+        context 'without title' do
           before do
-        payload['labels'] = { 'gitlab_alert_id' => metric_id }
+            payload['annotations'].delete('title')
           end
 
-      context 'without a starting time available' do
-        around do |example|
-          Timecop.freeze(starts_at) { example.run }
+          it { is_expected.to eq(expected_markdown) }
+        end
+
+        context 'without environment' do
+          before do
+            payload['labels'].delete('gitlab_environment_name')
           end
 
           it { is_expected.to eq(expected_markdown) }
         end
 
-      context 'with a starting time available' do
+        context 'without full_query' do
+          let(:expected_markdown) do
+            <<~MARKDOWN.chomp
+            #### Summary
+
+            **Start time:** #{presenter.starts_at}
+
+            MARKDOWN
+          end
+
           before do
-          payload['startsAt'] = starts_at
+            payload.delete('generatorURL')
           end
 
           it { is_expected.to eq(expected_markdown) }
         end
       end
     end
+  end
 end

lib/gitlab/alerting/alert.rb

@@ -69,6 +69,12 @@ module Gitlab
         end
       end
 
+      def y_label
+        strong_memoize(:y_label) do
+          parse_y_label_from_payload || title
+        end
+      end
+
       def alert_markdown
         strong_memoize(:alert_markdown) do
           parse_alert_markdown_from_payload
@@ -162,6 +168,10 @@ module Gitlab
       def parse_alert_markdown_from_payload
         payload&.dig('annotations', 'gitlab_incident_markdown')
       end
+
+      def parse_y_label_from_payload
+        payload&.dig('annotations', 'gitlab_y_label')
+      end
     end
   end
 end

spec/lib/gitlab/alerting/alert_spec.rb

@@ -192,6 +192,16 @@ describe Gitlab::Alerting::Alert do
     end
   end
 
+  describe '#y_label' do
+    subject { alert.y_label }
+
+    it_behaves_like 'parse payload', 'annotations/gitlab_y_label'
+
+    context 'when y_label is not included in the payload' do
+      it_behaves_like 'parse payload', 'annotations/title'
+    end
+  end
+
   describe '#alert_markdown' do
     subject { alert.alert_markdown }