Make Pod health dashboard available to users

- Add a new out-of-the-box dashboard for monitoring K8s pod health.

- Sort dashboards in the dashboard selector dropdown by their
display_name.

app/services/metrics/dashboard/pod_dashboard_service.rb

@@ -4,10 +4,29 @@ module Metrics
   module Dashboard
     class PodDashboardService < ::Metrics::Dashboard::PredefinedDashboardService
       DASHBOARD_PATH = 'config/prometheus/pod_metrics.yml'
-      DASHBOARD_NAME = 'Pod Health'
+      DASHBOARD_NAME = N_('K8s pod health')
 
       # SHA256 hash of dashboard content
-      DASHBOARD_VERSION = 'f12f641d2575d5dcb69e2c633ff5231dbd879ad35020567d8fc4e1090bfdb4b4'
+      DASHBOARD_VERSION = '0515db7a99078a2423b037f99251ba16bd163603c0a30229ae8aa7386e96421c'
+
+      SEQUENCE = [
+        STAGES::MetricEndpointInserter,
+        STAGES::VariableEndpointInserter,
+        STAGES::PanelIdsInserter,
+        STAGES::Sorter
+      ].freeze
+
+      class << self
+        def all_dashboard_paths(_project)
+          [{
+            path: DASHBOARD_PATH,
+            display_name: _(DASHBOARD_NAME),
+            default: false,
+            system_dashboard: false,
+            out_of_the_box_dashboard: out_of_the_box_dashboard?
+          }]
+        end
+      end
 
       private
 

changelogs/unreleased/218649-add-pod-health-dashboard.yml

+---
+title: Add a new K8s Pod health metrics dashboard
+merge_request: 37482
+author:
+type: added

config/prometheus/pod_metrics.yml

-dashboard: 'Pod metrics'
-priority: 10
+dashboard: 'K8s pod health'
+priority: 2
+
+templating:
+  variables:
+    pod:
+      label: 'Pod name'
+      type: metric_label_values
+      options:
+        series_selector: 'container_memory_working_set_bytes'
+        label: 'pod'
+
 panel_groups:
 - group: CPU metrics
   panels:
@@ -8,9 +18,9 @@ panel_groups:
     y_label: "Cores per pod"
     metrics:
     - id: pod_cpu_usage_seconds_total
-      query_range: 'rate(container_cpu_usage_seconds_total{pod_name="{{pod_name}}",container_name="POD"}[5m])'
+      query_range: 'rate(container_cpu_usage_seconds_total{pod="{{pod}}",container="POD"}[5m])'
       unit: "cores"
-      label: pod_name
+      label: pod
 - group: Memory metrics
   panels:
   - title: "Memory usage working set"
@@ -18,9 +28,9 @@ panel_groups:
     y_label: "Working set memory (MiB)"
     metrics:
     - id: pod_memory_working_set
-      query_range: 'container_memory_working_set_bytes{pod_name="{{pod_name}}",container_name="POD"}/1024/1024'
+      query_range: 'container_memory_working_set_bytes{pod="{{pod}}",container="POD"}/1024/1024'
       unit: "MiB"
-      label: pod_name
+      label: pod
 - group: Network metrics
   panels:
   - title: "Network Receive (In)"
@@ -28,17 +38,17 @@ panel_groups:
     y_label: "Received (KiB/sec)"
     metrics:
     - id: pod_network_receive
-      query_range: 'rate(container_network_receive_bytes_total{pod_name="{{pod_name}}",container_name="POD"}[5m])/1024'
+      query_range: 'rate(container_network_receive_bytes_total{pod="{{pod}}",container="POD"}[5m])/1024'
       unit: "KiB / sec"
-      label: pod_name
+      label: pod
   - title: "Network Transmit (Out)"
     type: "line-chart"
     y_label: "Transmitted (KiB/sec)"
     metrics:
     - id: pod_network_transmit
-      query_range: 'rate(container_network_transmit_bytes_total{pod_name="{{pod_name}}",container_name="POD"}[5m])/1024'
+      query_range: 'rate(container_network_transmit_bytes_total{pod="{{pod}}",container="POD"}[5m])/1024'
       unit: "KiB / sec"
-      label: pod_name
+      label: pod
 - group: Disk metrics
   panels:
   - title: "Disk Reads"
@@ -46,14 +56,14 @@ panel_groups:
     y_label: "Disk reads (KiB/sec)"
     metrics:
     - id: pod_disk_reads
-      query_range: 'rate(container_fs_reads_bytes_total{container_name="POD",pod_name="{{pod_name}}"}[5m])/1024'
+      query_range: 'rate(container_fs_reads_bytes_total{container="POD",pod="{{pod}}"}[5m])/1024'
       unit: "KiB / sec"
-      label: pod_name
+      label: pod
   - title: "Disk Writes"
     type: "line-chart"
     y_label: "Disk writes (KiB/sec)"
     metrics:
     - id: pod_disk_writes
-      query_range: 'rate(container_fs_writes_bytes_total{container_name="POD",pod_name="{{pod_name}}"}[5m])/1024'
+      query_range: 'rate(container_fs_writes_bytes_total{container="POD",pod="{{pod}}"}[5m])/1024'
       unit: "KiB / sec"
-      label: pod_name
+      label: pod

lib/gitlab/metrics/dashboard/finder.rb

@@ -14,10 +14,7 @@ module Gitlab
           ::Metrics::Dashboard::SelfMonitoringDashboardService,
 
           # This dashboard is displayed on the K8s cluster settings health page.
-          ::Metrics::Dashboard::ClusterDashboardService,
-
-          # This dashboard is not yet ready for the world.
-          ::Metrics::Dashboard::PodDashboardService
+          ::Metrics::Dashboard::ClusterDashboardService
         ].freeze
 
         class << self
@@ -72,9 +69,11 @@ module Gitlab
           #                              display_name: String,
           #                              default: Boolean }]
           def find_all_paths(project)
-            user_facing_dashboard_services(project).flat_map do |service|
+            dashboards = user_facing_dashboard_services(project).flat_map do |service|
               service.all_dashboard_paths(project)
             end
+
+            Gitlab::Utils.stable_sort_by(dashboards) { |dashboard| dashboard[:display_name].downcase }
           end
 
           private

locale/gitlab.pot

@@ -13570,6 +13570,9 @@ msgstr ""
 msgid "Just me"
 msgstr ""
 
+msgid "K8s pod health"
+msgstr ""
+
 msgid "Keep divergent refs"
 msgstr ""
 

spec/controllers/concerns/metrics_dashboard_spec.rb

@@ -165,13 +165,14 @@ RSpec.describe MetricsDashboard do
             it 'adds starred dashboard information and sorts the list' do
               all_dashboards = json_response['all_dashboards'].map { |dashboard| dashboard.slice('display_name', 'starred', 'user_starred_path') }
               expected_response = [
-                { "display_name" => "Overview", "starred" => false, 'user_starred_path' => api_v4_projects_metrics_user_starred_dashboards_path(id: project.id, params: { dashboard_path: 'config/prometheus/common_metrics.yml' }) },
                 { "display_name" => "anomaly.yml", "starred" => false, 'user_starred_path' => api_v4_projects_metrics_user_starred_dashboards_path(id: project.id, params: { dashboard_path: '.gitlab/dashboards/anomaly.yml' }) },
                 { "display_name" => "errors.yml", "starred" => true, 'user_starred_path' => api_v4_projects_metrics_user_starred_dashboards_path(id: project.id, params: { dashboard_path: '.gitlab/dashboards/errors.yml' }) },
+                { "display_name" => "K8s pod health", "starred" => false, 'user_starred_path' => api_v4_projects_metrics_user_starred_dashboards_path(id: project.id, params: { dashboard_path: 'config/prometheus/pod_metrics.yml' }) },
+                { "display_name" => "Overview", "starred" => false, 'user_starred_path' => api_v4_projects_metrics_user_starred_dashboards_path(id: project.id, params: { dashboard_path: 'config/prometheus/common_metrics.yml' }) },
                 { "display_name" => "test.yml", "starred" => true, 'user_starred_path' => api_v4_projects_metrics_user_starred_dashboards_path(id: project.id, params: { dashboard_path: '.gitlab/dashboards/test.yml' }) }
               ]
 
-              expect(all_dashboards).to eql expected_response
+              expect(all_dashboards).to eq(expected_response)
             end
           end
         end

spec/frontend/fixtures/metrics_dashboard.rb

@@ -8,7 +8,7 @@ RSpec.describe MetricsDashboard, '(JavaScript fixtures)', type: :controller do
 
   let_it_be(:user) { create(:user) }
   let_it_be(:namespace) { create(:namespace, name: 'monitoring' )}
-  let_it_be(:project) { project_with_dashboard_namespace('.gitlab/dashboards/test.yml', namespace: namespace) }
+  let_it_be(:project) { project_with_dashboard_namespace('.gitlab/dashboards/test.yml', nil, namespace: namespace) }
   let_it_be(:environment) { create(:environment, id: 1, project: project) }
   let_it_be(:params) { { environment: environment } }
 

spec/lib/gitlab/metrics/dashboard/finder_spec.rb

@@ -143,19 +143,41 @@ RSpec.describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store
   describe '.find_all_paths' do
     let(:all_dashboard_paths) { described_class.find_all_paths(project) }
     let(:system_dashboard) { { path: system_dashboard_path, display_name: 'Overview', default: true, system_dashboard: true, out_of_the_box_dashboard: true } }
+    let(:k8s_pod_health_dashboard) { { path: pod_dashboard_path, display_name: 'K8s pod health', default: false, system_dashboard: false, out_of_the_box_dashboard: true } }
 
-    it 'includes only the system dashboard by default' do
-      expect(all_dashboard_paths).to eq([system_dashboard])
+    it 'includes OOTB dashboards by default' do
+      expect(all_dashboard_paths).to eq([k8s_pod_health_dashboard, system_dashboard])
     end
 
     context 'when the project contains dashboards' do
-      let(:dashboard_path) { '.gitlab/dashboards/test.yml' }
-      let(:project) { project_with_dashboard(dashboard_path) }
+      let(:dashboard_content) { fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
+      let(:project) { project_with_dashboards(dashboards) }
+
+      let(:dashboards) do
+        {
+          '.gitlab/dashboards/metrics.yml' => dashboard_content,
+          '.gitlab/dashboards/better_metrics.yml' => dashboard_content
+        }
+      end
 
-      it 'includes system and project dashboards' do
-        project_dashboard = { path: dashboard_path, display_name: 'test.yml', default: false, system_dashboard: false, out_of_the_box_dashboard: false }
+      it 'includes OOTB and project dashboards' do
+        project_dashboard1 = {
+          path: '.gitlab/dashboards/metrics.yml',
+          display_name: 'metrics.yml',
+          default: false,
+          system_dashboard: false,
+          out_of_the_box_dashboard: false
+        }
+
+        project_dashboard2 = {
+          path: '.gitlab/dashboards/better_metrics.yml',
+          display_name: 'better_metrics.yml',
+          default: false,
+          system_dashboard: false,
+          out_of_the_box_dashboard: false
+        }
 
-        expect(all_dashboard_paths).to contain_exactly(system_dashboard, project_dashboard)
+        expect(all_dashboard_paths).to eq([project_dashboard2, k8s_pod_health_dashboard, project_dashboard1, system_dashboard])
       end
     end
 
@@ -185,7 +207,7 @@ RSpec.describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store
           out_of_the_box_dashboard: false
         }
 
-        expect(all_dashboard_paths).to contain_exactly(self_monitoring_dashboard, project_dashboard)
+        expect(all_dashboard_paths).to eq([self_monitoring_dashboard, project_dashboard])
       end
     end
   end

spec/services/metrics/dashboard/pod_dashboard_service_spec.rb

@@ -54,4 +54,20 @@ RSpec.describe Metrics::Dashboard::PodDashboardService, :use_clean_rails_memory_
       let(:dashboard_version) { subject.send(:dashboard_version) }
     end
   end
+
+  describe '.all_dashboard_paths' do
+    it 'returns the dashboard attributes' do
+      all_dashboards = described_class.all_dashboard_paths(project)
+
+      expect(all_dashboards).to eq(
+        [{
+          path: described_class::DASHBOARD_PATH,
+          display_name: described_class::DASHBOARD_NAME,
+          default: false,
+          system_dashboard: false,
+          out_of_the_box_dashboard: true
+        }]
+      )
+    end
+  end
 end

spec/support/helpers/metrics_dashboard_helpers.rb

 # frozen_string_literal: true
 
 module MetricsDashboardHelpers
-  def project_with_dashboard(dashboard_path, dashboard_yml = nil)
-    dashboard_yml ||= fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml')
-
-    create(:project, :custom_repo, files: { dashboard_path => dashboard_yml })
+  # @param dashboards [Hash<string, string>] - Should contain a hash where
+  #     each key is the path to a dashboard in the repository and each value is
+  #     the dashboard content.
+  #     Ex: { '.gitlab/dashboards/dashboard1.yml' => fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
+  def project_with_dashboards(dashboards, project_params = {})
+    create(:project, :custom_repo, **project_params, files: dashboards)
   end
 
-  def project_with_dashboard_namespace(dashboard_path, dashboard_yml = nil)
+  def project_with_dashboard(dashboard_path, dashboard_yml = nil, project_params = {})
     dashboard_yml ||= fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml')
 
-    create(:project, :custom_repo, namespace: namespace, path: 'monitor-project', files: { dashboard_path => dashboard_yml })
+    project_with_dashboards({ dashboard_path => dashboard_yml }, project_params)
+  end
+
+  def project_with_dashboard_namespace(dashboard_path, dashboard_yml = nil, project_params = {})
+    project_with_dashboard(dashboard_path, dashboard_yml, project_params.reverse_merge(path: 'monitor-project'))
   end
 
   def delete_project_dashboard(project, user, dashboard_path)

spec/support/shared_examples/services/metrics/dashboard_shared_examples.rb

@@ -62,7 +62,7 @@ end
 RSpec.shared_examples 'dashboard_version contains SHA256 hash of dashboard file content' do
   specify do
     dashboard = File.read(Rails.root.join(dashboard_path))
-    expect(Digest::SHA256.hexdigest(dashboard)).to eq(dashboard_version)
+    expect(dashboard_version).to eq(Digest::SHA256.hexdigest(dashboard))
   end
 end