Commit 8a19a256 authored by Mikolaj Wawrzyniak's avatar Mikolaj Wawrzyniak

Move cluster health dashboard services to core

AS next step towards moving cluster health dashbaord to gitlab
core, we need to move cluster metrics services to core
parent de284cbc
...@@ -10,29 +10,29 @@ module Metrics ...@@ -10,29 +10,29 @@ module Metrics
ALLOWED_FILE_TYPE = '.yml' ALLOWED_FILE_TYPE = '.yml'
USER_DASHBOARDS_DIR = ::Metrics::Dashboard::CustomDashboardService::DASHBOARD_ROOT USER_DASHBOARDS_DIR = ::Metrics::Dashboard::CustomDashboardService::DASHBOARD_ROOT
SEQUENCES = {
::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
].freeze,
::Metrics::Dashboard::SelfMonitoringDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter
].freeze,
::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
].freeze
}.freeze
steps :check_push_authorized, steps :check_push_authorized,
:check_branch_name, :check_branch_name,
:check_file_type, :check_file_type,
:check_dashboard_template, :check_dashboard_template,
:create_file, :create_file,
:refresh_repository_method_caches :refresh_repository_method_caches
class << self
def sequences
@sequences ||= {
::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
].freeze,
::Metrics::Dashboard::SelfMonitoringDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter
].freeze
}.freeze
end
end
def execute def execute
execute_steps execute_steps
...@@ -173,10 +173,8 @@ module Metrics ...@@ -173,10 +173,8 @@ module Metrics
end end
def sequence def sequence
self.class.sequences[dashboard_template] || [] SEQUENCES[dashboard_template] || []
end end
end end
end end
end end
Metrics::Dashboard::CloneDashboardService.prepend_if_ee('EE::Metrics::Dashboard::CloneDashboardService')
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
module Metrics module Metrics
module Dashboard module Dashboard
class ClusterDashboardService < ::Metrics::Dashboard::PredefinedDashboardService class ClusterDashboardService < ::Metrics::Dashboard::PredefinedDashboardService
DASHBOARD_PATH = 'ee/config/prometheus/cluster_metrics.yml' DASHBOARD_PATH = 'config/prometheus/cluster_metrics.yml'
DASHBOARD_NAME = 'Cluster' DASHBOARD_NAME = 'Cluster'
SEQUENCE = [ SEQUENCE = [
......
dashboard: 'Cluster health'
priority: 1
panel_groups:
- group: Cluster Health - group: Cluster Health
priority: 1 priority: 10
metrics: panels:
- title: "CPU Usage" - title: "CPU Usage"
type: "area-chart"
y_label: "CPU (cores)" y_label: "CPU (cores)"
required_metrics: ['container_cpu_usage_seconds_total']
weight: 1 weight: 1
queries: metrics:
- query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)' - id: cluster_health_cpu_usage
label: Usage (cores) query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)'
unit: "cores" unit: cores
appearance: label: Usage (cores)
line: - id: cluster_health_cpu_requested
width: 2 query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
area: unit: cores
opacity: 0 label: Requested (cores)
- query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' - id: cluster_health_cpu_capacity
label: Requested (cores) query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: "cores" unit: cores
appearance: label: Capacity (cores)
line: - title: "Memory Usage"
width: 2 type: "area-chart"
area:
opacity: 0
- query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
label: Capacity (cores)
unit: "cores"
appearance:
line:
type: 'dashed'
width: 2
area:
opacity: 0
- title: "Memory usage"
y_label: "Memory (GiB)" y_label: "Memory (GiB)"
required_metrics: ['container_memory_usage_bytes']
weight: 1 weight: 1
queries: metrics:
- query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30' - id: cluster_health_memory_usage
label: Usage (GiB) query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30'
unit: "GiB" unit: GiB
appearance: label: Usage (GiB)
line: - id: cluster_health_memory_requested
width: 2 query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
area: unit: GiB
opacity: 0 label: Requested (GiB)
- query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' - id: cluster_health_memory_capacity
label: Requested (GiB) query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: "GiB" unit: GiB
appearance: label: Capacity (GiB)
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
label: Capacity (GiB)
unit: "GiB"
appearance:
line:
type: 'dashed'
width: 2
area:
opacity: 0
# most likely this file can be removed, but until we are sure and have capacity to tackle that I've
# only moved it and added https://gitlab.com/gitlab-org/gitlab/-/issues/225869 to track work need to clean up codebase.
- group: Cluster Health
priority: 1
metrics:
- title: "CPU Usage"
y_label: "CPU (cores)"
required_metrics: ['container_cpu_usage_seconds_total']
weight: 1
queries:
- query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)'
label: Usage (cores)
unit: "cores"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
label: Requested (cores)
unit: "cores"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
label: Capacity (cores)
unit: "cores"
appearance:
line:
type: 'dashed'
width: 2
area:
opacity: 0
- title: "Memory usage"
y_label: "Memory (GiB)"
required_metrics: ['container_memory_usage_bytes']
weight: 1
queries:
- query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30'
label: Usage (GiB)
unit: "GiB"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
label: Requested (GiB)
unit: "GiB"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
label: Capacity (GiB)
unit: "GiB"
appearance:
line:
type: 'dashed'
width: 2
area:
opacity: 0
# frozen_string_literal: true
# Copies system dashboard definition in .yml file into designated
# .yml file inside `.gitlab/dashboards`
module EE
module Metrics
module Dashboard
module CloneDashboardService
extend ActiveSupport::Concern
class_methods do
extend ::Gitlab::Utils::Override
override :sequences
def sequences
super.merge(::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH => [::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter].freeze)
end
end
end
end
end
end
dashboard: 'Cluster health'
priority: 1
panel_groups:
- group: Cluster Health
priority: 10
panels:
- title: "CPU Usage"
type: "area-chart"
y_label: "CPU (cores)"
weight: 1
metrics:
- id: cluster_health_cpu_usage
query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)'
unit: cores
label: Usage (cores)
- id: cluster_health_cpu_requested
query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: cores
label: Requested (cores)
- id: cluster_health_cpu_capacity
query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: cores
label: Capacity (cores)
- title: "Memory Usage"
type: "area-chart"
y_label: "Memory (GiB)"
weight: 1
metrics:
- id: cluster_health_memory_usage
query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30'
unit: GiB
label: Usage (GiB)
- id: cluster_health_memory_requested
query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: GiB
label: Requested (GiB)
- id: cluster_health_memory_capacity
query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: GiB
label: Capacity (GiB)
# frozen_string_literal: true
module EE
module Gitlab
module Metrics
module Dashboard
module ServiceSelector
extend ActiveSupport::Concern
EE_SERVICES = [
::Metrics::Dashboard::ClusterMetricsEmbedService,
::Metrics::Dashboard::ClusterDashboardService
].freeze
class_methods do
extend ::Gitlab::Utils::Override
private
override :services
def services
EE_SERVICES + super
end
end
end
end
end
end
end
...@@ -6,22 +6,6 @@ module EE ...@@ -6,22 +6,6 @@ module EE
module Metrics module Metrics
module Dashboard module Dashboard
module Url module Url
# Matches dashboard urls for a metric chart embed
# for cluster metrics
#
# EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
def clusters_regex
strong_memoize(:clusters_regex) do
regex_for_project_metrics(
%r{
/clusters
/(?<cluster_id>\d+)
/?
}x
)
end
end
# Matches dashboard urls for a metric chart embed # Matches dashboard urls for a metric chart embed
# for a specifc firing GitLab alert # for a specifc firing GitLab alert
# #
......
...@@ -7,7 +7,7 @@ module Gitlab ...@@ -7,7 +7,7 @@ module Gitlab
include QueryAdditionalMetrics include QueryAdditionalMetrics
def query def query
AdditionalMetricsParser.load_groups_from_yaml('cluster_metrics.yml') AdditionalMetricsParser.load_groups_from_yaml('queries_cluster_metrics.yml')
.map(&query_group(base_query_context(8.hours.ago, Time.now))) .map(&query_group(base_query_context(8.hours.ago, Time.now)))
end end
end end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Metrics::Dashboard::ServiceSelector do
include MetricsDashboardHelpers
describe '#call' do
subject { described_class.call(arguments) }
context 'when cluster is provided' do
let(:arguments) { { cluster: "some cluster" } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster is provided and embedded is not true' do
let(:arguments) { { cluster: "some cluster", embedded: 'false' } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster dashboard_path is provided' do
let(:arguments) { { dashboard_path: ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster is provided and embed params' do
let(:arguments) do
{
cluster: "some cluster",
embedded: 'true',
cluster_type: 'project',
format: :json,
group: 'Food metrics',
title: 'Pizza Consumption',
y_label: 'Slice Count'
}
end
it { is_expected.to be Metrics::Dashboard::ClusterMetricsEmbedService }
end
context 'when metrics embed is for an alert' do
let(:arguments) { { embedded: true, prometheus_alert_id: 5 } }
it { is_expected.to be Metrics::Dashboard::GitlabAlertEmbedService }
end
end
end
...@@ -3,35 +3,6 @@ ...@@ -3,35 +3,6 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Gitlab::Metrics::Dashboard::Url do RSpec.describe Gitlab::Metrics::Dashboard::Url do
describe '#clusters_regex' do
let(:url) do
Gitlab::Routing.url_helpers.namespace_project_cluster_url(
'foo',
'bar',
'1',
group: 'Cluster Health',
title: 'Memory Usage',
y_label: 'Memory 20(GiB)',
anchor: 'title'
)
end
let(:expected_params) do
{
'url' => url,
'namespace' => 'foo',
'project' => 'bar',
'cluster_id' => '1',
'query' => '?group=Cluster+Health&title=Memory+Usage&y_label=Memory+20%28GiB%29',
'anchor' => '#title'
}
end
subject { described_class.clusters_regex }
it_behaves_like 'regex which matches url when expected'
end
describe '#alert_regex' do describe '#alert_regex' do
let(:url) do let(:url) do
Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_prometheus_alert_url( Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_prometheus_alert_url(
......
...@@ -12,7 +12,7 @@ RSpec.describe Gitlab::Prometheus::Queries::ClusterQuery do ...@@ -12,7 +12,7 @@ RSpec.describe Gitlab::Prometheus::Queries::ClusterQuery do
end end
it 'load cluster metrics from yaml' do it 'load cluster metrics from yaml' do
expect(Gitlab::Prometheus::AdditionalMetricsParser).to receive(:load_groups_from_yaml).with('cluster_metrics.yml').and_call_original expect(Gitlab::Prometheus::AdditionalMetricsParser).to receive(:load_groups_from_yaml).with('queries_cluster_metrics.yml').and_call_original
subject.query subject.query
end end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Metrics::Dashboard::CloneDashboardService, :use_clean_rails_memory_store_caching do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :repository) }
let_it_be(:environment) { create(:environment, project: project) }
describe '#execute' do
subject(:service_call) { described_class.new(project, user, params).execute }
context 'with rights to push to the repository' do
before do
project.add_maintainer(user)
end
context 'valid parameters' do
let(:commit_message) { 'test' }
let(:branch) { "#{Time.current.to_i}_dashboard_new_branch" }
let(:file_name) { 'custom_dashboard.yml' }
let(:file_content_hash) { YAML.safe_load(File.read(dashboard)) }
let(:params) do
{
dashboard: dashboard,
file_name: file_name,
commit_message: commit_message,
branch: branch
}
end
let(:stages) { ::Gitlab::Metrics::Dashboard::Stages }
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH,
[
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
]
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH,
[
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
]
end
end
end
end
...@@ -13,6 +13,8 @@ module Gitlab ...@@ -13,6 +13,8 @@ module Gitlab
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
SERVICES = [ SERVICES = [
::Metrics::Dashboard::ClusterMetricsEmbedService,
::Metrics::Dashboard::ClusterDashboardService,
::Metrics::Dashboard::GitlabAlertEmbedService, ::Metrics::Dashboard::GitlabAlertEmbedService,
::Metrics::Dashboard::CustomMetricEmbedService, ::Metrics::Dashboard::CustomMetricEmbedService,
::Metrics::Dashboard::GrafanaMetricEmbedService, ::Metrics::Dashboard::GrafanaMetricEmbedService,
...@@ -51,5 +53,3 @@ module Gitlab ...@@ -51,5 +53,3 @@ module Gitlab
end end
end end
end end
Gitlab::Metrics::Dashboard::ServiceSelector.prepend_if_ee('EE::Gitlab::Metrics::Dashboard::ServiceSelector')
...@@ -60,6 +60,22 @@ module Gitlab ...@@ -60,6 +60,22 @@ module Gitlab
Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args)
end end
# Matches dashboard urls for a metric chart embed
# for cluster metrics
#
# EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
def clusters_regex
strong_memoize(:clusters_regex) do
regex_for_project_metrics(
%r{
/clusters
/(?<cluster_id>\d+)
/?
}x
)
end
end
private private
def regex_for_project_metrics(path_suffix_pattern) def regex_for_project_metrics(path_suffix_pattern)
......
...@@ -109,6 +109,46 @@ RSpec.describe Gitlab::Metrics::Dashboard::ServiceSelector do ...@@ -109,6 +109,46 @@ RSpec.describe Gitlab::Metrics::Dashboard::ServiceSelector do
it { is_expected.to be Metrics::Dashboard::TransientEmbedService } it { is_expected.to be Metrics::Dashboard::TransientEmbedService }
end end
context 'when cluster is provided' do
let(:arguments) { { cluster: "some cluster" } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster is provided and embedded is not true' do
let(:arguments) { { cluster: "some cluster", embedded: 'false' } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster dashboard_path is provided' do
let(:arguments) { { dashboard_path: ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster is provided and embed params' do
let(:arguments) do
{
cluster: "some cluster",
embedded: 'true',
cluster_type: 'project',
format: :json,
group: 'Food metrics',
title: 'Pizza Consumption',
y_label: 'Slice Count'
}
end
it { is_expected.to be Metrics::Dashboard::ClusterMetricsEmbedService }
end
context 'when metrics embed is for an alert' do
let(:arguments) { { embedded: true, prometheus_alert_id: 5 } }
it { is_expected.to be Metrics::Dashboard::GitlabAlertEmbedService }
end
end end
end end
end end
...@@ -46,6 +46,35 @@ RSpec.describe Gitlab::Metrics::Dashboard::Url do ...@@ -46,6 +46,35 @@ RSpec.describe Gitlab::Metrics::Dashboard::Url do
end end
end end
describe '#clusters_regex' do
let(:url) do
Gitlab::Routing.url_helpers.namespace_project_cluster_url(
'foo',
'bar',
'1',
group: 'Cluster Health',
title: 'Memory Usage',
y_label: 'Memory 20(GiB)',
anchor: 'title'
)
end
let(:expected_params) do
{
'url' => url,
'namespace' => 'foo',
'project' => 'bar',
'cluster_id' => '1',
'query' => '?group=Cluster+Health&title=Memory+Usage&y_label=Memory+20%28GiB%29',
'anchor' => '#title'
}
end
subject { described_class.clusters_regex }
it_behaves_like 'regex which matches url when expected'
end
describe '#grafana_regex' do describe '#grafana_regex' do
let(:url) do let(:url) do
namespace_project_grafana_api_metrics_dashboard_url( namespace_project_grafana_api_metrics_dashboard_url(
......
...@@ -81,7 +81,18 @@ RSpec.describe Metrics::Dashboard::CloneDashboardService, :use_clean_rails_memor ...@@ -81,7 +81,18 @@ RSpec.describe Metrics::Dashboard::CloneDashboardService, :use_clean_rails_memor
allow(::Gitlab::Metrics::Dashboard::Processor).to receive(:new).and_return(double(process: file_content_hash)) allow(::Gitlab::Metrics::Dashboard::Processor).to receive(:new).and_return(double(process: file_content_hash))
end end
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH, [::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, ::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter, ::Gitlab::Metrics::Dashboard::Stages::Sorter] it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH,
[
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
]
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH,
[
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
]
it_behaves_like 'valid dashboard cloning process', it_behaves_like 'valid dashboard cloning process',
::Metrics::Dashboard::SelfMonitoringDashboardService::DASHBOARD_PATH, ::Metrics::Dashboard::SelfMonitoringDashboardService::DASHBOARD_PATH,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment