Commit 88e5269a authored by Rémy Coutable's avatar Rémy Coutable

Merge branch 'prometheus-monitoring-ee' into 'master'

[EE] Prometheus monitoring

See merge request !1374
parents 58d23076 4a627c15
import PrometheusGraph from './monitoring/prometheus_graph'; // TODO: Maybe Make this a bundle
/* eslint-disable func-names, space-before-function-paren, no-var, prefer-arrow-callback, wrap-iife, no-shadow, consistent-return, one-var, one-var-declaration-per-line, camelcase, default-case, no-new, quotes, no-duplicate-case, no-case-declarations, no-fallthrough, max-len */
/* global UsernameValidator */
/* global ActiveTabMemoizer */
......@@ -301,6 +302,8 @@ const UserCallout = require('./user_callout');
case 'ci:lints:show':
new gl.CILintEditor();
break;
case 'projects:environments:metrics':
new PrometheusGraph();
case 'users:show':
new UserCallout();
break;
......
This diff is collapsed.
......@@ -278,3 +278,71 @@
font-size: 20px;
}
}
.prometheus-graph {
text {
fill: $stat-graph-axis-fill;
}
}
.x-axis path,
.y-axis path,
.label-x-axis-line,
.label-y-axis-line {
fill: none;
stroke-width: 1;
shape-rendering: crispEdges;
}
.x-axis path,
.y-axis path {
stroke: $stat-graph-axis-fill;
}
.label-x-axis-line,
.label-y-axis-line {
stroke: $border-color;
}
.y-axis {
line {
stroke: $stat-graph-axis-fill;
stroke-width: 1;
}
}
.metric-area {
opacity: 0.8;
}
.prometheus-graph-overlay {
fill: none;
opacity: 0.0;
pointer-events: all;
}
.rect-text-metric {
fill: $white-light;
stroke-width: 1;
stroke: $black;
}
.rect-axis-text {
fill: $white-light;
}
.text-metric,
.text-median-metric,
.text-metric-usage,
.text-metric-date {
fill: $black;
}
.text-metric-date {
font-weight: 200;
}
.selected-metric-line {
stroke: $black;
stroke-width: 1;
}
......@@ -5,7 +5,7 @@ class Projects::EnvironmentsController < Projects::ApplicationController
before_action :authorize_create_deployment!, only: [:stop]
before_action :authorize_update_environment!, only: [:edit, :update]
before_action :authorize_admin_environment!, only: [:terminal, :terminal_websocket_authorize]
before_action :environment, only: [:show, :edit, :update, :stop, :terminal, :terminal_websocket_authorize, :status]
before_action :environment, only: [:show, :edit, :update, :stop, :terminal, :terminal_websocket_authorize, :metrics, :status]
before_action :verify_api_request!, only: :terminal_websocket_authorize
def index
......@@ -109,6 +109,19 @@ class Projects::EnvironmentsController < Projects::ApplicationController
end
end
def metrics
# Currently, this acts as a hint to load the metrics details into the cache
# if they aren't there already
@metrics = environment.metrics || {}
respond_to do |format|
format.html
format.json do
render json: @metrics, status: @metrics.any? ? :ok : :no_content
end
end
end
# The rollout status of an enviroment
def status
unless @environment.deployment_service_ready?
......
......@@ -74,6 +74,10 @@ module GitlabRoutingHelper
namespace_project_environment_path(environment.project.namespace, environment.project, environment, *args)
end
def environment_metrics_path(environment, *args)
metrics_namespace_project_environment_path(environment.project.namespace, environment.project, environment, *args)
end
def issue_path(entity, *args)
namespace_project_issue_path(entity.project.namespace, entity.project, entity, *args)
end
......
......@@ -149,6 +149,14 @@ class Environment < ActiveRecord::Base
project.deployment_service.rollout_status(self) if deployment_service_ready?
end
def has_metrics?
project.monitoring_service.present? && available? && last_deployment.present?
end
def metrics
project.monitoring_service.metrics(self) if has_metrics?
end
# An environment name is not necessarily suitable for use in URLs, DNS
# or other third-party contexts, so provide a slugified version. A slug has
# the following properties:
......
......@@ -114,6 +114,7 @@ class Project < ActiveRecord::Base
has_one :gitlab_issue_tracker_service, dependent: :destroy, inverse_of: :project
has_one :external_wiki_service, dependent: :destroy
has_one :kubernetes_service, dependent: :destroy, inverse_of: :project
has_one :prometheus_service, dependent: :destroy, inverse_of: :project
has_one :index_status, dependent: :destroy
has_one :mock_ci_service, dependent: :destroy
......@@ -874,6 +875,14 @@ class Project < ActiveRecord::Base
@deployment_service ||= deployment_services.reorder(nil).find_by(active: true)
end
def monitoring_services
services.where(category: :monitoring)
end
def monitoring_service
@monitoring_service ||= monitoring_services.reorder(nil).find_by(active: true)
end
def jira_tracker?
issues_tracker.to_param == 'jira'
end
......
# Base class for monitoring services
#
# These services integrate with a deployment solution like Prometheus
# to provide additional features for environments.
class MonitoringService < Service
default_value_for :category, 'monitoring'
def self.supported_events
%w()
end
# Environments have a number of metrics
def metrics(environment)
raise NotImplementedError
end
end
class PrometheusService < MonitoringService
include ReactiveCaching
self.reactive_cache_key = ->(service) { [service.class.model_name.singular, service.project_id] }
self.reactive_cache_lease_timeout = 30.seconds
self.reactive_cache_refresh_interval = 30.seconds
self.reactive_cache_lifetime = 1.minute
# Access to prometheus is directly through the API
prop_accessor :api_url
with_options presence: true, if: :activated? do
validates :api_url, url: true
end
after_save :clear_reactive_cache!
def initialize_properties
if properties.nil?
self.properties = {}
end
end
def title
'Prometheus'
end
def description
'Prometheus monitoring'
end
def help
'Retrieves `container_cpu_usage_seconds_total` and `container_memory_usage_bytes` from the configured Prometheus server. An `environment` label is required on each metric to identify the Environment.'
end
def self.to_param
'prometheus'
end
def fields
[
{
type: 'text',
name: 'api_url',
title: 'API URL',
placeholder: 'Prometheus API Base URL, like http://prometheus.example.com/'
}
]
end
# Check we can connect to the Prometheus API
def test(*args)
client.ping
{ success: true, result: 'Checked API endpoint' }
rescue Gitlab::PrometheusError => err
{ success: false, result: err }
end
def metrics(environment)
with_reactive_cache(environment.slug) do |data|
data
end
end
# Cache metrics for specific environment
def calculate_reactive_cache(environment_slug)
return unless active? && project && !project.pending_delete?
memory_query = %{sum(container_memory_usage_bytes{container_name="app",environment="#{environment_slug}"})/1024/1024}
cpu_query = %{sum(rate(container_cpu_usage_seconds_total{container_name="app",environment="#{environment_slug}"}[2m]))}
{
success: true,
metrics: {
# Memory used in MB
memory_values: client.query_range(memory_query, start: 8.hours.ago),
memory_current: client.query(memory_query),
# CPU Usage rate in cores.
cpu_values: client.query_range(cpu_query, start: 8.hours.ago),
cpu_current: client.query(cpu_query)
},
last_update: Time.now.utc
}
rescue Gitlab::PrometheusError => err
{ success: false, result: err.message }
end
def client
@prometheus ||= Gitlab::Prometheus.new(api_url: api_url)
end
end
......@@ -234,6 +234,7 @@ class Service < ActiveRecord::Base
mattermost
pipelines_email
pivotaltracker
prometheus
pushover
redmine
slack_slash_commands
......
- environment = local_assigns.fetch(:environment)
- return unless environment.has_metrics? && can?(current_user, :read_environment, environment)
= link_to environment_metrics_path(environment), title: 'See metrics', class: 'btn metrics-button' do
= icon('area-chart')
- @no_container = true
- page_title "Metrics for environment", @environment.name
= render "projects/pipelines/head"
%div{ class: container_class }
.top-area
.row
.col-sm-6
%h3.page-title
Environment:
= @environment.name
.col-sm-6
.nav-controls
= render 'projects/deployments/actions', deployment: @environment.last_deployment
.row
.col-sm-12
%svg.prometheus-graph{ 'graph-type' => 'cpu_values' }
.row
.col-sm-12
%svg.prometheus-graph{ 'graph-type' => 'memory_values' }
......@@ -8,6 +8,7 @@
%h3.page-title= @environment.name
.col-md-3
.nav-controls
= render 'projects/environments/metrics_button', environment: @environment
= render 'projects/environments/terminal_button', environment: @environment
= render 'projects/environments/external_url', environment: @environment
- if can?(current_user, :update_environment, @environment)
......
......@@ -192,6 +192,7 @@ constraints(ProjectUrlConstrainer.new) do
member do
post :stop
get :terminal
get :metrics
get :status, constraints: { format: :json }
get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', constraints: { format: nil }
end
......
......@@ -421,6 +421,14 @@ module API
desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.'
}
],
'prometheus' => [
{
required: true,
name: :api_url,
type: String,
desc: 'Prometheus API Base URL, like http://prometheus.example.com/'
}
],
'pushover' => [
{
required: true,
......@@ -604,6 +612,7 @@ module API
SlackSlashCommandsService,
PipelinesEmailService,
PivotaltrackerService,
PrometheusService,
PushoverService,
RedmineService,
SlackService,
......
module Gitlab
PrometheusError = Class.new(StandardError)
# Helper methods to interact with Prometheus network services & resources
class Prometheus
attr_reader :api_url
def initialize(api_url:)
@api_url = api_url
end
def ping
json_api_get('query', query: '1')
end
def query(query)
get_result('vector') do
json_api_get('query', query: query)
end
end
def query_range(query, start: 8.hours.ago)
get_result('matrix') do
json_api_get('query_range',
query: query,
start: start.to_f,
end: Time.now.utc.to_f,
step: 1.minute.to_i)
end
end
private
def json_api_get(type, args = {})
get(join_api_url(type, args))
rescue Errno::ECONNREFUSED
raise PrometheusError, 'Connection refused'
end
def join_api_url(type, args = {})
url = URI.parse(api_url)
rescue URI::Error
raise PrometheusError, "Invalid API URL: #{api_url}"
else
url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/')
url.query = args.to_query
url.to_s
end
def get(url)
handle_response(HTTParty.get(url))
end
def handle_response(response)
if response.code == 200 && response['status'] == 'success'
response['data'] || {}
elsif response.code == 400
raise PrometheusError, response['error'] || 'Bad data received'
else
raise PrometheusError, "#{response.code} - #{response.body}"
end
end
def get_result(expected_type)
data = yield
data['result'] if data['resultType'] == expected_type
end
end
end
......@@ -225,6 +225,52 @@ describe Projects::EnvironmentsController do
end
end
describe 'GET #metrics' do
before do
allow(controller).to receive(:environment).and_return(environment)
end
context 'when environment has no metrics' do
before do
expect(environment).to receive(:metrics).and_return(nil)
end
it 'returns a metrics page' do
get :metrics, environment_params
expect(response).to be_ok
end
context 'when requesting metrics as JSON' do
it 'returns a metrics JSON document' do
get :metrics, environment_params(format: :json)
expect(response).to have_http_status(204)
expect(json_response).to eq({})
end
end
end
context 'when environment has some metrics' do
before do
expect(environment).to receive(:metrics).and_return({
success: true,
metrics: {},
last_update: 42
})
end
it 'returns a metrics JSON document' do
get :metrics, environment_params(format: :json)
expect(response).to be_ok
expect(json_response['success']).to be(true)
expect(json_response['metrics']).to eq({})
expect(json_response['last_update']).to eq(42)
end
end
end
def environment_params(opts = {})
opts.reverse_merge(namespace_id: project.namespace,
project_id: project,
......
......@@ -247,4 +247,15 @@ FactoryGirl.define do
)
end
end
factory :prometheus_project, parent: :empty_project do
after :create do |project|
project.create_prometheus_service(
active: true,
properties: {
api_url: 'https://prometheus.example.com'
}
)
end
end
end
require 'spec_helper'
feature 'Environment > Metrics', :feature do
include PrometheusHelpers
given(:user) { create(:user) }
given(:project) { create(:prometheus_project) }
given(:pipeline) { create(:ci_pipeline, project: project) }
given(:build) { create(:ci_build, pipeline: pipeline) }
given(:environment) { create(:environment, project: project) }
given(:current_time) { Time.now.utc }
background do
project.add_developer(user)
create(:deployment, environment: environment, deployable: build)
stub_all_prometheus_requests(environment.slug)
login_as(user)
visit_environment(environment)
end
around do |example|
Timecop.freeze(current_time) { example.run }
end
context 'with deployments and related deployable present' do
scenario 'shows metrics' do
click_link('See metrics')
expect(page).to have_css('svg.prometheus-graph')
end
end
def visit_environment(environment)
visit namespace_project_environment_path(environment.project.namespace,
environment.project,
environment)
end
end
......@@ -37,13 +37,7 @@ feature 'Environment', :feature do
scenario 'does show deployment SHA' do
expect(page).to have_link(deployment.short_sha)
end
scenario 'does not show a re-deploy button for deployment without build' do
expect(page).not_to have_link('Re-deploy')
end
scenario 'does not show terminal button' do
expect(page).not_to have_terminal_button
end
end
......@@ -58,13 +52,7 @@ feature 'Environment', :feature do
scenario 'does show build name' do
expect(page).to have_link("#{build.name} (##{build.id})")
end
scenario 'does show re-deploy button' do
expect(page).to have_link('Re-deploy')
end
scenario 'does not show terminal button' do
expect(page).not_to have_terminal_button
end
......@@ -77,7 +65,7 @@ feature 'Environment', :feature do
scenario 'does allow to play manual action' do
expect(manual).to be_skipped
expect{ click_link(manual.name.humanize) }.not_to change { Ci::Pipeline.count }
expect { click_link(manual.name.humanize) }.not_to change { Ci::Pipeline.count }
expect(page).to have_content(manual.name)
expect(manual.reload).to be_pending
end
......@@ -111,9 +99,6 @@ feature 'Environment', :feature do
it 'displays a web terminal' do
expect(page).to have_selector('#terminal')
end
it 'displays a link to the environment external url' do
expect(page).to have_link(nil, href: environment.external_url)
end
end
......@@ -133,10 +118,6 @@ feature 'Environment', :feature do
given(:manual) { create(:ci_build, :manual, pipeline: pipeline, name: 'close_app') }
given(:deployment) { create(:deployment, environment: environment, deployable: build, on_stop: 'close_app') }
scenario 'does show stop button' do
expect(page).to have_link('Stop')
end
scenario 'does allow to stop environment' do
click_link('Stop')
......
%div
.top-area
.row
.col-sm-6
%h3.page-title
Metrics for environment
.row
.col-sm-12
%svg.prometheus-graph{ 'graph-type' => 'cpu_values' }
.row
.col-sm-12
%svg.prometheus-graph{ 'graph-type' => 'memory_values' }
\ No newline at end of file
import 'jquery';
import es6Promise from 'es6-promise';
import '~/lib/utils/common_utils';
import PrometheusGraph from '~/monitoring/prometheus_graph';
import { prometheusMockData } from './prometheus_mock_data';
es6Promise.polyfill();
describe('PrometheusGraph', () => {
const fixtureName = 'static/environments/metrics.html.raw';
const prometheusGraphContainer = '.prometheus-graph';
const prometheusGraphContents = `${prometheusGraphContainer}[graph-type=cpu_values]`;
preloadFixtures(fixtureName);
beforeEach(() => {
loadFixtures(fixtureName);
this.prometheusGraph = new PrometheusGraph();
const self = this;
const fakeInit = (metricsResponse) => {
self.prometheusGraph.transformData(metricsResponse);
self.prometheusGraph.createGraph();
};
spyOn(this.prometheusGraph, 'init').and.callFake(fakeInit);
});
it('initializes graph properties', () => {
// Test for the measurements
expect(this.prometheusGraph.margin).toBeDefined();
expect(this.prometheusGraph.marginLabelContainer).toBeDefined();
expect(this.prometheusGraph.originalWidth).toBeDefined();
expect(this.prometheusGraph.originalHeight).toBeDefined();
expect(this.prometheusGraph.height).toBeDefined();
expect(this.prometheusGraph.width).toBeDefined();
expect(this.prometheusGraph.backOffRequestCounter).toBeDefined();
// Test for the graph properties (colors, radius, etc.)
expect(this.prometheusGraph.graphSpecificProperties).toBeDefined();
expect(this.prometheusGraph.commonGraphProperties).toBeDefined();
});
it('transforms the data', () => {
this.prometheusGraph.init(prometheusMockData.metrics);
expect(this.prometheusGraph.data).toBeDefined();
expect(this.prometheusGraph.data.cpu_values.length).toBe(121);
expect(this.prometheusGraph.data.memory_values.length).toBe(121);
});
it('creates two graphs', () => {
this.prometheusGraph.init(prometheusMockData.metrics);
expect($(prometheusGraphContainer).length).toBe(2);
});
describe('Graph contents', () => {
beforeEach(() => {
this.prometheusGraph.init(prometheusMockData.metrics);
});
it('has axis, an area, a line and a overlay', () => {
const $graphContainer = $(prometheusGraphContents).find('.x-axis').parent();
expect($graphContainer.find('.x-axis')).toBeDefined();
expect($graphContainer.find('.y-axis')).toBeDefined();
expect($graphContainer.find('.prometheus-graph-overlay')).toBeDefined();
expect($graphContainer.find('.metric-line')).toBeDefined();
expect($graphContainer.find('.metric-area')).toBeDefined();
});
it('has legends, labels and an extra axis that labels the metrics', () => {
const $prometheusGraphContents = $(prometheusGraphContents);
const $axisLabelContainer = $(prometheusGraphContents).find('.label-x-axis-line').parent();
expect($prometheusGraphContents.find('.label-x-axis-line')).toBeDefined();
expect($prometheusGraphContents.find('.label-y-axis-line')).toBeDefined();
expect($prometheusGraphContents.find('.label-axis-text')).toBeDefined();
expect($prometheusGraphContents.find('.rect-axis-text')).toBeDefined();
expect($axisLabelContainer.find('rect').length).toBe(2);
expect($axisLabelContainer.find('text').length).toBe(4);
});
});
});
This diff is collapsed.
......@@ -142,6 +142,7 @@ project:
- slack_slash_commands_service
- irker_service
- pivotaltracker_service
- prometheus_service
- hipchat_service
- flowdock_service
- assembla_service
......
require 'spec_helper'
describe Gitlab::Prometheus, lib: true do
include PrometheusHelpers
subject { described_class.new(api_url: 'https://prometheus.example.com') }
describe '#ping' do
it 'issues a "query" request to the API endpoint' do
req_stub = stub_prometheus_request(prometheus_query_url('1'), body: prometheus_value_body('vector'))
expect(subject.ping).to eq({ "resultType" => "vector", "result" => [{ "metric" => {}, "value" => [1488772511.004, "0.000041021495238095323"] }] })
expect(req_stub).to have_been_requested
end
end
# This shared examples expect:
# - query_url: A query URL
# - execute_query: A query call
shared_examples 'failure response' do
context 'when request returns 400 with an error message' do
it 'raises a Gitlab::PrometheusError error' do
req_stub = stub_prometheus_request(query_url, status: 400, body: { error: 'bar!' })
expect { execute_query }
.to raise_error(Gitlab::PrometheusError, 'bar!')
expect(req_stub).to have_been_requested
end
end
context 'when request returns 400 without an error message' do
it 'raises a Gitlab::PrometheusError error' do
req_stub = stub_prometheus_request(query_url, status: 400)
expect { execute_query }
.to raise_error(Gitlab::PrometheusError, 'Bad data received')
expect(req_stub).to have_been_requested
end
end
context 'when request returns 500' do
it 'raises a Gitlab::PrometheusError error' do
req_stub = stub_prometheus_request(query_url, status: 500, body: { message: 'FAIL!' })
expect { execute_query }
.to raise_error(Gitlab::PrometheusError, '500 - {"message":"FAIL!"}')
expect(req_stub).to have_been_requested
end
end
end
describe '#query' do
let(:prometheus_query) { prometheus_cpu_query('env-slug') }
let(:query_url) { prometheus_query_url(prometheus_query) }
context 'when request returns vector results' do
it 'returns data from the API call' do
req_stub = stub_prometheus_request(query_url, body: prometheus_value_body('vector'))
expect(subject.query(prometheus_query)).to eq [{ "metric" => {}, "value" => [1488772511.004, "0.000041021495238095323"] }]
expect(req_stub).to have_been_requested
end
end
context 'when request returns matrix results' do
it 'returns nil' do
req_stub = stub_prometheus_request(query_url, body: prometheus_value_body('matrix'))
expect(subject.query(prometheus_query)).to be_nil
expect(req_stub).to have_been_requested
end
end
context 'when request returns no data' do
it 'returns []' do
req_stub = stub_prometheus_request(query_url, body: prometheus_empty_body('vector'))
expect(subject.query(prometheus_query)).to be_empty
expect(req_stub).to have_been_requested
end
end
it_behaves_like 'failure response' do
let(:execute_query) { subject.query(prometheus_query) }
end
end
describe '#query_range' do
let(:prometheus_query) { prometheus_memory_query('env-slug') }
let(:query_url) { prometheus_query_range_url(prometheus_query) }
around do |example|
Timecop.freeze { example.run }
end
context 'when a start time is passed' do
let(:query_url) { prometheus_query_range_url(prometheus_query, start: 2.hours.ago) }
it 'passed it in the requested URL' do
req_stub = stub_prometheus_request(query_url, body: prometheus_values_body('vector'))
subject.query_range(prometheus_query, start: 2.hours.ago)
expect(req_stub).to have_been_requested
end
end
context 'when request returns vector results' do
it 'returns nil' do
req_stub = stub_prometheus_request(query_url, body: prometheus_values_body('vector'))
expect(subject.query_range(prometheus_query)).to be_nil
expect(req_stub).to have_been_requested
end
end
context 'when request returns matrix results' do
it 'returns data from the API call' do
req_stub = stub_prometheus_request(query_url, body: prometheus_values_body('matrix'))
expect(subject.query_range(prometheus_query)).to eq([
{
"metric" => {},
"values" => [[1488758662.506, "0.00002996364761904785"], [1488758722.506, "0.00003090239047619091"]]
}
])
expect(req_stub).to have_been_requested
end
end
context 'when request returns no data' do
it 'returns []' do
req_stub = stub_prometheus_request(query_url, body: prometheus_empty_body('matrix'))
expect(subject.query_range(prometheus_query)).to be_empty
expect(req_stub).to have_been_requested
end
end
it_behaves_like 'failure response' do
let(:execute_query) { subject.query_range(prometheus_query) }
end
end
end
......@@ -271,7 +271,11 @@ describe Environment, models: true do
context 'when the environment is unavailable' do
let(:project) { create(:kubernetes_project) }
before { environment.stop }
before do
environment.stop
end
it { is_expected.to be_falsy }
end
end
......@@ -281,20 +285,25 @@ describe Environment, models: true do
subject { environment.terminals }
context 'when the environment has terminals' do
before { allow(environment).to receive(:deployment_service_ready?).and_return(true) }
before do
allow(environment).to receive(:deployment_service_ready?).and_return(true)
end
it 'returns the terminals from the deployment service' do
expect(project.deployment_service).
to receive(:terminals).with(environment).
and_return(:fake_terminals)
expect(project.deployment_service)
.to receive(:terminals).with(environment)
.and_return(:fake_terminals)
is_expected.to eq(:fake_terminals)
end
end
context 'when the environment does not have terminals' do
before { allow(environment).to receive(:deployment_service_ready?).and_return(false) }
it { is_expected.to eq(nil) }
before do
allow(environment).to receive(:deployment_service_ready?).and_return(false)
end
it { is_expected.to be_nil }
end
end
......@@ -320,6 +329,66 @@ describe Environment, models: true do
end
end
describe '#has_metrics?' do
subject { environment.has_metrics? }
context 'when the enviroment is available' do
context 'with a deployment service' do
let(:project) { create(:prometheus_project) }
context 'and a deployment' do
let!(:deployment) { create(:deployment, environment: environment) }
it { is_expected.to be_truthy }
end
context 'but no deployments' do
it { is_expected.to be_falsy }
end
end
context 'without a monitoring service' do
it { is_expected.to be_falsy }
end
end
context 'when the environment is unavailable' do
let(:project) { create(:prometheus_project) }
before do
environment.stop
end
it { is_expected.to be_falsy }
end
end
describe '#metrics' do
let(:project) { create(:prometheus_project) }
subject { environment.metrics }
context 'when the environment has metrics' do
before do
allow(environment).to receive(:has_metrics?).and_return(true)
end
it 'returns the metrics from the deployment service' do
expect(project.monitoring_service)
.to receive(:metrics).with(environment)
.and_return(:fake_metrics)
is_expected.to eq(:fake_metrics)
end
end
context 'when the environment does not have metrics' do
before do
allow(environment).to receive(:has_metrics?).and_return(false)
end
it { is_expected.to be_nil }
end
end
describe '#slug' do
it "is automatically generated" do
expect(environment.slug).not_to be_nil
......
require 'spec_helper'
describe PrometheusService, models: true, caching: true do
include PrometheusHelpers
include ReactiveCachingHelpers
let(:project) { create(:prometheus_project) }
let(:service) { project.prometheus_service }
describe "Associations" do
it { is_expected.to belong_to :project }
end
describe 'Validations' do
context 'when service is active' do
before { subject.active = true }
it { is_expected.to validate_presence_of(:api_url) }
end
context 'when service is inactive' do
before { subject.active = false }
it { is_expected.not_to validate_presence_of(:api_url) }
end
end
describe '#test' do
let!(:req_stub) { stub_prometheus_request(prometheus_query_url('1'), body: prometheus_value_body('vector')) }
context 'success' do
it 'reads the discovery endpoint' do
expect(service.test[:success]).to be_truthy
expect(req_stub).to have_been_requested
end
end
context 'failure' do
let!(:req_stub) { stub_prometheus_request(prometheus_query_url('1'), status: 404) }
it 'fails to read the discovery endpoint' do
expect(service.test[:success]).to be_falsy
expect(req_stub).to have_been_requested
end
end
end
describe '#metrics' do
let(:environment) { build_stubbed(:environment, slug: 'env-slug') }
subject { service.metrics(environment) }
around do |example|
Timecop.freeze { example.run }
end
context 'with valid data' do
before do
stub_reactive_cache(service, prometheus_data, 'env-slug')
end
it 'returns reactive data' do
is_expected.to eq(prometheus_data)
end
end
end
describe '#calculate_reactive_cache' do
let(:environment) { build_stubbed(:environment, slug: 'env-slug') }
around do |example|
Timecop.freeze { example.run }
end
subject do
service.calculate_reactive_cache(environment.slug)
end
context 'when service is inactive' do
before do
service.active = false
end
it { is_expected.to be_nil }
end
context 'when Prometheus responds with valid data' do
before do
stub_all_prometheus_requests(environment.slug)
end
it { expect(subject.to_json).to eq(prometheus_data.to_json) }
end
[404, 500].each do |status|
context "when Prometheus responds with #{status}" do
before do
stub_all_prometheus_requests(environment.slug, status: status, body: 'QUERY FAILED!')
end
it { is_expected.to eq(success: false, result: %(#{status} - "QUERY FAILED!")) }
end
end
end
end
......@@ -6,7 +6,9 @@ describe API::V3::Services, api: true do
let(:user) { create(:user) }
let(:project) { create(:empty_project, creator_id: user.id, namespace: user.namespace) }
Service.available_services_names.each do |service|
available_services = Service.available_services_names
available_services.delete('prometheus')
available_services.each do |service|
describe "DELETE /projects/:id/services/#{service.dasherize}" do
include_context service
......
module PrometheusHelpers
def prometheus_memory_query(environment_slug)
%{sum(container_memory_usage_bytes{container_name="app",environment="#{environment_slug}"})/1024/1024}
end
def prometheus_cpu_query(environment_slug)
%{sum(rate(container_cpu_usage_seconds_total{container_name="app",environment="#{environment_slug}"}[2m]))}
end
def prometheus_query_url(prometheus_query)
query = { query: prometheus_query }.to_query
"https://prometheus.example.com/api/v1/query?#{query}"
end
def prometheus_query_range_url(prometheus_query, start: 8.hours.ago)
query = {
query: prometheus_query,
start: start.to_f,
end: Time.now.utc.to_f,
step: 1.minute.to_i
}.to_query
"https://prometheus.example.com/api/v1/query_range?#{query}"
end
def stub_prometheus_request(url, body: {}, status: 200)
WebMock.stub_request(:get, url)
.to_return({
status: status,
headers: { 'Content-Type' => 'application/json' },
body: body.to_json
})
end
def stub_all_prometheus_requests(environment_slug, body: nil, status: 200)
stub_prometheus_request(
prometheus_query_url(prometheus_memory_query(environment_slug)),
status: status,
body: body || prometheus_value_body
)
stub_prometheus_request(
prometheus_query_range_url(prometheus_memory_query(environment_slug)),
status: status,
body: body || prometheus_values_body
)
stub_prometheus_request(
prometheus_query_url(prometheus_cpu_query(environment_slug)),
status: status,
body: body || prometheus_value_body
)
stub_prometheus_request(
prometheus_query_range_url(prometheus_cpu_query(environment_slug)),
status: status,
body: body || prometheus_values_body
)
end
def prometheus_data(last_update: Time.now.utc)
{
success: true,
metrics: {
memory_values: prometheus_values_body('matrix').dig(:data, :result),
memory_current: prometheus_value_body('vector').dig(:data, :result),
cpu_values: prometheus_values_body('matrix').dig(:data, :result),
cpu_current: prometheus_value_body('vector').dig(:data, :result)
},
last_update: last_update
}
end
def prometheus_empty_body(type)
{
"status": "success",
"data": {
"resultType": type,
"result": []
}
}
end
def prometheus_value_body(type = 'vector')
{
"status": "success",
"data": {
"resultType": type,
"result": [
{
"metric": {},
"value": [
1488772511.004,
"0.000041021495238095323"
]
}
]
}
}
end
def prometheus_values_body(type = 'matrix')
{
"status": "success",
"data": {
"resultType": type,
"result": [
{
"metric": {},
"values": [
[1488758662.506, "0.00002996364761904785"],
[1488758722.506, "0.00003090239047619091"]
]
}
]
}
}
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment