Commit a1d6a622 authored by Jan Provaznik's avatar Jan Provaznik

Merge branch 'sy-grafana-embeds-be' into 'master'

Embedding Grafana metrics panels in GFM

See merge request gitlab-org/gitlab!18285
parents c53477fa 2b93de8c
......@@ -3,21 +3,24 @@
# Provides an action which fetches a metrics dashboard according
# to the parameters specified by the controller.
module MetricsDashboard
include RenderServiceResults
extend ActiveSupport::Concern
def metrics_dashboard
result = dashboard_finder.find(
project_for_dashboard,
current_user,
metrics_dashboard_params
metrics_dashboard_params.to_h.symbolize_keys
)
if include_all_dashboards?
if include_all_dashboards? && result
result[:all_dashboards] = dashboard_finder.find_all_paths(project_for_dashboard)
end
respond_to do |format|
if result[:status] == :success
if result.nil?
format.json { continue_polling_response }
elsif result[:status] == :success
format.json { render dashboard_success_response(result) }
else
format.json { render dashboard_error_response(result) }
......@@ -56,7 +59,7 @@ module MetricsDashboard
def dashboard_error_response(result)
{
status: result[:http_status],
status: result[:http_status] || :bad_request,
json: result.slice(:all_dashboards, :message, :status)
}
end
......
......@@ -199,8 +199,7 @@ class Projects::EnvironmentsController < Projects::ApplicationController
def metrics_dashboard_params
params
.permit(:embedded, :group, :title, :y_label)
.to_h.symbolize_keys
.permit(:embedded, :group, :title, :y_label, :dashboard_path, :environment)
.merge(dashboard_path: params[:dashboard], environment: environment)
end
......
......@@ -2,6 +2,9 @@
class Projects::GrafanaApiController < Projects::ApplicationController
include RenderServiceResults
include MetricsDashboard
before_action :validate_feature_enabled!, only: [:metrics_dashboard]
def proxy
result = ::Grafana::ProxyService.new(
......@@ -19,6 +22,14 @@ class Projects::GrafanaApiController < Projects::ApplicationController
private
def metrics_dashboard_params
params.permit(:embedded, :grafana_url)
end
def validate_feature_enabled!
render_403 unless Feature.enabled?(:gfm_grafana_integration)
end
def query_params
params.permit(:query, :start, :end, :step)
end
......
......@@ -187,9 +187,10 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
resource :import, only: [:new, :create, :show]
resource :avatar, only: [:show, :destroy]
get 'grafana/proxy/:datasource_id/*proxy_path',
to: 'grafana_api#proxy',
as: :grafana_api
scope :grafana, as: :grafana_api do
get 'proxy/:datasource_id/*proxy_path', to: 'grafana_api#proxy'
get :metrics_dashboard, to: 'grafana_api#metrics_dashboard'
end
end
# End of the /-/ scope.
......
# frozen_string_literal: true
module Banzai
module Filter
# HTML filter that inserts a placeholder element for each
# reference to a grafana dashboard.
class InlineGrafanaMetricsFilter < Banzai::Filter::InlineEmbedsFilter
# Placeholder element for the frontend to use as an
# injection point for charts.
def create_element(params)
begin_loading_dashboard(params[:url])
doc.document.create_element(
'div',
class: 'js-render-metrics',
'data-dashboard-url': metrics_dashboard_url(params)
)
end
def embed_params(node)
return unless Feature.enabled?(:gfm_grafana_integration)
query_params = Gitlab::Metrics::Dashboard::Url.parse_query(node['href'])
return unless [:panelId, :from, :to].all? do |param|
query_params.include?(param)
end
{ url: node['href'], start: query_params[:from], end: query_params[:to] }
end
# Selects any links with an href contains the configured
# grafana domain for the project
def xpath_search
return unless grafana_url.present?
%(descendant-or-self::a[starts-with(@href, '#{grafana_url}')])
end
private
def project
context[:project]
end
def grafana_url
project&.grafana_integration&.grafana_url
end
def metrics_dashboard_url(params)
Gitlab::Routing.url_helpers.project_grafana_api_metrics_dashboard_url(
project,
embedded: true,
grafana_url: params[:url],
start: format_time(params[:start]),
end: format_time(params[:end])
)
end
# Formats a timestamp from Grafana for compatibility with
# parsing in JS via `new Date(timestamp)`
#
# @param time [String] Represents miliseconds since epoch
def format_time(time)
Time.at(time.to_i / 1000).utc.strftime('%FT%TZ')
end
# Fetches a dashboard and caches the result for the
# FE to fetch quickly while rendering charts
def begin_loading_dashboard(url)
::Gitlab::Metrics::Dashboard::Finder.find(
project,
embedded: true,
grafana_url: url
)
end
end
end
end
......@@ -8,14 +8,17 @@ module Banzai
include Gitlab::Utils::StrongMemoize
METRICS_CSS_CLASS = '.js-render-metrics'
URL = Gitlab::Metrics::Dashboard::Url
Embed = Struct.new(:project_path, :permission)
# Finds all embeds based on the css class the FE
# uses to identify the embedded content, removing
# only unnecessary nodes.
def call
nodes.each do |node|
path = paths_by_node[node]
user_has_access = user_access_by_path[path]
embed = embeds_by_node[node]
user_has_access = user_access_by_embed[embed]
node.remove unless user_has_access
end
......@@ -30,40 +33,69 @@ module Banzai
end
# Returns all nodes which the FE will identify as
# a metrics dashboard placeholder element
# a metrics embed placeholder element
#
# @return [Nokogiri::XML::NodeSet]
def nodes
@nodes ||= doc.css(METRICS_CSS_CLASS)
end
# Maps a node to the full path of a project.
# Maps a node to key properties of an embed.
# Memoized so we only need to run the regex to get
# the project full path from the url once per node.
#
# @return [Hash<Nokogiri::XML::Node, String>]
def paths_by_node
strong_memoize(:paths_by_node) do
nodes.each_with_object({}) do |node, paths|
paths[node] = path_for_node(node)
# @return [Hash<Nokogiri::XML::Node, Embed>]
def embeds_by_node
strong_memoize(:embeds_by_node) do
nodes.each_with_object({}) do |node, embeds|
embed = Embed.new
url = node.attribute('data-dashboard-url').to_s
set_path_and_permission(embed, url, URL.regex, :read_environment)
set_path_and_permission(embed, url, URL.grafana_regex, :read_project) unless embed.permission
embeds[node] = embed if embed.permission
end
end
end
# Gets a project's full_path from the dashboard url
# in the placeholder node. The FE will use the attr
# `data-dashboard-url`, so we want to check against that
# attribute directly in case a user has manually
# created a metrics element (rather than supporting
# an alternate attr in InlineMetricsFilter).
# Attempts to determine the path and permission attributes
# of a url based on expected dashboard url formats and
# sets the attributes on an Embed object
#
# @return [String]
def path_for_node(node)
url = node.attribute('data-dashboard-url').to_s
Gitlab::Metrics::Dashboard::Url.regex.match(url) do |m|
# @param embed [Embed]
# @param url [String]
# @param regex [RegExp]
# @param permission [Symbol]
def set_path_and_permission(embed, url, regex, permission)
return unless path = regex.match(url) do |m|
"#{$~[:namespace]}/#{$~[:project]}"
end
embed.project_path = path
embed.permission = permission
end
# Returns a mapping representing whether the current user
# has permission to view the embed for the project.
# Determined in a batch
#
# @return [Hash<Embed, Boolean>]
def user_access_by_embed
strong_memoize(:user_access_by_embed) do
unique_embeds.each_with_object({}) do |embed, access|
project = projects_by_path[embed.project_path]
access[embed] = Ability.allowed?(user, embed.permission, project)
end
end
end
# Returns a unique list of embeds
#
# @return [Array<Embed>]
def unique_embeds
embeds_by_node.values.uniq
end
# Maps a project's full path to a Project object.
......@@ -74,22 +106,17 @@ module Banzai
def projects_by_path
strong_memoize(:projects_by_path) do
Project.eager_load(:route, namespace: [:route])
.where_full_path_in(paths_by_node.values.uniq)
.where_full_path_in(unique_project_paths)
.index_by(&:full_path)
end
end
# Returns a mapping representing whether the current user
# has permission to view the metrics for the project.
# Determined in a batch
# Returns a list of the full_paths of every project which
# has an embed in the doc
#
# @return [Hash<Project, Boolean>]
def user_access_by_path
strong_memoize(:user_access_by_path) do
projects_by_path.each_with_object({}) do |(path, project), access|
access[path] = Ability.allowed?(user, :read_environment, project)
end
end
# @return [Array<String>]
def unique_project_paths
embeds_by_node.values.map(&:project_path).uniq
end
end
end
......
......@@ -30,6 +30,7 @@ module Banzai
Filter::ImageLazyLoadFilter,
Filter::ImageLinkFilter,
Filter::InlineMetricsFilter,
Filter::InlineGrafanaMetricsFilter,
Filter::TableOfContentsFilter,
Filter::AutolinkFilter,
Filter::ExternalLinkFilter,
......
......@@ -12,6 +12,7 @@ module Gitlab
# @param project [Project]
# @param user [User]
# @param environment [Environment]
# @param options [Hash<Symbol,Any>]
# @param options - embedded [Boolean] Determines whether the
# dashboard is to be rendered as part of an
# issue or location other than the primary
......@@ -31,6 +32,8 @@ module Gitlab
# @param options - cluster [Cluster]
# @param options - cluster_type [Symbol] The level of
# cluster, one of [:admin, :project, :group]
# @param options - grafana_url [String] URL pointing
# to a grafana dashboard panel
# @return [Hash]
def find(project, user, options = {})
service_for(options)
......
......@@ -18,6 +18,7 @@ module Gitlab
# @return [Gitlab::Metrics::Dashboard::Services::BaseService]
def call(params)
return SERVICES::CustomMetricEmbedService if custom_metric_embed?(params)
return SERVICES::GrafanaMetricEmbedService if grafana_metric_embed?(params)
return SERVICES::DynamicEmbedService if dynamic_embed?(params)
return SERVICES::DefaultEmbedService if params[:embedded]
return SERVICES::SystemDashboardService if system_dashboard?(params[:dashboard_path])
......@@ -40,6 +41,10 @@ module Gitlab
SERVICES::CustomMetricEmbedService.valid_params?(params)
end
def grafana_metric_embed?(params)
SERVICES::GrafanaMetricEmbedService.valid_params?(params)
end
def dynamic_embed?(params)
SERVICES::DynamicEmbedService.valid_params?(params)
end
......
......@@ -14,17 +14,31 @@ module Gitlab
def regex
%r{
(?<url>
#{Regexp.escape(Gitlab.config.gitlab.url)}
\/#{Project.reference_pattern}
#{gitlab_pattern}
#{project_pattern}
(?:\/\-)?
\/environments
\/(?<environment>\d+)
\/metrics
(?<query>
\?[a-zA-Z0-9%.()+_=-]+
(&[a-zA-Z0-9%.()+_=-]+)*
)?
(?<anchor>\#[a-z0-9_-]+)?
#{query_pattern}
#{anchor_pattern}
)
}x
end
# Matches dashboard urls for a Grafana embed.
#
# EX - https://<host>/<namespace>/<project>/grafana/metrics_dashboard
def grafana_regex
%r{
(?<url>
#{gitlab_pattern}
#{project_pattern}
(?:\/\-)?
\/grafana
\/metrics_dashboard
#{query_pattern}
#{anchor_pattern}
)
}x
end
......@@ -45,6 +59,24 @@ module Gitlab
def build_dashboard_url(*args)
Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args)
end
private
def gitlab_pattern
Regexp.escape(Gitlab.config.gitlab.url)
end
def project_pattern
"\/#{Project.reference_pattern}"
end
def query_pattern
'(?<query>\?[a-zA-Z0-9%.()+_=-]+(&[a-zA-Z0-9%.()+_=-]+)*)?'
end
def anchor_pattern
'(?<anchor>\#[a-z0-9_-]+)?'
end
end
end
end
......
......@@ -31,11 +31,13 @@ describe MetricsDashboard do
end
context 'when params are provided' do
let(:params) { { environment: environment } }
before do
allow(controller).to receive(:project).and_return(project)
allow(controller)
.to receive(:metrics_dashboard_params)
.and_return(environment: environment)
.and_return(params)
end
it 'returns the specified dashboard' do
......@@ -43,6 +45,15 @@ describe MetricsDashboard do
expect(json_response).not_to have_key('all_dashboards')
end
context 'when the params are in an alternate format' do
let(:params) { ActionController::Parameters.new({ environment: environment }).permit! }
it 'returns the specified dashboard' do
expect(json_response['dashboard']['dashboard']).to eq('Environment metrics')
expect(json_response).not_to have_key('all_dashboards')
end
end
context 'when parameters are provided and the list of all dashboards is required' do
before do
allow(controller).to receive(:include_all_dashboards?).and_return(true)
......
......@@ -94,4 +94,87 @@ describe Projects::GrafanaApiController do
end
end
end
describe 'GET #metrics_dashboard' do
let(:service_result) { { status: :success, dashboard: '{}' } }
let(:params) do
{
format: :json,
embedded: true,
grafana_url: 'https://grafana.example.com',
namespace_id: project.namespace.full_path,
project_id: project.name
}
end
before do
allow(Gitlab::Metrics::Dashboard::Finder)
.to receive(:find)
.and_return(service_result)
end
context 'when the result is still processing' do
let(:service_result) { nil }
it 'returns 204 no content' do
get :metrics_dashboard, params: params
expect(response).to have_gitlab_http_status(:no_content)
end
end
context 'when the result was successful' do
it 'returns the dashboard response' do
get :metrics_dashboard, params: params
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to eq({
'dashboard' => '{}',
'status' => 'success'
})
end
end
context 'when an error has occurred' do
shared_examples_for 'error response' do |http_status|
it "returns #{http_status}" do
get :metrics_dashboard, params: params
expect(response).to have_gitlab_http_status(http_status)
expect(json_response['status']).to eq('error')
expect(json_response['message']).to eq('error message')
end
end
context 'with an error accessing grafana' do
let(:service_result) do
{
http_status: :service_unavailable,
status: :error,
message: 'error message'
}
end
it_behaves_like 'error response', :service_unavailable
end
context 'with a processing error' do
let(:service_result) { { status: :error, message: 'error message' } }
it_behaves_like 'error response', :bad_request
end
end
context 'when grafana embeds are not enabled' do
before do
stub_feature_flags(gfm_grafana_integration: false)
end
it 'returns 403 immediately' do
get :metrics_dashboard, params: params
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Banzai::Filter::InlineGrafanaMetricsFilter do
include FilterSpecHelper
let_it_be(:project) { create(:project) }
let_it_be(:grafana_integration) { create(:grafana_integration, project: project) }
let(:input) { %(<a href="#{url}">example</a>) }
let(:doc) { filter(input) }
let(:url) { grafana_integration.grafana_url + dashboard_path }
let(:dashboard_path) do
'/d/XDaNK6amz/gitlab-omnibus-redis' \
'?from=1570397739557&to=1570484139557' \
'&var-instance=All&panelId=14'
end
context 'when feature flag is disabled' do
before do
stub_feature_flags(gfm_grafana_integration: false)
end
it 'leaves the markdown unchanged' do
expect(unescape(doc.to_s)).to eq(input)
end
end
it 'appends a metrics charts placeholder with dashboard url after metrics links' do
node = doc.at_css('.js-render-metrics')
expect(node).to be_present
dashboard_url = urls.project_grafana_api_metrics_dashboard_url(
project,
embedded: true,
grafana_url: url,
start: "2019-10-06T21:35:39Z",
end: "2019-10-07T21:35:39Z"
)
expect(node.attribute('data-dashboard-url').to_s).to eq(dashboard_url)
end
context 'when the dashboard link is part of a paragraph' do
let(:paragraph) { %(This is an <a href="#{url}">example</a> of metrics.) }
let(:input) { %(<p>#{paragraph}</p>) }
it 'appends the charts placeholder after the enclosing paragraph' do
expect(unescape(doc.at_css('p').to_s)).to include(paragraph)
expect(doc.at_css('.js-render-metrics')).to be_present
end
end
context 'when grafana is not configured' do
before do
allow(project).to receive(:grafana_integration).and_return(nil)
end
it 'leaves the markdown unchanged' do
expect(unescape(doc.to_s)).to eq(input)
end
end
context 'when parameters are missing' do
let(:dashboard_path) { '/d/XDaNK6amz/gitlab-omnibus-redis' }
it 'leaves the markdown unchanged' do
expect(unescape(doc.to_s)).to eq(input)
end
end
private
# Nokogiri escapes the URLs, but we don't care about that
# distinction for the purposes of this filter
def unescape(html)
CGI.unescapeHTML(html)
end
end
......@@ -18,30 +18,48 @@ describe Banzai::Filter::InlineMetricsRedactorFilter do
end
context 'with a metrics charts placeholder' do
let(:input) { %(<div class="js-render-metrics" data-dashboard-url="#{url}"></div>) }
shared_examples_for 'a supported metrics dashboard url' do
context 'no user is logged in' do
it 'redacts the placeholder' do
expect(doc.to_s).to be_empty
end
end
context 'no user is logged in' do
it 'redacts the placeholder' do
expect(doc.to_s).to be_empty
context 'the user does not have permission do see charts' do
let(:doc) { filter(input, current_user: build(:user)) }
it 'redacts the placeholder' do
expect(doc.to_s).to be_empty
end
end
end
context 'the user does not have permission do see charts' do
let(:doc) { filter(input, current_user: build(:user)) }
context 'the user has requisite permissions' do
let(:user) { create(:user) }
let(:doc) { filter(input, current_user: user) }
it 'redacts the placeholder' do
expect(doc.to_s).to be_empty
it 'leaves the placeholder' do
project.add_maintainer(user)
expect(doc.to_s).to eq input
end
end
end
context 'the user has requisite permissions' do
let(:user) { create(:user) }
let(:doc) { filter(input, current_user: user) }
let(:input) { %(<div class="js-render-metrics" data-dashboard-url="#{url}"></div>) }
it 'leaves the placeholder' do
project.add_maintainer(user)
it_behaves_like 'a supported metrics dashboard url'
context 'for a grafana dashboard' do
let(:url) { urls.project_grafana_api_metrics_dashboard_url(project, embedded: true) }
it_behaves_like 'a supported metrics dashboard url'
end
expect(doc.to_s).to eq input
context 'for an internal non-dashboard url' do
let(:url) { urls.project_url(project) }
it 'leaves the placeholder' do
expect(doc.to_s).to be_empty
end
end
end
......
......@@ -75,6 +75,17 @@ describe Gitlab::Metrics::Dashboard::ServiceSelector do
it { is_expected.to be Metrics::Dashboard::CustomMetricEmbedService }
end
context 'with a grafana link' do
let(:arguments) do
{
embedded: true,
grafana_url: 'https://grafana.example.com'
}
end
it { is_expected.to be Metrics::Dashboard::GrafanaMetricEmbedService }
end
end
end
end
......@@ -3,13 +3,41 @@
require 'spec_helper'
describe Gitlab::Metrics::Dashboard::Url do
describe '#regex' do
it 'returns a regular expression' do
expect(described_class.regex).to be_a Regexp
end
shared_examples_for 'a regex which matches the expected url' do
it { is_expected.to be_a Regexp }
it 'matches a metrics dashboard link with named params' do
url = Gitlab::Routing.url_helpers.metrics_namespace_project_environment_url(
expect(subject).to match url
subject.match(url) do |m|
expect(m.named_captures).to eq expected_params
end
end
end
shared_examples_for 'does not match non-matching urls' do
it 'does not match other gitlab urls that contain the term metrics' do
url = Gitlab::Routing.url_helpers.active_common_namespace_project_prometheus_metrics_url('foo', 'bar', :json)
expect(subject).not_to match url
end
it 'does not match other gitlab urls' do
url = Gitlab.config.gitlab.url
expect(subject).not_to match url
end
it 'does not match non-gitlab urls' do
url = 'https://www.super_awesome_site.com/'
expect(subject).not_to match url
end
end
describe '#regex' do
let(:url) do
Gitlab::Routing.url_helpers.metrics_namespace_project_environment_url(
'foo',
'bar',
1,
......@@ -18,8 +46,10 @@ describe Gitlab::Metrics::Dashboard::Url do
group: 'awesome group',
anchor: 'title'
)
end
expected_params = {
let(:expected_params) do
{
'url' => url,
'namespace' => 'foo',
'project' => 'bar',
......@@ -27,31 +57,40 @@ describe Gitlab::Metrics::Dashboard::Url do
'query' => '?dashboard=config%2Fprometheus%2Fcommon_metrics.yml&group=awesome+group&start=2019-08-02T05%3A43%3A09.000Z',
'anchor' => '#title'
}
expect(described_class.regex).to match url
described_class.regex.match(url) do |m|
expect(m.named_captures).to eq expected_params
end
end
it 'does not match other gitlab urls that contain the term metrics' do
url = Gitlab::Routing.url_helpers.active_common_namespace_project_prometheus_metrics_url('foo', 'bar', :json)
subject { described_class.regex }
expect(described_class.regex).not_to match url
end
it_behaves_like 'a regex which matches the expected url'
it_behaves_like 'does not match non-matching urls'
end
it 'does not match other gitlab urls' do
url = Gitlab.config.gitlab.url
describe '#grafana_regex' do
let(:url) do
Gitlab::Routing.url_helpers.namespace_project_grafana_api_metrics_dashboard_url(
'foo',
'bar',
start: '2019-08-02T05:43:09.000Z',
dashboard: 'config/prometheus/common_metrics.yml',
group: 'awesome group',
anchor: 'title'
)
end
expect(described_class.regex).not_to match url
let(:expected_params) do
{
'url' => url,
'namespace' => 'foo',
'project' => 'bar',
'query' => '?dashboard=config%2Fprometheus%2Fcommon_metrics.yml&group=awesome+group&start=2019-08-02T05%3A43%3A09.000Z',
'anchor' => '#title'
}
end
it 'does not match non-gitlab urls' do
url = 'https://www.super_awesome_site.com/'
subject { described_class.grafana_regex }
expect(described_class.regex).not_to match url
end
it_behaves_like 'a regex which matches the expected url'
it_behaves_like 'does not match non-matching urls'
end
describe '#build_dashboard_url' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment