Commit 51d040ef authored by Sarah Yasonik's avatar Sarah Yasonik Committed by Rémy Coutable

Add proxy endpoint for grafana proxy

Adds a new endpoint which proxies calls to the grafana endpoint
which proxies calls to any grafana datasource configured to
support it. This will be used as a part of embedding prometheus
metrics in GFM via grafana.
parent 6d670814
# frozen_string_literal: true
module RenderServiceResults
extend ActiveSupport::Concern
def success_response(result)
render({
status: result[:http_status],
json: result[:body]
})
end
def continue_polling_response
render({
status: :no_content,
json: {
status: _('processing'),
message: _('Not ready yet. Try again later.')
}
})
end
def error_response(result)
render({
status: result[:http_status] || :bad_request,
json: { status: result[:status], message: result[:message] }
})
end
end
# frozen_string_literal: true
class Projects::Environments::PrometheusApiController < Projects::ApplicationController
include RenderServiceResults
before_action :authorize_read_prometheus!
before_action :environment
......@@ -12,21 +14,10 @@ class Projects::Environments::PrometheusApiController < Projects::ApplicationCon
proxy_params
).execute
if result.nil?
return render status: :no_content, json: {
status: _('processing'),
message: _('Not ready yet. Try again later.')
}
end
if result[:status] == :success
render status: result[:http_status], json: result[:body]
else
render(
status: result[:http_status] || :bad_request,
json: { status: result[:status], message: result[:message] }
)
end
return continue_polling_response if result.nil?
return error_response(result) if result[:status] == :error
success_response(result)
end
private
......
# frozen_string_literal: true
class Projects::GrafanaApiController < Projects::ApplicationController
include RenderServiceResults
def proxy
result = ::Grafana::ProxyService.new(
project,
params[:datasource_id],
params[:proxy_path],
query_params.to_h
).execute
return continue_polling_response if result.nil?
return error_response(result) if result[:status] == :error
success_response(result)
end
private
def query_params
params.permit(:query, :start, :end, :step)
end
end
......@@ -13,4 +13,8 @@ class GrafanaIntegration < ApplicationRecord
addressable_url: { enforce_sanitization: true, ascii_only: true }
validates :token, :project, presence: true
def client
@client ||= ::Grafana::Client.new(api_url: grafana_url.chomp('/'), token: token)
end
end
# frozen_string_literal: true
# Proxies calls to a Grafana-integrated Prometheus instance
# through the Grafana proxy API
# This allows us to fetch and render metrics in GitLab from a Prometheus
# instance for which dashboards are configured in Grafana
module Grafana
class ProxyService < BaseService
include ReactiveCaching
self.reactive_cache_key = ->(service) { service.cache_key }
self.reactive_cache_lease_timeout = 30.seconds
self.reactive_cache_refresh_interval = 30.seconds
self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) }
attr_accessor :project, :datasource_id, :proxy_path, :query_params
# @param project_id [Integer] Project id for which grafana is configured.
#
# See #initialize for other parameters.
def self.from_cache(project_id, datasource_id, proxy_path, query_params)
project = Project.find(project_id)
new(project, datasource_id, proxy_path, query_params)
end
# @param project [Project] Project for which grafana is configured.
# @param datasource_id [String] Grafana datasource id for Prometheus instance
# @param proxy_path [String] Path to Prometheus endpoint; EX) 'api/v1/query_range'
# @param query_params [Hash<String, String>] Supported params: [query, start, end, step]
def initialize(project, datasource_id, proxy_path, query_params)
@project = project
@datasource_id = datasource_id
@proxy_path = proxy_path
@query_params = query_params
end
def execute
return cannot_proxy_response unless client
with_reactive_cache(*cache_key) { |result| result }
end
def calculate_reactive_cache(*)
return cannot_proxy_response unless client
response = client.proxy_datasource(
datasource_id: datasource_id,
proxy_path: proxy_path,
query: query_params
)
success(http_status: response.code, body: response.body)
rescue ::Grafana::Client::Error => error
service_unavailable_response(error)
end
# Required for ReactiveCaching; Usage overridden by
# self.reactive_cache_worker_finder
def id
nil
end
def cache_key
[project.id, datasource_id, proxy_path, query_params]
end
private
def client
project.grafana_integration&.client
end
def service_unavailable_response(exception)
error(exception.message, :service_unavailable)
end
def cannot_proxy_response
error('Proxy support for this API is not available currently')
end
end
end
---
title: Add endpoint to proxy requests to grafana's proxy endpoint
merge_request: 18210
author:
type: added
......@@ -186,6 +186,10 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
resource :import, only: [:new, :create, :show]
resource :avatar, only: [:show, :destroy]
get 'grafana/proxy/:datasource_id/*proxy_path',
to: 'grafana_api#proxy',
as: :grafana_api
end
# End of the /-/ scope.
......
# frozen_string_literal: true
module Grafana
class Client
Error = Class.new(StandardError)
# @param api_url [String] Base URL of the Grafana instance
# @param token [String] Admin-level API token for instance
def initialize(api_url:, token:)
@api_url = api_url
@token = token
end
# @param datasource_id [String] Grafana ID for the datasource
# @param proxy_path [String] Path to proxy - ex) 'api/v1/query_range'
def proxy_datasource(datasource_id:, proxy_path:, query: {})
http_get("#{@api_url}/api/datasources/proxy/#{datasource_id}/#{proxy_path}", query: query)
end
private
def http_get(url, params = {})
response = handle_request_exceptions do
Gitlab::HTTP.get(url, **request_params.merge(params))
end
handle_response(response)
end
def request_params
{
headers: {
'Authorization' => "Bearer #{@token}",
'Accept' => 'application/json',
'Content-Type' => 'application/json'
},
follow_redirects: false
}
end
def handle_request_exceptions
yield
rescue Gitlab::HTTP::Error
raise_error 'Error when connecting to Grafana'
rescue Net::OpenTimeout
raise_error 'Connection to Grafana timed out'
rescue SocketError
raise_error 'Received SocketError when trying to connect to Grafana'
rescue OpenSSL::SSL::SSLError
raise_error 'Grafana returned invalid SSL data'
rescue Errno::ECONNREFUSED
raise_error 'Connection refused'
rescue => e
raise_error "Grafana request failed due to #{e.class}"
end
def handle_response(response)
return response if response.code == 200
raise_error "Grafana response status code: #{response.code}"
end
def raise_error(message)
raise Client::Error, message
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Projects::GrafanaApiController do
let_it_be(:project) { create(:project) }
let_it_be(:user) { create(:user) }
before do
project.add_reporter(user)
sign_in(user)
end
describe 'GET #proxy' do
let(:proxy_service) { instance_double(Grafana::ProxyService) }
let(:params) do
{
namespace_id: project.namespace.full_path,
project_id: project.name,
proxy_path: 'api/v1/query_range',
datasource_id: '1',
query: 'rate(relevant_metric)',
start: '1570441248',
end: '1570444848',
step: '900'
}
end
before do
allow(Grafana::ProxyService).to receive(:new).and_return(proxy_service)
allow(proxy_service).to receive(:execute).and_return(service_result)
end
shared_examples_for 'error response' do |http_status|
it "returns #{http_status}" do
get :proxy, params: params
expect(response).to have_gitlab_http_status(http_status)
expect(json_response['status']).to eq('error')
expect(json_response['message']).to eq('error message')
end
end
context 'with a successful result' do
let(:service_result) { { status: :success, body: '{}' } }
it 'returns a grafana datasource response' do
get :proxy, params: params
expect(Grafana::ProxyService)
.to have_received(:new)
.with(project, '1', 'api/v1/query_range',
params.slice(:query, :start, :end, :step).stringify_keys)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to eq({})
end
end
context 'when the request is still unavailable' do
let(:service_result) { nil }
it 'returns 204 no content' do
get :proxy, params: params
expect(response).to have_gitlab_http_status(:no_content)
expect(json_response['status']).to eq('processing')
expect(json_response['message']).to eq('Not ready yet. Try again later.')
end
end
context 'when an error has occurred' do
context 'with an error accessing grafana' do
let(:service_result) do
{
http_status: :service_unavailable,
status: :error,
message: 'error message'
}
end
it_behaves_like 'error response', :service_unavailable
end
context 'with a processing error' do
let(:service_result) do
{
status: :error,
message: 'error message'
}
end
it_behaves_like 'error response', :bad_request
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Grafana::Client do
let(:grafana_url) { 'https://grafanatest.com/-/grafana-project' }
let(:token) { 'test-token' }
subject(:client) { described_class.new(api_url: grafana_url, token: token) }
shared_examples 'calls grafana api' do
let!(:grafana_api_request) { stub_grafana_request(grafana_api_url) }
it 'calls grafana api' do
subject
expect(grafana_api_request).to have_been_requested
end
end
shared_examples 'no redirects' do
let(:redirect_to) { 'https://redirected.example.com' }
let(:other_url) { 'https://grafana.example.org' }
let!(:redirected_req_stub) { stub_grafana_request(other_url) }
let!(:redirect_req_stub) do
stub_grafana_request(
grafana_api_url,
status: 302,
headers: { location: redirect_to }
)
end
it 'does not follow redirects' do
expect { subject }.to raise_exception(
Grafana::Client::Error,
'Grafana response status code: 302'
)
expect(redirect_req_stub).to have_been_requested
expect(redirected_req_stub).not_to have_been_requested
end
end
shared_examples 'handles exceptions' do
exceptions = {
Gitlab::HTTP::Error => 'Error when connecting to Grafana',
Net::OpenTimeout => 'Connection to Grafana timed out',
SocketError => 'Received SocketError when trying to connect to Grafana',
OpenSSL::SSL::SSLError => 'Grafana returned invalid SSL data',
Errno::ECONNREFUSED => 'Connection refused',
StandardError => 'Grafana request failed due to StandardError'
}
exceptions.each do |exception, message|
context "#{exception}" do
before do
stub_request(:get, grafana_api_url).to_raise(exception)
end
it do
expect { subject }
.to raise_exception(Grafana::Client::Error, message)
end
end
end
end
describe '#proxy_datasource' do
let(:grafana_api_url) do
'https://grafanatest.com/-/grafana-project/' \
'api/datasources/proxy/' \
'1/api/v1/query_range' \
'?query=rate(relevant_metric)' \
'&start=1570441248&end=1570444848&step=900'
end
subject do
client.proxy_datasource(
datasource_id: '1',
proxy_path: 'api/v1/query_range',
query: {
query: 'rate(relevant_metric)',
start: 1570441248,
end: 1570444848,
step: 900
}
)
end
it_behaves_like 'calls grafana api'
it_behaves_like 'no redirects'
it_behaves_like 'handles exceptions'
end
private
def stub_grafana_request(url, body: {}, status: 200, headers: {})
stub_request(:get, url)
.to_return(
status: status,
headers: { 'Content-Type' => 'application/json' }.merge(headers),
body: body.to_json
)
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Grafana::ProxyService do
include ReactiveCachingHelpers
let_it_be(:project) { create(:project) }
let_it_be(:grafana_integration) { create(:grafana_integration, project: project) }
let(:proxy_path) { 'api/v1/query_range' }
let(:datasource_id) { '1' }
let(:query_params) do
{
'query' => 'rate(relevant_metric)',
'start' => '1570441248',
'end' => '1570444848',
'step' => '900'
}
end
let(:cache_params) { [project.id, datasource_id, proxy_path, query_params] }
let(:service) do
described_class.new(project, datasource_id, proxy_path, query_params)
end
shared_examples_for 'initializes an instance' do
it 'initializes an instance of ProxyService class' do
expect(subject).to be_an_instance_of(described_class)
expect(subject.project).to eq(project)
expect(subject.datasource_id).to eq('1')
expect(subject.proxy_path).to eq('api/v1/query_range')
expect(subject.query_params).to eq(query_params)
end
end
describe '.from_cache' do
subject { described_class.from_cache(*cache_params) }
it_behaves_like 'initializes an instance'
end
describe '#initialize' do
subject { service }
it_behaves_like 'initializes an instance'
end
describe '#execute' do
subject(:result) { service.execute }
context 'when grafana integration is not configured' do
before do
allow(project).to receive(:grafana_integration).and_return(nil)
end
it 'returns error' do
expect(result).to eq(
status: :error,
message: 'Proxy support for this API is not available currently'
)
end
end
context 'with caching', :use_clean_rails_memory_store_caching do
context 'when value not present in cache' do
it 'returns nil' do
expect(ReactiveCachingWorker)
.to receive(:perform_async)
.with(service.class, service.id, *cache_params)
expect(result).to eq(nil)
end
end
context 'when value present in cache' do
let(:return_value) { { 'http_status' => 200, 'body' => 'body' } }
before do
stub_reactive_cache(service, return_value, cache_params)
end
it 'returns cached value' do
expect(ReactiveCachingWorker)
.not_to receive(:perform_async)
.with(service.class, service.id, *cache_params)
expect(result[:http_status]).to eq(return_value[:http_status])
expect(result[:body]).to eq(return_value[:body])
end
end
end
context 'call prometheus api' do
let(:client) { service.send(:client) }
before do
synchronous_reactive_cache(service)
end
context 'connection to grafana datasource succeeds' do
let(:response) { instance_double(Gitlab::HTTP::Response) }
let(:status_code) { 400 }
let(:body) { 'body' }
before do
allow(client).to receive(:proxy_datasource).and_return(response)
allow(response).to receive(:code).and_return(status_code)
allow(response).to receive(:body).and_return(body)
end
it 'returns the http status code and body from prometheus' do
expect(result).to eq(
http_status: status_code,
body: body,
status: :success
)
end
end
context 'connection to grafana datasource fails' do
before do
allow(client).to receive(:proxy_datasource)
.and_raise(Grafana::Client::Error, 'Network connection error')
end
it 'returns error' do
expect(result).to eq(
status: :error,
message: 'Network connection error',
http_status: :service_unavailable
)
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment