Commit 746f0ec3 authored by Paweł Chojnacki's avatar Paweł Chojnacki Committed by Rémy Coutable

Add sidekiq metrics endpoint and add http server to sidekiq

parent b12107a0
---
title: Add Prometheus metrics exporter to Sidekiq
merge_request: 13082
author:
...@@ -590,6 +590,12 @@ production: &base ...@@ -590,6 +590,12 @@ production: &base
ip_whitelist: ip_whitelist:
- 127.0.0.0/8 - 127.0.0.0/8
# Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics
sidekiq_exporter:
# enabled: true
# address: localhost
# port: 3807
# #
# 5. Extra customization # 5. Extra customization
# ========================== # ==========================
......
...@@ -524,6 +524,10 @@ Settings.webpack.dev_server['port'] ||= 3808 ...@@ -524,6 +524,10 @@ Settings.webpack.dev_server['port'] ||= 3808
Settings['monitoring'] ||= Settingslogic.new({}) Settings['monitoring'] ||= Settingslogic.new({})
Settings.monitoring['ip_whitelist'] ||= ['127.0.0.1/8'] Settings.monitoring['ip_whitelist'] ||= ['127.0.0.1/8']
Settings.monitoring['unicorn_sampler_interval'] ||= 10 Settings.monitoring['unicorn_sampler_interval'] ||= 10
Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({})
Settings.monitoring.sidekiq_exporter['enabled'] ||= false
Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost'
Settings.monitoring.sidekiq_exporter['port'] ||= 3807
# #
# Testing settings # Testing settings
......
...@@ -10,3 +10,9 @@ Prometheus::Client.configure do |config| ...@@ -10,3 +10,9 @@ Prometheus::Client.configure do |config|
config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir') config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir')
end end
end end
Sidekiq.configure_server do |config|
config.on(:startup) do
Gitlab::Metrics::SidekiqMetricsExporter.instance.start
end
end
module Gitlab
class Daemon
def self.initialize_instance(*args)
raise "#{name} singleton instance already initialized" if @instance
@instance = new(*args)
Kernel.at_exit(&@instance.method(:stop))
@instance
end
def self.instance
@instance ||= initialize_instance
end
attr_reader :thread
def thread?
!thread.nil?
end
def initialize
@mutex = Mutex.new
end
def enabled?
true
end
def start
return unless enabled?
@mutex.synchronize do
return thread if thread?
@thread = Thread.new { start_working }
end
end
def stop
@mutex.synchronize do
return unless thread?
stop_working
if thread
thread.wakeup if thread.alive?
thread.join
@thread = nil
end
end
end
private
def start_working
raise NotImplementedError
end
def stop_working
# no-ops
end
end
end
require 'logger' require 'logger'
module Gitlab module Gitlab
module Metrics module Metrics
class BaseSampler class BaseSampler < Daemon
def self.initialize_instance(*args)
raise "#{name} singleton instance already initialized" if @instance
@instance = new(*args)
at_exit(&@instance.method(:stop))
@instance
end
def self.instance
@instance
end
attr_reader :running
# interval - The sampling interval in seconds. # interval - The sampling interval in seconds.
def initialize(interval) def initialize(interval)
interval_half = interval.to_f / 2 interval_half = interval.to_f / 2
...@@ -22,44 +9,7 @@ module Gitlab ...@@ -22,44 +9,7 @@ module Gitlab
@interval = interval @interval = interval
@interval_steps = (-interval_half..interval_half).step(0.1).to_a @interval_steps = (-interval_half..interval_half).step(0.1).to_a
@mutex = Mutex.new super()
end
def enabled?
true
end
def start
return unless enabled?
@mutex.synchronize do
return if running
@running = true
@thread = Thread.new do
sleep(sleep_interval)
while running
safe_sample
sleep(sleep_interval)
end
end
end
end
def stop
@mutex.synchronize do
return unless running
@running = false
if @thread
@thread.wakeup if @thread.alive?
@thread.join
@thread = nil
end
end
end end
def safe_sample def safe_sample
...@@ -81,7 +31,7 @@ module Gitlab ...@@ -81,7 +31,7 @@ module Gitlab
# potentially missing anything that happens in between samples). # potentially missing anything that happens in between samples).
# 2. Don't sample data at the same interval two times in a row. # 2. Don't sample data at the same interval two times in a row.
def sleep_interval def sleep_interval
while step = @interval_steps.sample while (step = @interval_steps.sample)
if step != @last_step if step != @last_step
@last_step = step @last_step = step
...@@ -89,6 +39,25 @@ module Gitlab ...@@ -89,6 +39,25 @@ module Gitlab
end end
end end
end end
private
attr_reader :running
def start_working
@running = true
sleep(sleep_interval)
while running
safe_sample
sleep(sleep_interval)
end
end
def stop_working
@running = false
end
end end
end end
end end
require 'webrick'
require 'prometheus/client/rack/exporter'
module Gitlab
module Metrics
class SidekiqMetricsExporter < Daemon
def enabled?
Gitlab::Metrics.metrics_folder_present? && settings.enabled
end
def settings
Settings.monitoring.sidekiq_exporter
end
private
attr_reader :server
def start_working
@server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address)
server.mount "/", Rack::Handler::WEBrick, rack_app
server.start
end
def stop_working
server.shutdown
@server = nil
end
def rack_app
Rack::Builder.app do
use Rack::Deflater
use ::Prometheus::Client::Rack::Exporter
run -> (env) { [404, {}, ['']] }
end
end
end
end
end
require 'spec_helper'
describe Gitlab::Daemon do
subject { described_class.new }
before do
allow(subject).to receive(:start_working)
allow(subject).to receive(:stop_working)
end
describe '.instance' do
before do
allow(Kernel).to receive(:at_exit)
end
after(:each) do
described_class.instance_variable_set(:@instance, nil)
end
it 'provides instance of Daemon' do
expect(described_class.instance).to be_instance_of(described_class)
end
it 'subsequent invocations provide the same instance' do
expect(described_class.instance).to eq(described_class.instance)
end
it 'creates at_exit hook when instance is created' do
expect(described_class.instance).not_to be_nil
expect(Kernel).to have_received(:at_exit)
end
end
describe 'when Daemon is enabled' do
before do
allow(subject).to receive(:enabled?).and_return(true)
end
describe 'when Daemon is stopped' do
describe '#start' do
it 'starts the Daemon' do
expect { subject.start.join }.to change { subject.thread? }.from(false).to(true)
expect(subject).to have_received(:start_working)
end
end
describe '#stop' do
it "doesn't shutdown stopped Daemon" do
expect { subject.stop }.not_to change { subject.thread? }
expect(subject).not_to have_received(:start_working)
end
end
end
describe 'when Daemon is running' do
before do
subject.start.join
end
describe '#start' do
it "doesn't start running Daemon" do
expect { subject.start.join }.not_to change { subject.thread? }
expect(subject).to have_received(:start_working).once
end
end
describe '#stop' do
it 'shutdowns Daemon' do
expect { subject.stop }.to change { subject.thread? }.from(true).to(false)
expect(subject).to have_received(:stop_working)
end
end
end
end
describe 'when Daemon is disabled' do
before do
allow(subject).to receive(:enabled?).and_return(false)
end
describe '#start' do
it "doesn't start working" do
expect(subject.start).to be_nil
expect { subject.start }.not_to change { subject.thread? }
expect(subject).not_to have_received(:start_working)
end
end
describe '#stop' do
it "doesn't stop working" do
expect { subject.stop }.not_to change { subject.thread? }
expect(subject).not_to have_received(:stop_working)
end
end
end
end
...@@ -11,7 +11,7 @@ describe Gitlab::Metrics::InfluxSampler do ...@@ -11,7 +11,7 @@ describe Gitlab::Metrics::InfluxSampler do
it 'runs once and gathers a sample at a given interval' do it 'runs once and gathers a sample at a given interval' do
expect(sampler).to receive(:sleep).with(a_kind_of(Numeric)).twice expect(sampler).to receive(:sleep).with(a_kind_of(Numeric)).twice
expect(sampler).to receive(:sample).once expect(sampler).to receive(:sample).once
expect(sampler).to receive(:running).and_return(false, true, false) expect(sampler).to receive(:running).and_return(true, false)
sampler.start.join sampler.start.join
end end
......
require 'spec_helper'
describe Gitlab::Metrics::SidekiqMetricsExporter do
let(:exporter) { described_class.new }
let(:server) { double('server') }
before do
allow(::WEBrick::HTTPServer).to receive(:new).and_return(server)
allow(server).to receive(:mount)
allow(server).to receive(:start)
allow(server).to receive(:shutdown)
end
describe 'when exporter is enabled' do
before do
allow(Settings.monitoring.sidekiq_exporter).to receive(:enabled).and_return(true)
end
describe 'when exporter is stopped' do
describe '#start' do
it 'starts the exporter' do
expect { exporter.start.join }.to change { exporter.thread? }.from(false).to(true)
expect(server).to have_received(:start)
end
describe 'with custom settings' do
let(:port) { 99999 }
let(:address) { 'sidekiq_exporter_address' }
before do
allow(Settings.monitoring.sidekiq_exporter).to receive(:port).and_return(port)
allow(Settings.monitoring.sidekiq_exporter).to receive(:address).and_return(address)
end
it 'starts server with port and address from settings' do
exporter.start.join
expect(::WEBrick::HTTPServer).to have_received(:new).with(
Port: port,
BindAddress: address
)
end
end
end
describe '#stop' do
it "doesn't shutdown stopped server" do
expect { exporter.stop }.not_to change { exporter.thread? }
expect(server).not_to have_received(:shutdown)
end
end
end
describe 'when exporter is running' do
before do
exporter.start.join
end
describe '#start' do
it "doesn't start running server" do
expect { exporter.start.join }.not_to change { exporter.thread? }
expect(server).to have_received(:start).once
end
end
describe '#stop' do
it 'shutdowns server' do
expect { exporter.stop }.to change { exporter.thread? }.from(true).to(false)
expect(server).to have_received(:shutdown)
end
end
end
end
describe 'when exporter is disabled' do
before do
allow(Settings.monitoring.sidekiq_exporter).to receive(:enabled).and_return(false)
end
describe '#start' do
it "doesn't start" do
expect(exporter.start).to be_nil
expect { exporter.start }.not_to change { exporter.thread? }
expect(server).not_to have_received(:start)
end
end
describe '#stop' do
it "doesn't shutdown" do
expect { exporter.stop }.not_to change { exporter.thread? }
expect(server).not_to have_received(:shutdown)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment