Create a separate metrics server that can serve Sidekiq metrics

bin/metrics-server

+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../metrics_server/metrics_server'
+
+begin
+  target = ENV['METRICS_SERVER_TARGET']
+  raise "Required: METRICS_SERVER_TARGET=[sidekiq]" unless target == 'sidekiq'
+
+  metrics_dir = ENV["prometheus_multiproc_dir"] || File.absolute_path("tmp/prometheus_multiproc_dir/#{target}")
+
+  # Re-raise exceptions in threads on the main thread.
+  Thread.abort_on_exception = true
+  MetricsServer.new(target, metrics_dir).start
+end

lib/gitlab/utils/strong_memoize.rb

 # frozen_string_literal: true
 
-require_dependency 'gitlab/utils'
-
 module Gitlab
   module Utils
     module StrongMemoize

metrics_server/dependencies.rb

+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+require 'shellwords'
+require 'fileutils'
+
+require 'active_support/concern'
+require 'active_support/inflector'
+
+require 'prometheus/client'
+require 'rack'
+
+require_relative 'settings_overrides'
+
+require_relative '../lib/gitlab/daemon'
+require_relative '../lib/gitlab/utils/strong_memoize'
+require_relative '../lib/prometheus/cleanup_multiproc_dir_service'
+require_relative '../lib/gitlab/metrics/prometheus'
+require_relative '../lib/gitlab/metrics'
+require_relative '../lib/gitlab/metrics/exporter/base_exporter'
+require_relative '../lib/gitlab/metrics/exporter/sidekiq_exporter'
+require_relative '../lib/gitlab/health_checks/probes/collection'
+require_relative '../lib/gitlab/health_checks/probes/status'
+
+# rubocop:enable Naming/FileName

metrics_server/metrics_server.rb

+# frozen_string_literal: true
+
+require_relative '../config/bundler_setup'
+
+require_relative 'dependencies'
+
+class MetricsServer # rubocop:disable Gitlab/NamespacedClass
+  class << self
+    def spawn(target, gitlab_config: nil)
+      cmd = "#{Rails.root}/bin/metrics-server"
+      env = {
+        'METRICS_SERVER_TARGET' => target,
+        'GITLAB_CONFIG' => gitlab_config
+      }
+
+      Process.spawn(env, cmd, err: $stderr, out: $stdout).tap do |pid|
+        Process.detach(pid)
+      end
+    end
+  end
+
+  def initialize(target, metrics_dir)
+    @target = target
+    @metrics_dir = metrics_dir
+  end
+
+  def start
+    ::Prometheus::Client.configure do |config|
+      config.multiprocess_files_dir = @metrics_dir
+    end
+
+    FileUtils.mkdir_p(@metrics_dir, mode: 0700)
+    ::Prometheus::CleanupMultiprocDirService.new.execute
+
+    settings = Settings.monitoring.sidekiq_exporter
+    exporter_class = "Gitlab::Metrics::Exporter::#{@target.camelize}Exporter".constantize
+    server = exporter_class.instance(settings, synchronous: true)
+
+    server.start
+  end
+end

metrics_server/settings_overrides.rb

+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+# Sidekiq-cluster code is loaded both inside a Rails/Rspec
+# context as well as outside of it via CLI invocation. When it
+# is loaded outside of a Rails/Rspec context we do not have access
+# to all necessary constants. For example, we need Rails.root to
+# determine the location of bin/metrics-server.
+# Here we make the necessary constants available conditionally.
+require_relative '../scripts/override_rails_constants' unless Object.const_defined?('Rails')
+
+require_relative '../config/settings'
+
+# rubocop:enable Naming/FileName

scripts/override_rails_constants.rb

+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+require 'active_support/environment_inquirer'
+
+module Rails # rubocop:disable Gitlab/NamespacedClass
+  extend self
+
+  def env
+    @env ||= ActiveSupport::EnvironmentInquirer.new(
+      ENV["RAILS_ENV"].presence || ENV["RACK_ENV"].presence || "test"
+    )
+  end
+
+  def root
+    Pathname.new(File.expand_path('..', __dir__))
+  end
+end
+
+# rubocop:enable Naming/FileName

scripts/setup-test-env

@@ -13,17 +13,7 @@ require 'active_support/string_inquirer'
 
 ENV['SKIP_RAILS_ENV_IN_RAKE'] = 'true'
 
-module Rails
-  extend self
-
-  def root
-    Pathname.new(File.expand_path('..', __dir__))
-  end
-
-  def env
-    @_env ||= ActiveSupport::StringInquirer.new(ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "test")
-  end
-end
+require_relative 'override_rails_constants'
 
 ActiveSupport::Dependencies.autoload_paths << 'lib'
 

spec/commands/metrics_server/metrics_server_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require_relative '../../../metrics_server/metrics_server'
+
+# End-to-end tests for the metrics server process we use to serve metrics
+# from forking applications (Sidekiq, Puma) to the Prometheus scraper.
+RSpec.describe 'bin/metrics-server', :aggregate_failures do
+  let(:config_file) { Tempfile.new('gitlab.yml') }
+  let(:config) do
+    {
+      'test' => {
+        'monitoring' => {
+          'sidekiq_exporter' => {
+            'address' => 'localhost',
+            'enabled' => true,
+            'port' => 3807
+          }
+        }
+      }
+    }
+  end
+
+  context 'with a running server' do
+    before do
+      # We need to send a request to localhost
+      WebMock.allow_net_connect!
+
+      config_file.write(YAML.dump(config))
+      config_file.close
+      @pid = MetricsServer.spawn('sidekiq', gitlab_config: config_file.path)
+    end
+
+    after do
+      webmock_enable!
+
+      if @pid
+        Timeout.timeout(5) do
+          Process.kill('TERM', @pid)
+          Process.waitpid(@pid)
+        end
+      end
+    rescue Errno::ESRCH => _
+      # 'No such process' means the process died before
+    ensure
+      config_file.unlink
+    end
+
+    it 'serves /metrics endpoint' do
+      expect do
+        Timeout.timeout(5) do
+          http_ok = false
+          until http_ok
+            sleep 1
+            response = Gitlab::HTTP.try_get("http://localhost:3807/metrics", allow_local_requests: true)
+            http_ok = response&.success?
+          end
+        end
+      end.not_to raise_error
+    end
+  end
+end

spec/metrics_server/metrics_server_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+require_relative '../../metrics_server/metrics_server'
+require_relative '../support/helpers/next_instance_of'
+
+RSpec.describe MetricsServer do # rubocop:disable RSpec/FilePath
+  include NextInstanceOf
+
+  describe '.spawn' do
+    let(:env) do
+      {
+        'METRICS_SERVER_TARGET' => 'sidekiq',
+        'GITLAB_CONFIG' => nil
+      }
+    end
+
+    it 'spawns a process with the correct environment variables and detaches it' do
+      expect(Process).to receive(:spawn).with(env, anything, err: $stderr, out: $stdout).and_return(99)
+      expect(Process).to receive(:detach).with(99)
+
+      described_class.spawn('sidekiq')
+    end
+  end
+
+  describe '#start' do
+    let(:exporter_class) { Class.new(Gitlab::Metrics::Exporter::BaseExporter) }
+    let(:exporter_double) { double('fake_exporter', start: true) }
+    let(:prometheus_client_double) { double(::Prometheus::Client) }
+    let(:prometheus_config) { ::Prometheus::Client::Configuration.new }
+    let(:metrics_dir) { Dir.mktmpdir }
+    let(:settings_double) { double(:settings, sidekiq_exporter: {}) }
+
+    subject(:metrics_server) { described_class.new('fake', metrics_dir)}
+
+    before do
+      stub_env('prometheus_multiproc_dir', metrics_dir)
+      stub_const('Gitlab::Metrics::Exporter::FakeExporter', exporter_class)
+      allow(exporter_class).to receive(:instance).with({}, synchronous: true).and_return(exporter_double)
+      allow(Settings).to receive(:monitoring).and_return(settings_double)
+    end
+
+    after do
+      Dir.rmdir(metrics_dir)
+    end
+
+    it 'configures ::Prometheus::Client' do
+      allow(prometheus_client_double).to receive(:configuration).and_return(prometheus_config)
+
+      metrics_server.start
+
+      expect(prometheus_config.multiprocess_files_dir).to eq metrics_dir
+    end
+
+    it 'ensures that metrics directory exists in correct mode (0700)' do
+      expect(FileUtils).to receive(:mkdir_p).with(metrics_dir, mode: 0700)
+
+      metrics_server.start
+    end
+
+    it 'removes any old metrics files' do
+      FileUtils.touch("#{metrics_dir}/remove_this.db")
+
+      expect { metrics_server.start }.to change { Dir.empty?(metrics_dir) }.from(false).to(true)
+    end
+
+    it 'starts a metrics server' do
+      expect(exporter_double).to receive(:start)
+
+      metrics_server.start
+    end
+
+    it 'sends the correct Settings to the exporter instance' do
+      expect(Settings).to receive(:monitoring).and_return(settings_double)
+      expect(settings_double).to receive(:sidekiq_exporter)
+
+      metrics_server.start
+    end
+  end
+end

spec/tooling/quality/test_level_spec.rb

@@ -28,7 +28,7 @@ RSpec.describe Quality::TestLevel do
     context 'when level is unit' do
       it 'returns a pattern' do
         expect(subject.pattern(:unit))
-          .to eq("spec/{bin,channels,config,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,javascripts,lib,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb")
+          .to eq("spec/{bin,channels,config,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,javascripts,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb")
       end
     end
 
@@ -110,7 +110,7 @@ RSpec.describe Quality::TestLevel do
     context 'when level is unit' do
       it 'returns a regexp' do
         expect(subject.regexp(:unit))
-          .to eq(%r{spec/(bin|channels|config|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|javascripts|lib|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)})
+          .to eq(%r{spec/(bin|channels|config|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|javascripts|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)})
       end
     end
 

tooling/quality/test_level.rb

@@ -33,6 +33,7 @@ module Quality
         initializers
         javascripts
         lib
+        metrics_server
         models
         policies
         presenters