Add metrics for Redis usage during web requests

Adds metrics to measure the amount of calls and
request time spent in Redis during web requests
(e.g. Rails, Grape).

These are measured through a new `RailsRedisMiddleware`.

changelogs/unreleased/osw-add-redis-metrics-to-web-requests.yml

+---
+title: Add metrics for Redis usage during web requests
+merge_request: 32605
+author:
+type: added

config/initializers/zz_metrics.rb

@@ -147,6 +147,7 @@ if Gitlab::Metrics.enabled? && !Rails.env.test? && !(Rails.env.development? && d
   Gitlab::Application.configure do |config|
     config.middleware.use(Gitlab::Metrics::RackMiddleware)
     config.middleware.use(Gitlab::Middleware::RailsQueueDuration)
+    config.middleware.use(Gitlab::Metrics::RedisRackMiddleware)
   end
 
   Sidekiq.configure_server do |config|

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -92,6 +92,8 @@ The following metrics are available:
 | `gitlab_view_rendering_duration_seconds`                       | Histogram |                   10.2 | Duration for views (histogram)                                                                      | `controller`, `action`, `view`                      |
 | `http_requests_total`                                          | Counter   |                    9.4 | Rack request count                                                                                  | `method`                                            |
 | `http_request_duration_seconds`                                | Histogram |                    9.4 | HTTP response time from rack middleware                                                             | `method`, `status`                                  |
+| `http_redis_requests_duration_seconds`                         | Histogram |                   13.1 | Redis requests duration during web transactions                                                     | `controller`, `action`                              |
+| `http_redis_requests_total`                                    | Counter   |                   13.1 | Redis requests count during web transactions                                                        | `controller`, `action`                              |
 | `pipelines_created_total`                                      | Counter   |                    9.4 | Counter of pipelines created                                                                        |                                                     |
 | `rack_uncaught_errors_total`                                   | Counter   |                    9.4 | Rack connections handling uncaught errors count                                                     |                                                     |
 | `user_session_logins_total`                                    | Counter   |                    9.4 | Counter of how many users have logged in                                                            |                                                     |

lib/gitlab/metrics/redis_rack_middleware.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    # Rack middleware for tracking Redis metrics from Grape and Web requests.
+    class RedisRackMiddleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        transaction = Gitlab::Metrics.current_transaction
+
+        @app.call(env)
+      ensure
+        record_metrics(transaction) if transaction
+      end
+
+      private
+
+      def record_metrics(transaction)
+        labels = transaction.labels
+        query_time = Gitlab::Instrumentation::Redis.query_time
+        request_count = Gitlab::Instrumentation::Redis.get_request_count
+
+        Gitlab::Metrics.counter(:http_redis_requests_total,
+                                'Amount of calls to Redis servers during web requests',
+                                Gitlab::Metrics::Transaction::BASE_LABELS).increment(labels, request_count)
+
+        Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds,
+                                  'Query time for Redis servers during web requests',
+                                  Gitlab::Metrics::Transaction::BASE_LABELS,
+                                  Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS).observe(labels, query_time)
+      end
+    end
+  end
+end

spec/lib/gitlab/metrics/redis_rack_middleware_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Metrics::RedisRackMiddleware do
+  let(:app) { double(:app) }
+  let(:middleware) { described_class.new(app) }
+  let(:env) { {} }
+  let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) }
+
+  before do
+    allow(app).to receive(:call).with(env).and_return('wub wub')
+  end
+
+  describe '#call' do
+    context 'when metrics are disabled' do
+      before do
+        allow(Gitlab::Metrics).to receive(:current_transaction).and_return(nil)
+      end
+
+      it 'calls the app' do
+        expect(middleware.call(env)).to eq('wub wub')
+      end
+
+      it 'does not record metrics' do
+        expect(Gitlab::Metrics).not_to receive(:counter)
+        expect(Gitlab::Metrics).not_to receive(:histogram)
+
+        middleware.call(env)
+      end
+    end
+
+    context 'when metrics are enabled' do
+      let(:counter) { double(Prometheus::Client::Counter, increment: nil) }
+      let(:histogram) { double(Prometheus::Client::Histogram, observe: nil) }
+      let(:redis_query_time) { 0.1 }
+      let(:redis_requests_count) { 2 }
+
+      before do
+        allow(Gitlab::Instrumentation::Redis).to receive(:query_time) { redis_query_time }
+        allow(Gitlab::Instrumentation::Redis).to receive(:get_request_count) { redis_requests_count }
+
+        allow(Gitlab::Metrics).to receive(:counter)
+          .with(:http_redis_requests_total,
+                an_instance_of(String),
+                Gitlab::Metrics::Transaction::BASE_LABELS)
+          .and_return(counter)
+
+        allow(Gitlab::Metrics).to receive(:histogram)
+          .with(:http_redis_requests_duration_seconds,
+                an_instance_of(String),
+                Gitlab::Metrics::Transaction::BASE_LABELS,
+                Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS)
+          .and_return(histogram)
+
+        allow(Gitlab::Metrics).to receive(:current_transaction).and_return(transaction)
+      end
+
+      it 'calls the app' do
+        expect(middleware.call(env)).to eq('wub wub')
+      end
+
+      it 'records redis metrics' do
+        expect(counter).to receive(:increment).with(transaction.labels, redis_requests_count)
+        expect(histogram).to receive(:observe).with(transaction.labels, redis_query_time)
+
+        middleware.call(env)
+      end
+
+      it 'records redis metrics if an error is raised' do
+        expect(counter).to receive(:increment).with(transaction.labels, redis_requests_count)
+        expect(histogram).to receive(:observe).with(transaction.labels, redis_query_time)
+
+        allow(app).to receive(:call).with(env).and_raise(StandardError)
+
+        expect { middleware.call(env) }.to raise_error(StandardError)
+      end
+    end
+  end
+end