Add user ID based allowlist for Rack::Attack

For https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/341 we are
finding that in order to tighten rate limiting, it would help us to
have an allowlist for a small number of existing users that currently
exceed the limits.

This implements
https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/697.

changelogs/unreleased/jv-rack-attack-user-bypass.yml

+---
+title: Add user ID based allowlist for Rack::Attack
+merge_request: 49127
+author:
+type: changed

doc/user/admin_area/settings/user_and_ip_rate_limits.md

@@ -59,6 +59,29 @@ are marked with `"throttle_safelist":"throttle_bypass_header"` in
 To disable the bypass mechanism, make sure the environment variable
 `GITLAB_THROTTLE_BYPASS_HEADER` is unset or empty.
 
+## Allowing specific users to bypass authenticated request rate limiting
+
+Similarly to the bypass header described above, it is possible to allow
+a certain set of users to bypass the rate limiter. This only applies
+to authenticated requests: with unauthenticated requests, by definition
+GitLab does not know who the user is.
+
+The allowlist is configured as a comma-separated list of user IDs in
+the `GITLAB_THROTTLE_USER_ALLOWLIST` environment variable. If you want
+users 1, 53 and 217 to bypass the authenticated request rate limiter,
+the allowlist configuration would be `1,53,217`.
+
+- For [Omnibus](https://docs.gitlab.com/omnibus/settings/environment-variables.html),
+  set `'GITLAB_THROTTLE_USER_ALLOWLIST' => '1,53,217'` in `gitlab_rails['env']`.
+- For source installations, set `export GITLAB_THROTTLE_USER_ALLOWLIST=1,53,217`
+  in `/etc/default/gitlab`.
+
+Requests that bypassed the rate limiter because of the user allowlist
+are marked with `"throttle_safelist":"throttle_user_allowlist"` in
+[`production_json.log`](../../../administration/logs.md#production_jsonlog).
+
+At application startup, the allowlist is logged in [`auth.log`](../../../administration/logs.md#authlog).
+
 <!-- ## Troubleshooting
 
 Include any troubleshooting steps that you can foresee. If you know beforehand what issues

lib/gitlab/rack_attack.rb

@@ -11,6 +11,13 @@ module Gitlab
       Rack::Attack.throttled_response_retry_after_header = true
       # Configure the throttles
       configure_throttles(rack_attack)
+
+      configure_user_allowlist
+    end
+
+    def self.configure_user_allowlist
+      @user_allowlist = nil
+      user_allowlist
     end
 
     def self.configure_throttles(rack_attack)
@@ -25,7 +32,7 @@ module Gitlab
       throttle_or_track(rack_attack, 'throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req|
         if req.api_request? &&
            Gitlab::Throttle.settings.throttle_authenticated_api_enabled
-          req.authenticated_user_id([:api])
+          req.throttled_user_id([:api])
         end
       end
 
@@ -41,7 +48,7 @@ module Gitlab
       throttle_or_track(rack_attack, 'throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req|
         if req.web_request? &&
            Gitlab::Throttle.settings.throttle_authenticated_web_enabled
-          req.authenticated_user_id([:api, :rss, :ics])
+          req.throttled_user_id([:api, :rss, :ics])
         end
       end
 
@@ -60,7 +67,7 @@ module Gitlab
            req.api_request? &&
            req.protected_path? &&
            Gitlab::Throttle.protected_paths_enabled?
-          req.authenticated_user_id([:api])
+          req.throttled_user_id([:api])
         end
       end
 
@@ -69,7 +76,7 @@ module Gitlab
            req.web_request? &&
            req.protected_path? &&
            Gitlab::Throttle.protected_paths_enabled?
-          req.authenticated_user_id([:api, :rss, :ics])
+          req.throttled_user_id([:api, :rss, :ics])
         end
       end
 
@@ -95,6 +102,14 @@ module Gitlab
 
       dry_run_config.split(',').map(&:strip).include?(name)
     end
+
+    def self.user_allowlist
+      @user_allowlist ||= begin
+        list = UserAllowlist.new(ENV['GITLAB_THROTTLE_USER_ALLOWLIST'])
+        Gitlab::AuthLogger.info(gitlab_throttle_user_allowlist: list.to_a)
+        list
+      end
+    end
   end
 end
 ::Gitlab::RackAttack.prepend_if_ee('::EE::Gitlab::RackAttack')

lib/gitlab/rack_attack/request.rb

@@ -7,8 +7,15 @@ module Gitlab
         !(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id)
       end
 
-      def authenticated_user_id(request_formats)
-        request_authenticator.user(request_formats)&.id
+      def throttled_user_id(request_formats)
+        user_id = authenticated_user_id(request_formats)
+
+        if Gitlab::RackAttack.user_allowlist.include?(user_id)
+          Gitlab::Instrumentation::Throttle.safelist = 'throttle_user_allowlist'
+          return
+        end
+
+        user_id
       end
 
       def authenticated_runner_id
@@ -49,6 +56,10 @@ module Gitlab
 
       private
 
+      def authenticated_user_id(request_formats)
+        request_authenticator.user(request_formats)&.id
+      end
+
       def request_authenticator
         @request_authenticator ||= Gitlab::Auth::RequestAuthenticator.new(self)
       end

lib/gitlab/rack_attack/user_allowlist.rb

+# frozen_string_literal: true
+
+require 'set'
+
+module Gitlab
+  module RackAttack
+    class UserAllowlist
+      extend Forwardable
+
+      def_delegators :@set, :empty?, :include?, :to_a
+
+      def initialize(list)
+        @set = Set.new
+
+        list.to_s.split(',').each do |id|
+          @set << Integer(id) unless id.blank?
+        rescue ArgumentError
+          Gitlab::AuthLogger.error(message: 'ignoring invalid user allowlist entry', entry: id)
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/rack_attack/user_allowlist_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::RackAttack::UserAllowlist do
+  using RSpec::Parameterized::TableSyntax
+
+  subject { described_class.new(input)}
+
+  where(:input, :elements) do
+    nil | []
+    '' |  []
+    '123' | [123]
+    '123,456' | [123, 456]
+    '123,foobar, 456,' | [123, 456]
+  end
+
+  with_them do
+    it 'has the expected elements' do
+      expect(subject).to contain_exactly(*elements)
+    end
+
+    it 'implements empty?' do
+      expect(subject.empty?).to eq(elements.empty?)
+    end
+
+    it 'implements include?' do
+      unless elements.empty?
+        expect(subject).to include(elements.first)
+      end
+    end
+  end
+end

spec/lib/gitlab/rack_attack_spec.rb

@@ -75,5 +75,22 @@ RSpec.describe Gitlab::RackAttack, :aggregate_failures do
         expect(fake_rack_attack).to have_received(:throttle).with(throttle.to_s, throttles[throttle])
       end
     end
+
+    context 'user allowlist' do
+      subject { described_class.user_allowlist }
+
+      it 'is empty' do
+        described_class.configure(fake_rack_attack)
+
+        expect(subject).to be_empty
+      end
+
+      it 'reflects GITLAB_THROTTLE_USER_ALLOWLIST' do
+        stub_env('GITLAB_THROTTLE_USER_ALLOWLIST', '123,456')
+        described_class.configure(fake_rack_attack)
+
+        expect(subject).to contain_exactly(123, 456)
+      end
+    end
   end
 end

spec/support/shared_examples/requests/rack_attack_shared_examples.rb

@@ -23,6 +23,11 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do
     settings_to_set[:"#{throttle_setting_prefix}_period_in_seconds"] = period_in_seconds
   end
 
+  after do
+    stub_env('GITLAB_THROTTLE_USER_ALLOWLIST', nil)
+    Gitlab::RackAttack.configure_user_allowlist
+  end
+
   context 'when the throttle is enabled' do
     before do
       settings_to_set[:"#{throttle_setting_prefix}_enabled"] = true
@@ -30,6 +35,8 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do
     end
 
     it 'rejects requests over the rate limit' do
+      expect(Gitlab::Instrumentation::Throttle).not_to receive(:safelist=)
+
       # At first, allow requests under the rate limit.
       requests_per_period.times do
         make_request(request_args)
@@ -40,6 +47,18 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do
       expect_rejection { make_request(request_args) }
     end
 
+    it 'does not reject requests if the user is in the allowlist' do
+      stub_env('GITLAB_THROTTLE_USER_ALLOWLIST', user.id.to_s)
+      Gitlab::RackAttack.configure_user_allowlist
+
+      expect(Gitlab::Instrumentation::Throttle).to receive(:safelist=).with('throttle_user_allowlist').at_least(:once)
+
+      (requests_per_period + 1).times do
+        make_request(request_args)
+        expect(response).not_to have_gitlab_http_status(:too_many_requests)
+      end
+    end
+
     it 'allows requests after throttling and then waiting for the next period' do
       requests_per_period.times do
         make_request(request_args)
@@ -167,6 +186,11 @@ RSpec.shared_examples 'rate-limited web authenticated requests' do
     settings_to_set[:"#{throttle_setting_prefix}_period_in_seconds"] = period_in_seconds
   end
 
+  after do
+    stub_env('GITLAB_THROTTLE_USER_ALLOWLIST', nil)
+    Gitlab::RackAttack.configure_user_allowlist
+  end
+
   context 'when the throttle is enabled' do
     before do
       settings_to_set[:"#{throttle_setting_prefix}_enabled"] = true
@@ -174,6 +198,8 @@ RSpec.shared_examples 'rate-limited web authenticated requests' do
     end
 
     it 'rejects requests over the rate limit' do
+      expect(Gitlab::Instrumentation::Throttle).not_to receive(:safelist=)
+
       # At first, allow requests under the rate limit.
       requests_per_period.times do
         request_authenticated_web_url
@@ -184,6 +210,18 @@ RSpec.shared_examples 'rate-limited web authenticated requests' do
       expect_rejection { request_authenticated_web_url }
     end
 
+    it 'does not reject requests if the user is in the allowlist' do
+      stub_env('GITLAB_THROTTLE_USER_ALLOWLIST', user.id.to_s)
+      Gitlab::RackAttack.configure_user_allowlist
+
+      expect(Gitlab::Instrumentation::Throttle).to receive(:safelist=).with('throttle_user_allowlist').at_least(:once)
+
+      (requests_per_period + 1).times do
+        request_authenticated_web_url
+        expect(response).not_to have_gitlab_http_status(:too_many_requests)
+      end
+    end
+
     it 'allows requests after throttling and then waiting for the next period' do
       requests_per_period.times do
         request_authenticated_web_url