Commit f2fd09cd authored by Stan Hu's avatar Stan Hu

Merge branch 'bvl-rack-attack-dry-run' into 'master'

Add a dry-run mode for rack attack

See merge request gitlab-org/gitlab!47545
parents 932b4392 83e89f4f
# frozen_string_literal: true
# Specs for this file can be found on:
# * spec/lib/gitlab/throttle_spec.rb
# * spec/requests/rack_attack_global_spec.rb
module Gitlab::Throttle
def self.settings
Gitlab::CurrentSettings.current_application_settings
end
# Returns true if we should use the Admin Area protected paths throttle
def self.protected_paths_enabled?
self.settings.throttle_protected_paths_enabled?
end
def self.omnibus_protected_paths_present?
Rack::Attack.throttles.key?('protected paths')
end
def self.bypass_header
env_value = ENV['GITLAB_THROTTLE_BYPASS_HEADER']
return unless env_value.present?
"HTTP_#{env_value.upcase.tr('-', '_')}"
end
def self.unauthenticated_options
limit_proc = proc { |req| settings.throttle_unauthenticated_requests_per_period }
period_proc = proc { |req| settings.throttle_unauthenticated_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
def self.authenticated_api_options
limit_proc = proc { |req| settings.throttle_authenticated_api_requests_per_period }
period_proc = proc { |req| settings.throttle_authenticated_api_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
def self.authenticated_web_options
limit_proc = proc { |req| settings.throttle_authenticated_web_requests_per_period }
period_proc = proc { |req| settings.throttle_authenticated_web_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
def self.protected_paths_options
limit_proc = proc { |req| settings.throttle_protected_paths_requests_per_period }
period_proc = proc { |req| settings.throttle_protected_paths_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
end
class Rack::Attack
# Order conditions by how expensive they are:
# 1. The most expensive is the `req.unauthenticated?` and
# `req.authenticated_user_id` as it performs an expensive
# DB/Redis query to validate the request
# 2. Slightly less expensive is the need to query DB/Redis
# to unmarshal settings (`Gitlab::Throttle.settings`)
#
# We deliberately skip `/-/health|liveness|readiness`
# from Rack Attack as they need to always be accessible
# by Load Balancer and additional measure is implemented
# (token and whitelisting) to prevent abuse.
throttle('throttle_unauthenticated', Gitlab::Throttle.unauthenticated_options) do |req|
if !req.should_be_skipped? &&
Gitlab::Throttle.settings.throttle_unauthenticated_enabled &&
req.unauthenticated?
req.ip
end
end
throttle('throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req|
if req.api_request? &&
Gitlab::Throttle.settings.throttle_authenticated_api_enabled
req.authenticated_user_id([:api])
end
end
# Product analytics feature is in experimental stage.
# At this point we want to limit amount of events registered
# per application (aid stands for application id).
throttle('throttle_product_analytics_collector', limit: 100, period: 60) do |req|
if req.product_analytics_collector_request?
req.params['aid']
end
end
throttle('throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req|
if req.web_request? &&
Gitlab::Throttle.settings.throttle_authenticated_web_enabled
req.authenticated_user_id([:api, :rss, :ics])
end
end
throttle('throttle_unauthenticated_protected_paths', Gitlab::Throttle.protected_paths_options) do |req|
if req.post? &&
!req.should_be_skipped? &&
req.protected_path? &&
Gitlab::Throttle.protected_paths_enabled? &&
req.unauthenticated?
req.ip
end
end
throttle('throttle_authenticated_protected_paths_api', Gitlab::Throttle.protected_paths_options) do |req|
if req.post? &&
req.api_request? &&
req.protected_path? &&
Gitlab::Throttle.protected_paths_enabled?
req.authenticated_user_id([:api])
end
end
throttle('throttle_authenticated_protected_paths_web', Gitlab::Throttle.protected_paths_options) do |req|
if req.post? &&
req.web_request? &&
req.protected_path? &&
Gitlab::Throttle.protected_paths_enabled?
req.authenticated_user_id([:api, :rss, :ics])
end
end
safelist('throttle_bypass_header') do |req|
Gitlab::Throttle.bypass_header.present? &&
req.get_header(Gitlab::Throttle.bypass_header) == '1'
end
class Request
def unauthenticated?
!(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id)
end
def authenticated_user_id(request_formats)
request_authenticator.user(request_formats)&.id
end
def authenticated_runner_id
request_authenticator.runner&.id
end
def api_request?
path.start_with?('/api')
end
def api_internal_request?
path =~ %r{^/api/v\d+/internal/}
end
def health_check_request?
path =~ %r{^/-/(health|liveness|readiness)}
end
def product_analytics_collector_request?
path.start_with?('/-/collector/i')
end
def should_be_skipped?
api_internal_request? || health_check_request?
end
def web_request?
!api_request? && !health_check_request?
end
def protected_path?
!protected_path_regex.nil?
end
def protected_path_regex
path =~ protected_paths_regex
end
private
def request_authenticator
@request_authenticator ||= Gitlab::Auth::RequestAuthenticator.new(self)
end
def protected_paths
Gitlab::CurrentSettings.current_application_settings.protected_paths
end
def protected_paths_regex
Regexp.union(protected_paths.map { |path| /\A#{Regexp.escape(path)}/ })
end
end
end
::Rack::Attack.extend_if_ee('::EE::Gitlab::Rack::Attack')
::Rack::Attack::Request.prepend_if_ee('::EE::Gitlab::Rack::Attack::Request')
Gitlab::RackAttack.configure(::Rack::Attack)
......@@ -6,7 +6,7 @@ ActiveSupport::Notifications.subscribe(/rack_attack/) do |name, start, finish, r
req = payload[:request]
case req.env['rack.attack.match_type']
when :throttle, :blocklist
when :throttle, :blocklist, :track
rack_attack_info = {
message: 'Rack_Attack',
env: req.env['rack.attack.match_type'],
......
# frozen_string_literal: true
module EE::Gitlab::Rack::Attack
Rack::Attack.throttle('throttle_incident_management_notification_web', EE::Gitlab::Throttle.incident_management_options) do |req|
if req.web_request? &&
req.path.include?('alerts/notify') &&
EE::Gitlab::Throttle.settings.throttle_incident_management_notification_enabled
req.path
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module Rack
module Attack
module Request
extend ::Gitlab::Utils::Override
override :should_be_skipped?
def should_be_skipped?
super || geo?
end
def geo?
::Gitlab::Geo::JwtRequestDecoder.geo_auth_attempt?(env['HTTP_AUTHORIZATION']) if env['HTTP_AUTHORIZATION']
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module RackAttack
extend ActiveSupport::Concern
class_methods do
extend ::Gitlab::Utils::Override
override :configure_throttles
def configure_throttles(rack_attack)
super
throttle_or_track(rack_attack, 'throttle_incident_management_notification_web', EE::Gitlab::Throttle.incident_management_options) do |req|
if req.web_request? &&
req.path.include?('alerts/notify') &&
EE::Gitlab::Throttle.settings.throttle_incident_management_notification_enabled
req.path
end
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module RackAttack
module Request
extend ::Gitlab::Utils::Override
override :should_be_skipped?
def should_be_skipped?
super || geo?
end
def geo?
::Gitlab::Geo::JwtRequestDecoder.geo_auth_attempt?(env['HTTP_AUTHORIZATION']) if env['HTTP_AUTHORIZATION']
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::RackAttack, :aggregate_failures do
describe '.configure' do
let(:fake_rack_attack) { class_double("Rack::Attack") }
let(:fake_rack_attack_request) { class_double("Rack::Attack::Request") }
before do
stub_const("Rack::Attack", fake_rack_attack)
stub_const("Rack::Attack::Request", fake_rack_attack_request)
allow(fake_rack_attack).to receive(:throttle)
allow(fake_rack_attack).to receive(:track)
allow(fake_rack_attack).to receive(:safelist)
allow(fake_rack_attack).to receive(:blocklist)
end
it 'adds the incident management throttle' do
described_class.configure(fake_rack_attack)
expect(fake_rack_attack).to have_received(:throttle)
.with('throttle_incident_management_notification_web', Gitlab::Throttle.authenticated_web_options)
end
end
end
# frozen_string_literal: true
# Integration specs for throttling can be found in:
# spec/requests/rack_attack_global_spec.rb
module Gitlab
module RackAttack
def self.configure(rack_attack)
# This adds some methods used by our throttles to the `Rack::Request`
rack_attack::Request.include(Gitlab::RackAttack::Request)
# Configure the throttles
configure_throttles(rack_attack)
end
def self.configure_throttles(rack_attack)
throttle_or_track(rack_attack, 'throttle_unauthenticated', Gitlab::Throttle.unauthenticated_options) do |req|
if !req.should_be_skipped? &&
Gitlab::Throttle.settings.throttle_unauthenticated_enabled &&
req.unauthenticated?
req.ip
end
end
throttle_or_track(rack_attack, 'throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req|
if req.api_request? &&
Gitlab::Throttle.settings.throttle_authenticated_api_enabled
req.authenticated_user_id([:api])
end
end
# Product analytics feature is in experimental stage.
# At this point we want to limit amount of events registered
# per application (aid stands for application id).
throttle_or_track(rack_attack, 'throttle_product_analytics_collector', limit: 100, period: 60) do |req|
if req.product_analytics_collector_request?
req.params['aid']
end
end
throttle_or_track(rack_attack, 'throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req|
if req.web_request? &&
Gitlab::Throttle.settings.throttle_authenticated_web_enabled
req.authenticated_user_id([:api, :rss, :ics])
end
end
throttle_or_track(rack_attack, 'throttle_unauthenticated_protected_paths', Gitlab::Throttle.protected_paths_options) do |req|
if req.post? &&
!req.should_be_skipped? &&
req.protected_path? &&
Gitlab::Throttle.protected_paths_enabled? &&
req.unauthenticated?
req.ip
end
end
throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_api', Gitlab::Throttle.protected_paths_options) do |req|
if req.post? &&
req.api_request? &&
req.protected_path? &&
Gitlab::Throttle.protected_paths_enabled?
req.authenticated_user_id([:api])
end
end
throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_web', Gitlab::Throttle.protected_paths_options) do |req|
if req.post? &&
req.web_request? &&
req.protected_path? &&
Gitlab::Throttle.protected_paths_enabled?
req.authenticated_user_id([:api, :rss, :ics])
end
end
rack_attack.safelist('throttle_bypass_header') do |req|
Gitlab::Throttle.bypass_header.present? &&
req.get_header(Gitlab::Throttle.bypass_header) == '1'
end
end
def self.throttle_or_track(rack_attack, throttle_name, *args, &block)
if track?(throttle_name)
rack_attack.track(throttle_name, *args, &block)
else
rack_attack.throttle(throttle_name, *args, &block)
end
end
def self.track?(name)
dry_run_config = ENV['GITLAB_THROTTLE_DRY_RUN'].to_s.strip
return false if dry_run_config.empty?
return true if dry_run_config == '*'
dry_run_config.split(',').map(&:strip).include?(name)
end
end
end
::Gitlab::RackAttack.prepend_if_ee('::EE::Gitlab::RackAttack')
# frozen_string_literal: true
module Gitlab
module RackAttack
module Request
def unauthenticated?
!(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id)
end
def authenticated_user_id(request_formats)
request_authenticator.user(request_formats)&.id
end
def authenticated_runner_id
request_authenticator.runner&.id
end
def api_request?
path.start_with?('/api')
end
def api_internal_request?
path =~ %r{^/api/v\d+/internal/}
end
def health_check_request?
path =~ %r{^/-/(health|liveness|readiness|metrics)}
end
def product_analytics_collector_request?
path.start_with?('/-/collector/i')
end
def should_be_skipped?
api_internal_request? || health_check_request?
end
def web_request?
!api_request? && !health_check_request?
end
def protected_path?
!protected_path_regex.nil?
end
def protected_path_regex
path =~ protected_paths_regex
end
private
def request_authenticator
@request_authenticator ||= Gitlab::Auth::RequestAuthenticator.new(self)
end
def protected_paths
Gitlab::CurrentSettings.current_application_settings.protected_paths
end
def protected_paths_regex
Regexp.union(protected_paths.map { |path| /\A#{Regexp.escape(path)}/ })
end
end
end
end
::Gitlab::RackAttack::Request.prepend_if_ee('::EE::Gitlab::RackAttack::Request')
# frozen_string_literal: true
module Gitlab
class Throttle
def self.settings
Gitlab::CurrentSettings.current_application_settings
end
# Returns true if we should use the Admin Area protected paths throttle
def self.protected_paths_enabled?
self.settings.throttle_protected_paths_enabled?
end
def self.omnibus_protected_paths_present?
Rack::Attack.throttles.key?('protected paths')
end
def self.bypass_header
env_value = ENV['GITLAB_THROTTLE_BYPASS_HEADER']
return unless env_value.present?
"HTTP_#{env_value.upcase.tr('-', '_')}"
end
def self.unauthenticated_options
limit_proc = proc { |req| settings.throttle_unauthenticated_requests_per_period }
period_proc = proc { |req| settings.throttle_unauthenticated_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
def self.authenticated_api_options
limit_proc = proc { |req| settings.throttle_authenticated_api_requests_per_period }
period_proc = proc { |req| settings.throttle_authenticated_api_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
def self.authenticated_web_options
limit_proc = proc { |req| settings.throttle_authenticated_web_requests_per_period }
period_proc = proc { |req| settings.throttle_authenticated_web_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
def self.protected_paths_options
limit_proc = proc { |req| settings.throttle_protected_paths_requests_per_period }
period_proc = proc { |req| settings.throttle_protected_paths_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::RackAttack, :aggregate_failures do
describe '.configure' do
let(:fake_rack_attack) { class_double("Rack::Attack") }
let(:fake_rack_attack_request) { class_double("Rack::Attack::Request") }
let(:throttles) do
{
throttle_unauthenticated: Gitlab::Throttle.unauthenticated_options,
throttle_authenticated_api: Gitlab::Throttle.authenticated_api_options,
throttle_product_analytics_collector: { limit: 100, period: 60 },
throttle_unauthenticated_protected_paths: Gitlab::Throttle.unauthenticated_options,
throttle_authenticated_protected_paths_api: Gitlab::Throttle.authenticated_api_options,
throttle_authenticated_protected_paths_web: Gitlab::Throttle.authenticated_web_options
}
end
before do
stub_const("Rack::Attack", fake_rack_attack)
stub_const("Rack::Attack::Request", fake_rack_attack_request)
allow(fake_rack_attack).to receive(:throttle)
allow(fake_rack_attack).to receive(:track)
allow(fake_rack_attack).to receive(:safelist)
allow(fake_rack_attack).to receive(:blocklist)
end
it 'extends the request class' do
described_class.configure(fake_rack_attack)
expect(fake_rack_attack_request).to include(described_class::Request)
end
it 'configures the safelist' do
described_class.configure(fake_rack_attack)
expect(fake_rack_attack).to have_received(:safelist).with('throttle_bypass_header')
end
it 'configures throttles if no dry-run was configured' do
described_class.configure(fake_rack_attack)
throttles.each do |throttle, options|
expect(fake_rack_attack).to have_received(:throttle).with(throttle.to_s, options)
end
end
it 'configures tracks if dry-run was configured for all throttles' do
stub_env('GITLAB_THROTTLE_DRY_RUN', '*')
described_class.configure(fake_rack_attack)
throttles.each do |throttle, options|
expect(fake_rack_attack).to have_received(:track).with(throttle.to_s, options)
end
expect(fake_rack_attack).not_to have_received(:throttle)
end
it 'configures tracks and throttles with a selected set of dry-runs' do
dry_run_throttles = throttles.each_key.first(2)
regular_throttles = throttles.keys[2..-1]
stub_env('GITLAB_THROTTLE_DRY_RUN', dry_run_throttles.join(','))
described_class.configure(fake_rack_attack)
dry_run_throttles.each do |throttle|
expect(fake_rack_attack).to have_received(:track).with(throttle.to_s, throttles[throttle])
end
regular_throttles.each do |throttle|
expect(fake_rack_attack).to have_received(:throttle).with(throttle.to_s, throttles[throttle])
end
end
end
end
......@@ -106,7 +106,7 @@ RSpec.describe 'Rack Attack global throttles' do
let(:request_jobs_url) { '/api/v4/jobs/request' }
let(:runner) { create(:ci_runner) }
it 'does not cont as unauthenticated' do
it 'does not count as unauthenticated' do
(1 + requests_per_period).times do
post request_jobs_url, params: { token: runner.token }
expect(response).to have_gitlab_http_status(:no_content)
......@@ -114,6 +114,17 @@ RSpec.describe 'Rack Attack global throttles' do
end
end
context 'when the request is to a health endpoint' do
let(:health_endpoint) { '/-/metrics' }
it 'does not throttle the requests' do
(1 + requests_per_period).times do
get health_endpoint
expect(response).to have_gitlab_http_status(:ok)
end
end
end
it 'logs RackAttack info into structured logs' do
requests_per_period.times do
get url_that_does_not_require_authentication
......@@ -133,6 +144,14 @@ RSpec.describe 'Rack Attack global throttles' do
get url_that_does_not_require_authentication
end
it_behaves_like 'tracking when dry-run mode is set' do
let(:throttle_name) { 'throttle_unauthenticated' }
def do_request
get url_that_does_not_require_authentication
end
end
end
context 'when the throttle is disabled' do
......@@ -231,6 +250,10 @@ RSpec.describe 'Rack Attack global throttles' do
let(:post_params) { { user: { login: 'username', password: 'password' } } }
def do_request
post protected_path_that_does_not_require_authentication, params: post_params
end
before do
settings_to_set[:throttle_protected_paths_requests_per_period] = requests_per_period # 1
settings_to_set[:throttle_protected_paths_period_in_seconds] = period_in_seconds # 10_000
......@@ -244,7 +267,7 @@ RSpec.describe 'Rack Attack global throttles' do
it 'allows requests over the rate limit' do
(1 + requests_per_period).times do
post protected_path_that_does_not_require_authentication, params: post_params
do_request
expect(response).to have_gitlab_http_status(:ok)
end
end
......@@ -258,12 +281,16 @@ RSpec.describe 'Rack Attack global throttles' do
it 'rejects requests over the rate limit' do
requests_per_period.times do
post protected_path_that_does_not_require_authentication, params: post_params
do_request
expect(response).to have_gitlab_http_status(:ok)
end
expect_rejection { post protected_path_that_does_not_require_authentication, params: post_params }
end
it_behaves_like 'tracking when dry-run mode is set' do
let(:throttle_name) { 'throttle_unauthenticated_protected_paths' }
end
end
end
......
......@@ -110,6 +110,14 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do
expect { make_request(request_args) }.not_to exceed_query_limit(control_count)
end
end
it_behaves_like 'tracking when dry-run mode is set' do
let(:throttle_name) { throttle_types[throttle_setting_prefix] }
def do_request
make_request(request_args)
end
end
end
context 'when the throttle is disabled' do
......@@ -245,6 +253,14 @@ RSpec.shared_examples 'rate-limited web authenticated requests' do
expect(Gitlab::AuthLogger).to receive(:error).with(arguments).once
expect { request_authenticated_web_url }.not_to exceed_query_limit(control_count)
end
it_behaves_like 'tracking when dry-run mode is set' do
let(:throttle_name) { throttle_types[throttle_setting_prefix] }
def do_request
request_authenticated_web_url
end
end
end
context 'when the throttle is disabled' do
......@@ -269,3 +285,63 @@ RSpec.shared_examples 'rate-limited web authenticated requests' do
end
end
end
# Requires:
# - #do_request - This needs to be a method so the result isn't memoized
# - throttle_name
RSpec.shared_examples 'tracking when dry-run mode is set' do
let(:dry_run_config) { '*' }
# we can't use `around` here, because stub_env isn't supported outside of the
# example itself
before do
stub_env('GITLAB_THROTTLE_DRY_RUN', dry_run_config)
reset_rack_attack
end
after do
stub_env('GITLAB_THROTTLE_DRY_RUN', '')
reset_rack_attack
end
def reset_rack_attack
Rack::Attack.reset!
Rack::Attack.clear_configuration
Gitlab::RackAttack.configure(Rack::Attack)
end
it 'does not throttle the requests when `*` is configured' do
(1 + requests_per_period).times do
do_request
expect(response).not_to have_gitlab_http_status(:too_many_requests)
end
end
it 'logs RackAttack info into structured logs' do
arguments = a_hash_including({
message: 'Rack_Attack',
env: :track,
remote_ip: '127.0.0.1',
matched: throttle_name
})
expect(Gitlab::AuthLogger).to receive(:error).with(arguments)
(1 + requests_per_period).times do
do_request
end
end
context 'when configured with the the throttled name in a list' do
let(:dry_run_config) do
"throttle_list, #{throttle_name}, other_throttle"
end
it 'does not throttle' do
(1 + requests_per_period).times do
do_request
expect(response).not_to have_gitlab_http_status(:too_many_requests)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment