Add Gitlab::Redis::RateLimiting Redis instance

This instance is not used yet, but will soon be used for rate limiting
via Rack::Attack and Gitlab::ApplicationRateLimiter. Having this
configuration option in place will allow us to build on that, and also
create merge requests to support this option in Omnibus and our Helm
charts.

.gitignore

@@ -44,6 +44,7 @@ eslint-report.html
 /config/redis.queues.yml
 /config/redis.shared_state.yml
 /config/redis.trace_chunks.yml
+/config/redis.rate_limiting.yml
 /config/unicorn.rb
 /config/puma.rb
 /config/secrets.yml

app/controllers/admin/dashboard_controller.rb

@@ -15,7 +15,13 @@ class Admin::DashboardController < Admin::ApplicationController
     @groups = Group.order_id_desc.with_route.limit(10)
     @notices = Gitlab::ConfigChecker::PumaRuggedChecker.check
     @notices += Gitlab::ConfigChecker::ExternalDatabaseChecker.check
-    @redis_versions = [Gitlab::Redis::Queues, Gitlab::Redis::SharedState, Gitlab::Redis::Cache, Gitlab::Redis::TraceChunks].map(&:version).uniq
+    @redis_versions = [
+      Gitlab::Redis::Queues,
+      Gitlab::Redis::SharedState,
+      Gitlab::Redis::Cache,
+      Gitlab::Redis::TraceChunks,
+      Gitlab::Redis::RateLimiting
+    ].map(&:version).uniq
   end
   # rubocop: enable CodeReuse/ActiveRecord
 

app/controllers/health_controller.rb

@@ -16,6 +16,7 @@ class HealthController < ActionController::Base
     Gitlab::HealthChecks::Redis::QueuesCheck,
     Gitlab::HealthChecks::Redis::SharedStateCheck,
     Gitlab::HealthChecks::Redis::TraceChunksCheck,
+    Gitlab::HealthChecks::Redis::RateLimitingCheck,
     Gitlab::HealthChecks::GitalyCheck
   ].freeze
 

config/README.md

@@ -176,5 +176,46 @@ searched):
 `GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` environment variable
 2. the configuration file `redis.trace_chunks.yml`
 3. the configuration file pointed to by the
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
+4. the configuration file `redis.shared_state.yml`
+5. the configuration file pointed to by the
 `GITLAB_REDIS_CONFIG_FILE` environment variable
-4. the configuration file `resque.yml`
+6. the configuration file `resque.yml`
+
+## redis.rate_limiting.yml
+
+If configured, `redis.rate_limiting.yml` overrides the `resque.yml` settings to
+configure the Redis database instance used for clients of
+`::Gitlab::Redis::RateLimiting` which stores
+[rate limiting](https://docs.gitlab.com/ee/user/admin_area/settings/user_and_ip_rate_limits.html)
+state.
+
+Settings here can be overridden by the environment variable
+`GITLAB_REDIS_RATE_LIMITING_CONFIG_FILE` which provides
+an alternate location for configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `rate_limiting` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_RATE_LIMITING_CONFIG_FILE` environment variable
+2. URL from `redis.rate_limiting.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
+4. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+5. URL from `resque.yml`
+6. `redis://localhost:6384`
+
+The order of precedence for all other configuration settings for `rate_limiting`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_RATE_LIMITING_CONFIG_FILE` environment variable
+2. the configuration file `redis.rate_limiting.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
+4. the configuration file `redis.cache.yml`
+5. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+6. the configuration file `resque.yml`

config/application.rb

@@ -23,6 +23,8 @@ module Gitlab
     require_dependency Rails.root.join('lib/gitlab/redis/cache')
     require_dependency Rails.root.join('lib/gitlab/redis/queues')
     require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
+    require_dependency Rails.root.join('lib/gitlab/redis/trace_chunks')
+    require_dependency Rails.root.join('lib/gitlab/redis/rate_limiting')
     require_dependency Rails.root.join('lib/gitlab/current_settings')
     require_dependency Rails.root.join('lib/gitlab/middleware/read_only')
     require_dependency Rails.root.join('lib/gitlab/middleware/basic_health_check')

config/initializers/7_redis.rb

@@ -11,3 +11,4 @@ Gitlab::Redis::Cache.with { nil }
 Gitlab::Redis::Queues.with { nil }
 Gitlab::Redis::SharedState.with { nil }
 Gitlab::Redis::TraceChunks.with { nil }
+Gitlab::Redis::RateLimiting.with { nil }

config/redis.rate_limiting.yml.example

+# If you change this file in a merge request, please also create
+# a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+  url: redis://localhost:6379/4
+  #
+  # url: redis://localhost:6382
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26382 # point to sentinel, not to redis port
+  #   -
+  #     host: replica2
+  #     port: 26382 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379/14
+  #
+  # url: redis://localhost:6382
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.rate_limiting.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ee/administration/redis/index.html
+  ##
+  # url: redis://master:6382
+  # sentinels:
+  #   -
+  #     host: replica1
+  #     port: 26382 # point to sentinel, not to redis port
+  #   -
+  #     host: replica2
+  #     port: 26382 # point to sentinel, not to redis port

doc/administration/redis/replication_and_failover.md

@@ -647,6 +647,7 @@ persistence classes.
 | `shared_state` | Store session-related and other persistent data. |
 | `actioncable`  | Pub/Sub queue backend for ActionCable.           |
 | `trace_chunks` | Store [CI trace chunks](../job_logs.md#enable-or-disable-incremental-logging) data. |
+| `rate_limiting` | Store [rate limiting](../../user/admin_area/settings/user_and_ip_rate_limits.md) state. |
 
 To make this work with Sentinel:
 
@@ -659,6 +660,7 @@ To make this work with Sentinel:
    gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL
    gitlab_rails['redis_actioncable_instance'] = REDIS_ACTIONCABLE_URL
    gitlab_rails['redis_trace_chunks_instance'] = REDIS_TRACE_CHUNKS_URL
+   gitlab_rails['redis_rate_limiting_instance'] = REDIS_RATE_LIMITING_URL
 
    # Configure the Sentinels
    gitlab_rails['redis_cache_sentinels'] = [
@@ -681,6 +683,10 @@ To make this work with Sentinel:
      { host: TRACE_CHUNKS_SENTINEL_HOST, port: 26379 },
      { host: TRACE_CHUNKS_SENTINEL_HOST2, port: 26379 }
    ]
+   gitlab_rails['redis_rate_limiting_sentinels'] = [
+     { host: RATE_LIMITING_SENTINEL_HOST, port: 26379 },
+     { host: RATE_LIMITING_SENTINEL_HOST2, port: 26379 }
+   ]
    ```
 
    - Redis URLs should be in the format: `redis://:PASSWORD@SENTINEL_PRIMARY_NAME`, where:

doc/development/redis.md

@@ -15,6 +15,8 @@ GitLab uses [Redis](https://redis.io) for the following distinct purposes:
 - To manage the shared application state.
 - To store CI trace chunks.
 - As a Pub/Sub queue backend for ActionCable.
+- CI trace chunk storage.
+- Rate limiting state storage.
 
 In most environments (including the GDK), all of these point to the same
 Redis instance.

lib/gitlab/health_checks/redis/rate_limiting_check.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module HealthChecks
+    module Redis
+      class RateLimitingCheck
+        extend SimpleAbstractCheck
+
+        class << self
+          def check_up
+            check
+          end
+
+          private
+
+          def metric_prefix
+            'redis_rate_limiting_ping'
+          end
+
+          def successful?(result)
+            result == 'PONG'
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def check
+            catch_timeout 10.seconds do
+              Gitlab::Redis::RateLimiting.with(&:ping)
+            end
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end

lib/gitlab/health_checks/redis/redis_check.rb

@@ -21,7 +21,8 @@ module Gitlab
             ::Gitlab::HealthChecks::Redis::CacheCheck.check_up &&
               ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up &&
               ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up &&
-              ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up
+              ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up &&
+              ::Gitlab::HealthChecks::Redis::RateLimitingCheck.check_up
           end
         end
       end

lib/gitlab/instrumentation/redis.rb

@@ -9,8 +9,9 @@ module Gitlab
       Queues = Class.new(RedisBase)
       SharedState = Class.new(RedisBase).enable_redis_cluster_validation
       TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation
+      RateLimiting = Class.new(RedisBase).enable_redis_cluster_validation
 
-      STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze
+      STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks, RateLimiting].freeze
 
       # Milliseconds represented in seconds (from 1 millisecond to 2 seconds).
       QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze

lib/gitlab/redis/rate_limiting.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Redis
+    class RateLimiting < ::Gitlab::Redis::Wrapper
+      # The data we store on RateLimiting used to be stored on Cache.
+      def self.config_fallback
+        Cache
+      end
+    end
+  end
+end

scripts/prepare_build.sh

@@ -50,6 +50,9 @@ sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
 cp config/redis.trace_chunks.yml.example config/redis.trace_chunks.yml
 sed -i 's|url:.*$|url: redis://redis:6379/13|g' config/redis.trace_chunks.yml
 
+cp config/redis.rate_limiting.yml.example config/redis.rate_limiting.yml
+sed -i 's|url:.*$|url: redis://redis:6379/14|g' config/redis.rate_limiting.yml
+
 if [ "$SETUP_DB" != "false" ]; then
   setup_db
 elif getent hosts postgres; then

spec/lib/gitlab/health_checks/probes/collection_spec.rb

@@ -16,6 +16,8 @@ RSpec.describe Gitlab::HealthChecks::Probes::Collection do
           Gitlab::HealthChecks::Redis::CacheCheck,
           Gitlab::HealthChecks::Redis::QueuesCheck,
           Gitlab::HealthChecks::Redis::SharedStateCheck,
+          Gitlab::HealthChecks::Redis::TraceChunksCheck,
+          Gitlab::HealthChecks::Redis::RateLimitingCheck,
           Gitlab::HealthChecks::GitalyCheck
         ]
       end

spec/lib/gitlab/health_checks/redis/rate_limiting_check_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_relative '../simple_check_shared'
+
+RSpec.describe Gitlab::HealthChecks::Redis::RateLimitingCheck do
+  include_examples 'simple_check', 'redis_rate_limiting_ping', 'RedisRateLimiting', 'PONG'
+end

spec/lib/gitlab/instrumentation/redis_spec.rb

@@ -76,7 +76,8 @@ RSpec.describe Gitlab::Instrumentation::Redis do
                             details_row.merge(storage: 'Cache'),
                             details_row.merge(storage: 'Queues'),
                             details_row.merge(storage: 'SharedState'),
-                            details_row.merge(storage: 'TraceChunks'))
+                            details_row.merge(storage: 'TraceChunks'),
+                            details_row.merge(storage: 'RateLimiting'))
     end
   end
 end

spec/lib/gitlab/redis/rate_limiting_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Redis::RateLimiting do
+  let(:instance_specific_config_file) { "config/redis.rate_limiting.yml" }
+  let(:environment_config_file_name) { "GITLAB_REDIS_RATE_LIMITING_CONFIG_FILE" }
+  let(:cache_config_file) { nil }
+
+  before do
+    allow(Gitlab::Redis::Cache).to receive(:config_file_name).and_return(cache_config_file)
+  end
+
+  include_examples "redis_shared_examples"
+
+  describe '.config_file_name' do
+    subject { described_class.config_file_name }
+
+    let(:rails_root) { Dir.mktmpdir('redis_shared_examples') }
+
+    before do
+      # Undo top-level stub of config_file_name because we are testing that method now.
+      allow(described_class).to receive(:config_file_name).and_call_original
+
+      allow(described_class).to receive(:rails_root).and_return(rails_root)
+      FileUtils.mkdir_p(File.join(rails_root, 'config'))
+    end
+
+    after do
+      FileUtils.rm_rf(rails_root)
+    end
+
+    context 'when there is only a resque.yml' do
+      before do
+        FileUtils.touch(File.join(rails_root, 'config/resque.yml'))
+      end
+
+      it { expect(subject).to eq("#{rails_root}/config/resque.yml") }
+
+      context 'and there is a global env override' do
+        before do
+          stub_env('GITLAB_REDIS_CONFIG_FILE', 'global override')
+        end
+
+        it { expect(subject).to eq('global override') }
+
+        context 'and Cache has a different config file' do
+          let(:cache_config_file) { 'cache config file' }
+
+          it { expect(subject).to eq('cache config file') }
+        end
+      end
+    end
+  end
+end

spec/support/redis.rb

@@ -38,4 +38,12 @@ RSpec.configure do |config|
 
     redis_trace_chunks_cleanup!
   end
+
+  config.around(:each, :clean_gitlab_redis_rate_limiting) do |example|
+    redis_rate_limiting_cleanup!
+
+    example.run
+
+    redis_rate_limiting_cleanup!
+  end
 end

spec/support/redis/redis_helpers.rb

@@ -22,4 +22,9 @@ module RedisHelpers
   def redis_trace_chunks_cleanup!
     Gitlab::Redis::TraceChunks.with(&:flushdb)
   end
+
+  # Usage: rate limiting state (for Rack::Attack)
+  def redis_rate_limiting_cleanup!
+    Gitlab::Redis::RateLimiting.with(&:flushdb)
+  end
 end