Merge branch 'jv-redis-wrapper-dry' into 'master'

Factor out redis store duplication

See merge request gitlab-org/gitlab!62689

lib/gitlab/redis/cache.rb

 # frozen_string_literal: true
 
-# please require all dependencies below:
-require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present?
-
 module Gitlab
   module Redis
     class Cache < ::Gitlab::Redis::Wrapper
       CACHE_NAMESPACE = 'cache:gitlab'
-      DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'
-      REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'
 
       class << self
         def default_url
-          DEFAULT_REDIS_CACHE_URL
-        end
-
-        def config_file_name
-          # if ENV set for this class, use it even if it points to a file does not exist
-          file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME]
-          return file_name unless file_name.nil?
-
-          # otherwise, if config files exists for this class, use it
-          file_name = config_file_path('redis.cache.yml')
-          return file_name if File.file?(file_name)
-
-          # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
-          super
-        end
-
-        def instrumentation_class
-          ::Gitlab::Instrumentation::Redis::Cache
+          'redis://localhost:6380'
         end
       end
     end

lib/gitlab/redis/queues.rb

 # frozen_string_literal: true
 
-# please require all dependencies below:
+# We need this require for MailRoom
 require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
 
 module Gitlab
@@ -8,29 +8,10 @@ module Redis
     class Queues < ::Gitlab::Redis::Wrapper
       SIDEKIQ_NAMESPACE = 'resque:gitlab'
       MAILROOM_NAMESPACE = 'mail_room:gitlab'
-      DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'
-      REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'
 
       class << self
         def default_url
-          DEFAULT_REDIS_QUEUES_URL
-        end
-
-        def config_file_name
-          # if ENV set for this class, use it even if it points to a file does not exist
-          file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME]
-          return file_name if file_name
-
-          # otherwise, if config files exists for this class, use it
-          file_name = config_file_path('redis.queues.yml')
-          return file_name if File.file?(file_name)
-
-          # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
-          super
-        end
-
-        def instrumentation_class
-          ::Gitlab::Instrumentation::Redis::Queues
+          'redis://localhost:6381'
         end
       end
     end

lib/gitlab/redis/shared_state.rb

 # frozen_string_literal: true
 
-# please require all dependencies below:
-require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
-
 module Gitlab
   module Redis
     class SharedState < ::Gitlab::Redis::Wrapper
@@ -10,29 +7,10 @@ class SharedState < ::Gitlab::Redis::Wrapper
       USER_SESSIONS_NAMESPACE = 'session:user:gitlab'
       USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab'
       IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2'
-      DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'
-      REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'
 
       class << self
         def default_url
-          DEFAULT_REDIS_SHARED_STATE_URL
-        end
-
-        def config_file_name
-          # if ENV set for this class, use it even if it points to a file does not exist
-          file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME]
-          return file_name if file_name
-
-          # otherwise, if config files exists for this class, use it
-          file_name = config_file_path('redis.shared_state.yml')
-          return file_name if File.file?(file_name)
-
-          # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent
-          super
-        end
-
-        def instrumentation_class
-          ::Gitlab::Instrumentation::Redis::SharedState
+          'redis://localhost:6382'
         end
       end
     end

lib/gitlab/redis/wrapper.rb

 # frozen_string_literal: true
 
 # This file should only be used by sub-classes, not directly by any clients of the sub-classes
-# please require all dependencies below:
+
+# Explicitly load parts of ActiveSupport because MailRoom does not load
+# Rails.
 require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/module/delegation'
+require 'active_support/core_ext/string/inflections'
 
 module Gitlab
   module Redis
     class Wrapper
-      DEFAULT_REDIS_URL = 'redis://localhost:6379'
-      REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'
-
       class << self
         delegate :params, :url, to: :new
 
@@ -52,32 +52,35 @@ def _raw_config
         end
 
         def default_url
-          DEFAULT_REDIS_URL
+          raise NotImplementedError
         end
 
-        # Return the absolute path to a Rails configuration file
-        #
-        # We use this instead of `Rails.root` because for certain tasks
-        # utilizing these classes, `Rails` might not be available.
         def config_file_path(filename)
-          File.expand_path("../../../config/#{filename}", __dir__)
+          path = File.join(rails_root, 'config', filename)
+          return path if File.file?(path)
         end
 
-        def config_file_name
-          # if ENV set for wrapper class, use it even if it points to a file does not exist
-          file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME]
-          return file_name unless file_name.nil?
+        # We need this local implementation of Rails.root because MailRoom
+        # doesn't load Rails.
+        def rails_root
+          File.expand_path('../../..', __dir__)
+        end
 
-          # otherwise, if config files exists for wrapper class, use it
-          file_name = config_file_path('resque.yml')
-          return file_name if File.file?(file_name)
+        def config_file_name
+          [
+            ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"],
+            config_file_path("redis.#{store_name.underscore}.yml"),
+            ENV['GITLAB_REDIS_CONFIG_FILE'],
+            config_file_path('resque.yml')
+          ].compact.first
+        end
 
-          # nil will force use of DEFAULT_REDIS_URL when config file is absent
-          nil
+        def store_name
+          name.demodulize
         end
 
         def instrumentation_class
-          raise NotImplementedError
+          "::Gitlab::Instrumentation::Redis::#{store_name}".constantize
         end
       end
 

spec/config/mail_room_spec.rb

@@ -43,7 +43,7 @@
 
   context 'when both incoming email and service desk email are enabled' do
     let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' }
-    let(:queues_config_path) { 'spec/fixtures/config/redis_queues_new_format_host.yml' }
+    let(:queues_config_path) { 'spec/fixtures/config/redis_new_format_host.yml' }
     let(:gitlab_redis_queues) { Gitlab::Redis::Queues.new(Rails.env) }
 
     it 'contains the intended configuration' do
@@ -72,7 +72,7 @@
 
   context 'when both incoming email and service desk email are enabled for Microsoft Graph' do
     let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled_ms_graph.yml' }
-    let(:queues_config_path) { 'spec/fixtures/config/redis_queues_new_format_host.yml' }
+    let(:queues_config_path) { 'spec/fixtures/config/redis_new_format_host.yml' }
     let(:gitlab_redis_queues) { Gitlab::Redis::Queues.new(Rails.env) }
 
     it 'contains the intended configuration' do

spec/fixtures/config/redis_cache_config_with_env.yml

-test:
-  url: <%= ENV['TEST_GITLAB_REDIS_CACHE_URL'] %>

spec/fixtures/config/redis_cache_new_format_host.yml

-# redis://[:password@]host[:port][/db-number][?option=value]
-# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
-development:
-  url: redis://:mynewpassword@localhost:6380/10
-  sentinels:
-    -
-      host: localhost
-      port: 26380 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26380 # point to sentinel, not to redis port
-test:
-  url: redis://:mynewpassword@localhost:6380/10
-  sentinels:
-    -
-      host: localhost
-      port: 26380 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26380 # point to sentinel, not to redis port
-production:
-  url: redis://:mynewpassword@localhost:6380/10
-  sentinels:
-    -
-      host: replica1
-      port: 26380 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26380 # point to sentinel, not to redis port

spec/fixtures/config/redis_cache_new_format_socket.yml

-development:
-  url: unix:/path/to/redis.cache.sock
-test:
-  url: unix:/path/to/redis.cache.sock
-production:
-  url: unix:/path/to/redis.cache.sock

spec/fixtures/config/redis_cache_old_format_host.yml

-# redis://[:password@]host[:port][/db-number][?option=value]
-# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
-development: redis://:mypassword@localhost:6380/10
-test: redis://:mypassword@localhost:6380/10
-production: redis://:mypassword@localhost:6380/10

spec/fixtures/config/redis_cache_old_format_socket.yml

-development: unix:/path/to/old/redis.cache.sock
-test: unix:/path/to/old/redis.cache.sock
-production: unix:/path/to/old/redis.cache.sock

spec/fixtures/config/redis_new_format_host.yml

 # redis://[:password@]host[:port][/db-number][?option=value]
 # more details: http://www.iana.org/assignments/uri-schemes/prov/redis
 development:
-  url: redis://:mynewpassword@localhost:6379/99
+  url: redis://:mynewpassword@development-host:6379/99
   sentinels:
     -
-      host: localhost
+      host: development-replica1
       port: 26379 # point to sentinel, not to redis port
     -
-      host: replica2
+      host: development-replica2
       port: 26379 # point to sentinel, not to redis port
 test:
-  url: redis://:mynewpassword@localhost:6379/99
+  url: redis://:mynewpassword@test-host:6379/99
   sentinels:
     -
-      host: localhost
+      host: test-replica1
       port: 26379 # point to sentinel, not to redis port
     -
-      host: replica2
+      host: test-replica2
       port: 26379 # point to sentinel, not to redis port
 production:
-  url: redis://:mynewpassword@localhost:6379/99
+  url: redis://:mynewpassword@production-host:6379/99
   sentinels:
     -
-      host: replica1
+      host: production-replica1
       port: 26379 # point to sentinel, not to redis port
     -
-      host: replica2
+      host: production-replica2
       port: 26379 # point to sentinel, not to redis port

spec/fixtures/config/redis_queues_config_with_env.yml

-test:
-  url: <%= ENV['TEST_GITLAB_REDIS_QUEUES_URL'] %>

spec/fixtures/config/redis_queues_new_format_host.yml

-# redis://[:password@]host[:port][/db-number][?option=value]
-# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
-development:
-  url: redis://:mynewpassword@localhost:6381/11
-  sentinels:
-    -
-      host: localhost
-      port: 26381 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26381 # point to sentinel, not to redis port
-test:
-  url: redis://:mynewpassword@localhost:6381/11
-  sentinels:
-    -
-      host: localhost
-      port: 26381 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26381 # point to sentinel, not to redis port
-production:
-  url: redis://:mynewpassword@localhost:6381/11
-  sentinels:
-    -
-      host: replica1
-      port: 26381 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26381 # point to sentinel, not to redis port

spec/fixtures/config/redis_queues_new_format_socket.yml

-development:
-  url: unix:/path/to/redis.queues.sock
-test:
-  url: unix:/path/to/redis.queues.sock
-production:
-  url: unix:/path/to/redis.queues.sock

spec/fixtures/config/redis_queues_old_format_host.yml

-# redis://[:password@]host[:port][/db-number][?option=value]
-# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
-development: redis://:mypassword@localhost:6381/11
-test: redis://:mypassword@localhost:6381/11
-production: redis://:mypassword@localhost:6381/11

spec/fixtures/config/redis_queues_old_format_socket.yml

-development: unix:/path/to/old/redis.queues.sock
-test: unix:/path/to/old/redis.queues.sock
-production: unix:/path/to/old/redis.queues.sock

spec/fixtures/config/redis_shared_state_config_with_env.yml

-test:
-  url: <%= ENV['TEST_GITLAB_REDIS_SHARED_STATE_URL'] %>

spec/fixtures/config/redis_shared_state_new_format_host.yml

-# redis://[:password@]host[:port][/db-number][?option=value]
-# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
-development:
-  url: redis://:mynewpassword@localhost:6382/12
-  sentinels:
-    -
-      host: localhost
-      port: 26382 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26382 # point to sentinel, not to redis port
-test:
-  url: redis://:mynewpassword@localhost:6382/12
-  sentinels:
-    -
-      host: localhost
-      port: 26382 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26382 # point to sentinel, not to redis port
-production:
-  url: redis://:mynewpassword@localhost:6382/12
-  sentinels:
-    -
-      host: replica1
-      port: 26382 # point to sentinel, not to redis port
-    -
-      host: replica2
-      port: 26382 # point to sentinel, not to redis port

spec/fixtures/config/redis_shared_state_new_format_socket.yml

-development:
-  url: unix:/path/to/redis.shared_state.sock
-test:
-  url: unix:/path/to/redis.shared_state.sock
-production:
-  url: unix:/path/to/redis.shared_state.sock

spec/fixtures/config/redis_shared_state_old_format_host.yml

-# redis://[:password@]host[:port][/db-number][?option=value]
-# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
-development: redis://:mypassword@localhost:6382/12
-test: redis://:mypassword@localhost:6382/12
-production: redis://:mypassword@localhost:6382/12