Commit 4484de15 authored by Sean McGivern's avatar Sean McGivern

Move SidekiqConfig methods used by CLI to own module

These methods can't use any Rails dependencies, which also means the
SidekiqConfig module definition - i.e. constants - can't use Rails
dependencies either.

Move those methods to their own module to make this clearer. Also switch
to defining methods on the singleton class in both modules to make
private methods more convenient.
parent 4ac2a641
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
require 'optparse' require 'optparse'
require_relative '../../lib/gitlab' require_relative '../../lib/gitlab'
require_relative '../../lib/gitlab/utils' require_relative '../../lib/gitlab/utils'
require_relative '../../lib/gitlab/sidekiq_config' require_relative '../../lib/gitlab/sidekiq_config/cli_methods'
require_relative '../lib/gitlab/sidekiq_cluster' require_relative '../lib/gitlab/sidekiq_cluster'
require_relative '../lib/gitlab/sidekiq_cluster/cli' require_relative '../lib/gitlab/sidekiq_cluster/cli'
......
...@@ -42,10 +42,10 @@ module Gitlab ...@@ -42,10 +42,10 @@ module Gitlab
queue_groups = SidekiqCluster.parse_queues(argv) queue_groups = SidekiqCluster.parse_queues(argv)
all_queues = SidekiqConfig.worker_queues(@rails_path) all_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path)
queue_groups.map! do |queues| queue_groups.map! do |queues|
SidekiqConfig.expand_queues(queues, all_queues) SidekiqConfig::CliMethods.expand_queues(queues, all_queues)
end end
if @negate_queues if @negate_queues
......
...@@ -32,7 +32,7 @@ describe Gitlab::SidekiqCluster::CLI do ...@@ -32,7 +32,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'with --negate flag' do context 'with --negate flag' do
it 'starts Sidekiq workers for all queues in all_queues.yml except the ones in argv' do it 'starts Sidekiq workers for all queues in all_queues.yml except the ones in argv' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(['baz']) expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(['baz'])
expect(Gitlab::SidekiqCluster).to receive(:start) expect(Gitlab::SidekiqCluster).to receive(:start)
.with([['baz']], default_options) .with([['baz']], default_options)
.and_return([]) .and_return([])
...@@ -43,7 +43,7 @@ describe Gitlab::SidekiqCluster::CLI do ...@@ -43,7 +43,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'with --max-concurrency flag' do context 'with --max-concurrency flag' do
it 'starts Sidekiq workers for specified queues with a max concurrency' do it 'starts Sidekiq workers for specified queues with a max concurrency' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(%w(foo bar baz)) expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(%w(foo bar baz))
expect(Gitlab::SidekiqCluster).to receive(:start) expect(Gitlab::SidekiqCluster).to receive(:start)
.with([%w(foo bar baz), %w(solo)], default_options.merge(max_concurrency: 2)) .with([%w(foo bar baz), %w(solo)], default_options.merge(max_concurrency: 2))
.and_return([]) .and_return([])
...@@ -54,7 +54,7 @@ describe Gitlab::SidekiqCluster::CLI do ...@@ -54,7 +54,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'queue namespace expansion' do context 'queue namespace expansion' do
it 'starts Sidekiq workers for all queues in all_queues.yml with a namespace in argv' do it 'starts Sidekiq workers for all queues in all_queues.yml with a namespace in argv' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(['cronjob:foo', 'cronjob:bar']) expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(['cronjob:foo', 'cronjob:bar'])
expect(Gitlab::SidekiqCluster).to receive(:start) expect(Gitlab::SidekiqCluster).to receive(:start)
.with([['cronjob', 'cronjob:foo', 'cronjob:bar']], default_options) .with([['cronjob', 'cronjob:foo', 'cronjob:bar']], default_options)
.and_return([]) .and_return([])
......
# frozen_string_literal: true # frozen_string_literal: true
require 'yaml' require 'yaml'
require 'set'
module Gitlab module Gitlab
module SidekiqConfig module SidekiqConfig
QUEUE_CONFIG_PATHS = begin class << self
result = %w[app/workers/all_queues.yml] include Gitlab::SidekiqConfig::CliMethods
result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
result
end.freeze
# This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside
# of bundler/Rails context, so we cannot use any gem or Rails methods.
def self.worker_queues(rails_path = Rails.root.to_s)
@worker_queues ||= {}
@worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
full_path = File.join(rails_path, path)
File.exist?(full_path) ? YAML.load_file(full_path) : []
end
end
# This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside
# of bundler/Rails context, so we cannot use any gem or Rails methods.
def self.expand_queues(queues, all_queues = self.worker_queues)
return [] if queues.empty?
queues_set = all_queues.to_set def redis_queues
queues.flat_map do |queue|
[queue, *queues_set.grep(/\A#{queue}:/)]
end
end
def self.redis_queues
# Not memoized, because this can change during the life of the application # Not memoized, because this can change during the life of the application
Sidekiq::Queue.all.map(&:name) Sidekiq::Queue.all.map(&:name)
end end
def self.config_queues def config_queues
@config_queues ||= begin @config_queues ||= begin
config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml')) config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml'))
config[:queues].map(&:first) config[:queues].map(&:first)
end end
end end
def self.cron_workers def cron_workers
@cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize } @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
end end
def self.workers def workers
@workers ||= begin @workers ||= begin
result = find_workers(Rails.root.join('app', 'workers')) result = find_workers(Rails.root.join('app', 'workers'))
result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee? result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee?
...@@ -59,7 +31,9 @@ module Gitlab ...@@ -59,7 +31,9 @@ module Gitlab
end end
end end
def self.find_workers(root) private
def find_workers(root)
concerns = root.join('concerns').to_s concerns = root.join('concerns').to_s
workers = Dir[root.join('**', '*.rb')] workers = Dir[root.join('**', '*.rb')]
...@@ -75,4 +49,5 @@ module Gitlab ...@@ -75,4 +49,5 @@ module Gitlab
workers.select { |w| w < Sidekiq::Worker } workers.select { |w| w < Sidekiq::Worker }
end end
end end
end
end end
# frozen_string_literal: true
require 'yaml'
require 'set'
# These methods are called by `sidekiq-cluster`, which runs outside of
# the bundler/Rails context, so we cannot use any gem or Rails methods.
module Gitlab
module SidekiqConfig
module CliMethods
# The methods in this module are used as module methods
# rubocop:disable Gitlab/ModuleWithInstanceVariables
extend self
QUEUE_CONFIG_PATHS = begin
result = %w[app/workers/all_queues.yml]
result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
result
end.freeze
def worker_queues(rails_path = Rails.root.to_s)
@worker_queues ||= {}
@worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
full_path = File.join(rails_path, path)
File.exist?(full_path) ? YAML.load_file(full_path) : []
end
end
def expand_queues(queues, all_queues = self.worker_queues)
return [] if queues.empty?
queues_set = all_queues.to_set
queues.flat_map do |queue|
[queue, *queues_set.grep(/\A#{queue}:/)]
end
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment