Exclude sidekiq queues from execution in sidekiq-cluster

bin/sidekiq-cluster

 #!/usr/bin/env ruby
 
 require 'optparse'
+require_relative '../lib/gitlab/sidekiq_config'
 require_relative '../lib/gitlab/sidekiq_cluster'
 require_relative '../lib/gitlab/sidekiq_cluster/cli'
 

config/initializers/sidekiq.rb

@@ -68,14 +68,12 @@ end
 # The Sidekiq client API always adds the queue to the Sidekiq queue
 # list, but mail_room and gitlab-shell do not. This is only necessary
 # for monitoring.
-config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s)
+queues = Gitlab::SidekiqConfig.queues
 
 begin
   Sidekiq.redis do |conn|
     conn.pipelined do
-      config[:queues].each do |queue|
-        conn.sadd('queues', queue[0])
-      end
+      queues.each { |queue| conn.sadd('queues', queue) }
     end
   end
 rescue Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED

lib/gitlab/sidekiq_cluster/cli.rb

@@ -30,7 +30,12 @@ module Gitlab
 
         option_parser.parse!(argv)
 
-        queues = SidekiqCluster.parse_queues(argv)
+        queues =
+          if @negated_queues&.any?
+            [SidekiqConfig.queues(@rails_path, except: @negated_queues)]
+          else
+            SidekiqCluster.parse_queues(argv)
+          end
 
         @logger.info("Starting cluster with #{queues.length} processes")
 
@@ -93,6 +98,10 @@ module Gitlab
             @rails_path = path
           end
 
+          opt.on('-n', '--negate [QUEUE,QUEUE]', "Run workers for all queues except these") do |queues|
+            @negated_queues = queues.split(',')
+          end
+
           opt.on('-i', '--interval INT', 'The number of seconds to wait between worker checks') do |int|
             @interval = int.to_i
           end

lib/gitlab/sidekiq_config.rb

+require 'yaml'
+
+module Gitlab
+  module SidekiqConfig
+    def self.queues(rails_path = Rails.root.to_s, except: [])
+      queues_file_path = File.join(rails_path, 'config', 'sidekiq_queues.yml')
+
+      YAML.load_file(queues_file_path).fetch(:queues).map { |queue, _| queue } - except
+    end
+  end
+end

spec/lib/gitlab/sidekiq_cluster/cli_spec.rb

@@ -12,13 +12,29 @@ describe Gitlab::SidekiqCluster::CLI do
 
     context 'with arguments' do
       it 'starts the Sidekiq workers' do
-        expect(Gitlab::SidekiqCluster).to receive(:start).and_return([])
+        expect(Gitlab::SidekiqCluster).to receive(:start).with([['foo']], 'test', Dir.pwd).and_return([])
         expect(cli).to receive(:write_pid)
         expect(cli).to receive(:trap_signals)
         expect(cli).to receive(:start_loop)
 
         cli.run(%w(foo))
       end
+
+      context 'with --negate argument' do
+        it 'starts Sidekiq workers for all queues except the negated ones' do
+          expect(Gitlab::SidekiqConfig).to receive(:queues)
+                                             .with(Dir.pwd, except: ['foo', 'bar'])
+                                             .and_return(['baz'])
+          expect(Gitlab::SidekiqCluster).to receive(:start)
+                                              .with([['baz']], 'test', Dir.pwd)
+                                              .and_return([])
+          expect(cli).to receive(:write_pid)
+          expect(cli).to receive(:trap_signals)
+          expect(cli).to receive(:start_loop)
+
+          cli.run(%w(-n foo,bar))
+        end
+      end
     end
   end
 

spec/lib/gitlab/sidekiq_config_spec.rb

+require 'rails_helper'
+
+describe Gitlab::SidekiqConfig do
+  describe '.queues' do
+    let(:queues_file_path) { Rails.root.join('config', 'sidekiq_queues.yml') }
+
+    context 'without except argument' do
+      it 'returns all queues defined on config/sidekiq_queues.yml file' do
+        expected_queues = YAML.load_file(queues_file_path)[:queues].map { |queue, _| queue }
+
+        expect(described_class.queues).to eq(expected_queues)
+      end
+    end
+
+    context 'with except argument' do
+      it 'returns queues on config/sidekiq_queues.yml filtering out excluded ones' do
+        expected_queues =
+          YAML.load_file(queues_file_path)[:queues].map { |queue, _| queue } - ['webhook']
+
+        expect(described_class.queues(except: ['webhook'])).to eq(expected_queues)
+      end
+    end
+  end
+end