Merge branch 'jh-337-enable-add-sidekiq-worker-in-jh-folder' into 'master'

Enable add sidekiq worker in jh folder

See merge request gitlab-org/gitlab!73703

lib/gitlab/sidekiq_config.rb

@@ -6,11 +6,13 @@ module Gitlab
   module SidekiqConfig
     FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml'
     EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml'
+    JH_QUEUE_CONFIG_PATH = 'jh/app/workers/all_queues.yml'
     SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml'
 
     QUEUE_CONFIG_PATHS = [
       FOSS_QUEUE_CONFIG_PATH,
-      (EE_QUEUE_CONFIG_PATH if Gitlab.ee?)
+      (EE_QUEUE_CONFIG_PATH if Gitlab.ee?),
+      (JH_QUEUE_CONFIG_PATH if Gitlab.jh?)
     ].compact.freeze
 
     # This maps workers not in our application code to queues. We need
@@ -33,7 +35,7 @@ module Gitlab
         weight: 2,
         tags: []
       )
-    }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
+    }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false, jh: false) }.freeze
 
     class << self
       include Gitlab::SidekiqConfig::CliMethods
@@ -58,10 +60,14 @@ module Gitlab
         @workers ||= begin
           result = []
           result.concat(DEFAULT_WORKERS.values)
-          result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false))
+          result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false, jh: false))
 
           if Gitlab.ee?
-            result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true))
+            result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true, jh: false))
+          end
+
+          if Gitlab.jh?
+            result.concat(find_workers(Rails.root.join('jh', 'app', 'workers'), ee: false, jh: true))
           end
 
           result
@@ -69,16 +75,26 @@ module Gitlab
       end
 
       def workers_for_all_queues_yml
-        workers.partition(&:ee?).reverse.map(&:sort)
+        workers.each_with_object([[], [], []]) do |worker, array|
+          if worker.jh?
+            array[2].push(worker)
+          elsif worker.ee?
+            array[1].push(worker)
+          else
+            array[0].push(worker)
+          end
+        end.map(&:sort)
       end
 
       # YAML.load_file is OK here as we control the file contents
       def all_queues_yml_outdated?
-        foss_workers, ee_workers = workers_for_all_queues_yml
+        foss_workers, ee_workers, jh_workers = workers_for_all_queues_yml
 
         return true if foss_workers != YAML.load_file(FOSS_QUEUE_CONFIG_PATH)
 
-        Gitlab.ee? && ee_workers != YAML.load_file(EE_QUEUE_CONFIG_PATH)
+        return true if Gitlab.ee? && ee_workers != YAML.load_file(EE_QUEUE_CONFIG_PATH)
+
+        Gitlab.jh? && File.exist?(JH_QUEUE_CONFIG_PATH) && jh_workers != YAML.load_file(JH_QUEUE_CONFIG_PATH)
       end
 
       def queues_for_sidekiq_queues_yml
@@ -120,14 +136,14 @@ module Gitlab
 
       private
 
-      def find_workers(root, ee:)
+      def find_workers(root, ee:, jh:)
         concerns = root.join('concerns').to_s
 
         Dir[root.join('**', '*.rb')]
           .reject { |path| path.start_with?(concerns) }
           .map { |path| worker_from_path(path, root) }
           .select { |worker| worker < Sidekiq::Worker }
-          .map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: ee) }
+          .map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: ee, jh: jh) }
       end
 
       def worker_from_path(path, root)

lib/gitlab/sidekiq_config/cli_methods.rb

@@ -18,6 +18,7 @@ module Gitlab
       QUEUE_CONFIG_PATHS = begin
         result = %w[app/workers/all_queues.yml]
         result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
+        result << 'jh/app/workers/all_queues.yml' if Gitlab.jh?
         result
       end.freeze
 

lib/gitlab/sidekiq_config/worker.rb

@@ -13,15 +13,20 @@ module Gitlab
                :worker_has_external_dependencies?,
                to: :klass
 
-      def initialize(klass, ee:)
+      def initialize(klass, ee:, jh: false)
         @klass = klass
         @ee = ee
+        @jh = jh
       end
 
       def ee?
         @ee
       end
 
+      def jh?
+        @jh
+      end
+
       def ==(other)
         to_yaml == case other
                    when self.class

lib/tasks/gitlab/sidekiq.rake

@@ -36,13 +36,17 @@ namespace :gitlab do
           # Do not edit it manually!
         BANNER
 
-        foss_workers, ee_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml
+        foss_workers, ee_workers, jh_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml
 
         write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, banner, foss_workers)
 
         if Gitlab.ee?
           write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, banner, ee_workers)
         end
+
+        if Gitlab.jh?
+          write_yaml(Gitlab::SidekiqConfig::JH_QUEUE_CONFIG_PATH, banner, jh_workers)
+        end
       end
 
       desc 'GitLab | Sidekiq | Validate that all_queues.yml matches worker definitions'
@@ -57,6 +61,7 @@ namespace :gitlab do
 
             - #{Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH}
             - #{Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH}
+            #{"- " + Gitlab::SidekiqConfig::JH_QUEUE_CONFIG_PATH if Gitlab.jh?}
 
           MSG
         end

spec/lib/gitlab/sidekiq_config/cli_methods_spec.rb

@@ -11,12 +11,12 @@ RSpec.describe Gitlab::SidekiqConfig::CliMethods do
     end
 
     def stub_exists(exists: true)
-      ['app/workers/all_queues.yml', 'ee/app/workers/all_queues.yml'].each do |path|
+      ['app/workers/all_queues.yml', 'ee/app/workers/all_queues.yml', 'jh/app/workers/all_queues.yml'].each do |path|
         allow(File).to receive(:exist?).with(expand_path(path)).and_return(exists)
       end
     end
 
-    def stub_contents(foss_queues, ee_queues)
+    def stub_contents(foss_queues, ee_queues, jh_queues)
       allow(YAML).to receive(:load_file)
                        .with(expand_path('app/workers/all_queues.yml'))
                        .and_return(foss_queues)
@@ -24,6 +24,10 @@ RSpec.describe Gitlab::SidekiqConfig::CliMethods do
       allow(YAML).to receive(:load_file)
                        .with(expand_path('ee/app/workers/all_queues.yml'))
                        .and_return(ee_queues)
+
+      allow(YAML).to receive(:load_file)
+                       .with(expand_path('jh/app/workers/all_queues.yml'))
+                       .and_return(jh_queues)
     end
 
     before do
@@ -45,8 +49,9 @@ RSpec.describe Gitlab::SidekiqConfig::CliMethods do
         end
 
         it 'flattens and joins the contents' do
-          expected_queues = %w[queue_a queue_b]
-          expected_queues = expected_queues.first(1) unless Gitlab.ee?
+          expected_queues = %w[queue_a]
+          expected_queues << 'queue_b' if Gitlab.ee?
+          expected_queues << 'queue_c' if Gitlab.jh?
 
           expect(described_class.worker_queues(dummy_root))
             .to match_array(expected_queues)
@@ -55,7 +60,7 @@ RSpec.describe Gitlab::SidekiqConfig::CliMethods do
 
       context 'when the file contains an array of hashes' do
         before do
-          stub_contents([{ name: 'queue_a' }], [{ name: 'queue_b' }])
+          stub_contents([{ name: 'queue_a' }], [{ name: 'queue_b' }], [{ name: 'queue_c' }])
         end
 
         include_examples 'valid file contents'

spec/lib/gitlab/sidekiq_config/worker_spec.rb

@@ -18,19 +18,26 @@ RSpec.describe Gitlab::SidekiqConfig::Worker do
       get_tags: attributes[:tags]
     )
 
-    described_class.new(inner_worker, ee: false)
+    described_class.new(inner_worker, ee: false, jh: false)
   end
 
   describe '#ee?' do
     it 'returns the EE status set on creation' do
-      expect(described_class.new(double, ee: true)).to be_ee
-      expect(described_class.new(double, ee: false)).not_to be_ee
+      expect(described_class.new(double, ee: true, jh: false)).to be_ee
+      expect(described_class.new(double, ee: false, jh: false)).not_to be_ee
+    end
+  end
+
+  describe '#jh?' do
+    it 'returns the JH status set on creation' do
+      expect(described_class.new(double, ee: false, jh: true)).to be_jh
+      expect(described_class.new(double, ee: false, jh: false)).not_to be_jh
     end
   end
 
   describe '#==' do
     def worker_with_yaml(yaml)
-      described_class.new(double, ee: false).tap do |worker|
+      described_class.new(double, ee: false, jh: false).tap do |worker|
         allow(worker).to receive(:to_yaml).and_return(yaml)
       end
     end
@@ -57,7 +64,7 @@ RSpec.describe Gitlab::SidekiqConfig::Worker do
 
         expect(worker).to receive(meth)
 
-        described_class.new(worker, ee: false).send(meth)
+        described_class.new(worker, ee: false, jh: false).send(meth)
       end
     end
   end