Merge branch 'rd-extract-ee-specific-sidekiq-queue-config' into 'master'

Extracts EE specific Sidekiq queue config to a new file

See merge request gitlab-org/gitlab-ee!8470

app/workers/all_queues.yml

@@ -24,10 +24,8 @@
 - cronjob:prune_web_hook_logs
 
 - gcp_cluster:cluster_install_app
-- gcp_cluster:cluster_update_app
 - gcp_cluster:cluster_provision
 - gcp_cluster:cluster_wait_for_app_installation
-- gcp_cluster:cluster_wait_for_app_update
 - gcp_cluster:wait_for_cluster_creation
 - gcp_cluster:cluster_wait_for_ingress_ip_address
 - gcp_cluster:cluster_platform_configure
@@ -83,10 +81,9 @@
 
 - todos_destroyer:todos_destroyer_confidential_issue
 - todos_destroyer:todos_destroyer_entity_leave
-- todos_destroyer:todos_destroyer_project_private
 - todos_destroyer:todos_destroyer_group_private
+- todos_destroyer:todos_destroyer_project_private
 - todos_destroyer:todos_destroyer_private_features
-- todos_destroyer:todos_destroyer_group_private
 
 - default
 - mailers # ActionMailer::DeliveryJob.queue_name
@@ -135,66 +132,3 @@
 - create_note_diff_file
 - delete_diff_files
 - detect_repository_languages
-
-# EE-specific queues
-
-# Deprecated queues: remove after 10.7
-- geo_base_scheduler
-- geo_file_download
-- geo_project_sync
-- geo_repository_shard_sync
-
-- cronjob:clear_shared_runners_minutes
-- cronjob:geo_file_download_dispatch
-- cronjob:geo_metrics_update
-- cronjob:geo_prune_event_log
-- cronjob:geo_migrated_local_files_clean_up
-- cronjob:geo_repository_sync
-- cronjob:geo_repository_verification_primary_batch
-- cronjob:geo_repository_verification_secondary_scheduler
-- cronjob:geo_sidekiq_cron_config
-- cronjob:geo_scheduler_per_shard_scheduler
-- cronjob:geo_scheduler_primary_per_shard_scheduler
-- cronjob:geo_scheduler_secondary_per_shard_scheduler
-- cronjob:geo_repository_verification_secondary_shard
-- cronjob:historical_data
-- cronjob:ldap_all_groups_sync
-- cronjob:ldap_sync
-- cronjob:update_all_mirrors
-- cronjob:pseudonymizer
-
-- geo:geo_batch_project_registry
-- geo:geo_batch_project_registry_scheduler
-
-- geo:geo_scheduler_scheduler
-- geo:geo_scheduler_primary_scheduler
-- geo:geo_scheduler_secondary_scheduler
-- geo:geo_file_download
-- geo:geo_file_removal
-- geo:geo_file_registry_removal
-- geo:geo_hashed_storage_attachments_migration
-- geo:geo_hashed_storage_migration
-- geo:geo_project_sync
-- geo:geo_rename_repository
-- geo:geo_repositories_clean_up
-- geo:geo_repository_destroy
-- geo:geo_repository_shard_sync
-- geo:geo_repository_verification_primary_shard
-- geo:geo_repository_verification_primary_single
-- geo:geo_repository_verification_secondary_single
-
-- pipeline_default:store_security_reports
-
-- admin_emails
-- create_github_webhook
-- elastic_batch_project_indexer
-- elastic_commit_indexer
-- elastic_indexer
-- export_csv
-- ldap_group_sync
-- new_epic
-- project_import_schedule
-- project_update_repository_storage
-- rebase
-- repository_update_mirror
-- chat_notification

doc/development/sidekiq_style_guide.md

@@ -17,8 +17,8 @@ would be `process_something`. If you're not sure what queue a worker uses,
 you can find it using `SomeWorker.queue`. There is almost never a reason to
 manually override the queue name using `sidekiq_options queue: :some_queue`.
 
-You must always add any new queues to `app/workers/all_queues.yml` otherwise
-your worker will not run.
+You must always add any new queues to `app/workers/all_queues.yml` or `ee/app/workers/all_queues.yml`
+otherwise your worker will not run.
 
 ## Queue Namespaces
 

ee/app/workers/all_queues.yml

+---
+- cronjob:clear_shared_runners_minutes
+- cronjob:geo_file_download_dispatch
+- cronjob:geo_metrics_update
+- cronjob:geo_prune_event_log
+- cronjob:geo_migrated_local_files_clean_up
+- cronjob:geo_repository_sync
+- cronjob:geo_repository_verification_primary_batch
+- cronjob:geo_repository_verification_secondary_scheduler
+- cronjob:geo_sidekiq_cron_config
+- cronjob:geo_scheduler_per_shard_scheduler
+- cronjob:geo_scheduler_primary_per_shard_scheduler
+- cronjob:geo_scheduler_secondary_per_shard_scheduler
+- cronjob:geo_repository_verification_secondary_shard
+- cronjob:historical_data
+- cronjob:ldap_all_groups_sync
+- cronjob:ldap_sync
+- cronjob:update_all_mirrors
+- cronjob:pseudonymizer
+
+- gcp_cluster:cluster_update_app
+- gcp_cluster:cluster_wait_for_app_update
+
+- geo:geo_batch_project_registry
+- geo:geo_batch_project_registry_scheduler
+- geo:geo_file_download
+- geo:geo_file_removal
+- geo:geo_file_registry_removal
+- geo:geo_hashed_storage_attachments_migration
+- geo:geo_hashed_storage_migration
+- geo:geo_project_sync
+- geo:geo_rename_repository
+- geo:geo_repositories_clean_up
+- geo:geo_repository_destroy
+- geo:geo_repository_shard_sync
+- geo:geo_repository_verification_primary_shard
+- geo:geo_repository_verification_primary_single
+- geo:geo_repository_verification_secondary_single
+- geo:geo_scheduler_scheduler
+- geo:geo_scheduler_primary_scheduler
+- geo:geo_scheduler_secondary_scheduler
+
+- pipeline_default:store_security_reports
+
+- admin_emails
+- chat_notification
+- create_github_webhook
+- elastic_batch_project_indexer
+- elastic_commit_indexer
+- elastic_indexer
+- export_csv
+- ldap_group_sync
+- new_epic
+- project_import_schedule
+- project_update_repository_storage
+- rebase
+- repository_update_mirror

lib/gitlab/sidekiq_config.rb

@@ -5,11 +5,18 @@ require 'set'
 
 module Gitlab
   module SidekiqConfig
+    QUEUE_CONFIG_PATHS = %w[app/workers/all_queues.yml ee/app/workers/all_queues.yml].freeze
+
     # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
     # of bundler/Rails context, so we cannot use any gem or Rails methods.
     def self.worker_queues(rails_path = Rails.root.to_s)
       @worker_queues ||= {}
-      @worker_queues[rails_path] ||= YAML.load_file(File.join(rails_path, 'app/workers/all_queues.yml'))
+
+      @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
+        full_path = File.join(rails_path, path)
+
+        File.exist?(full_path) ? YAML.load_file(full_path) : []
+      end
     end
 
     # This method is called by `bin/sidekiq-cluster` in EE, which runs outside

spec/workers/every_sidekiq_worker_spec.rb

@@ -11,7 +11,7 @@ describe 'Every Sidekiq worker' do
     expect(Gitlab::SidekiqConfig.cron_workers.map(&:queue)).to all(start_with('cronjob:'))
   end
 
-  it 'has its queue in app/workers/all_queues.yml', :aggregate_failures do
+  it 'has its queue in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS', :aggregate_failures do
     file_worker_queues = Gitlab::SidekiqConfig.worker_queues.to_set
 
     worker_queues = Gitlab::SidekiqConfig.workers.map(&:queue).to_set
@@ -19,10 +19,10 @@ describe 'Every Sidekiq worker' do
     worker_queues << 'default'
 
     missing_from_file = worker_queues - file_worker_queues - DEPRECATED_QUEUES
-    expect(missing_from_file).to be_empty, "expected #{missing_from_file.to_a.inspect} to be in app/workers/all_queues.yml"
+    expect(missing_from_file).to be_empty, "expected #{missing_from_file.to_a.inspect} to be in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS"
 
     unncessarily_in_file = file_worker_queues - worker_queues - DEPRECATED_QUEUES
-    expect(unncessarily_in_file).to be_empty, "expected #{unncessarily_in_file.to_a.inspect} not to be in app/workers/all_queues.yml"
+    expect(unncessarily_in_file).to be_empty, "expected #{unncessarily_in_file.to_a.inspect} not to be in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS"
   end
 
   it 'has its queue or namespace in config/sidekiq_queues.yml', :aggregate_failures do