Commit 0f1e54ad authored by Robert Speicher's avatar Robert Speicher

Merge branch 'rd-extract-ee-specific-sidekiq-queue-config' into 'master'

Extracts EE specific Sidekiq queue config to a new file

See merge request gitlab-org/gitlab-ee!8470
parents a111a9fb a3aa0477
......@@ -24,10 +24,8 @@
- cronjob:prune_web_hook_logs
- gcp_cluster:cluster_install_app
- gcp_cluster:cluster_update_app
- gcp_cluster:cluster_provision
- gcp_cluster:cluster_wait_for_app_installation
- gcp_cluster:cluster_wait_for_app_update
- gcp_cluster:wait_for_cluster_creation
- gcp_cluster:cluster_wait_for_ingress_ip_address
- gcp_cluster:cluster_platform_configure
......@@ -83,10 +81,9 @@
- todos_destroyer:todos_destroyer_confidential_issue
- todos_destroyer:todos_destroyer_entity_leave
- todos_destroyer:todos_destroyer_project_private
- todos_destroyer:todos_destroyer_group_private
- todos_destroyer:todos_destroyer_project_private
- todos_destroyer:todos_destroyer_private_features
- todos_destroyer:todos_destroyer_group_private
- default
- mailers # ActionMailer::DeliveryJob.queue_name
......@@ -135,66 +132,3 @@
- create_note_diff_file
- delete_diff_files
- detect_repository_languages
# EE-specific queues
# Deprecated queues: remove after 10.7
- geo_base_scheduler
- geo_file_download
- geo_project_sync
- geo_repository_shard_sync
- cronjob:clear_shared_runners_minutes
- cronjob:geo_file_download_dispatch
- cronjob:geo_metrics_update
- cronjob:geo_prune_event_log
- cronjob:geo_migrated_local_files_clean_up
- cronjob:geo_repository_sync
- cronjob:geo_repository_verification_primary_batch
- cronjob:geo_repository_verification_secondary_scheduler
- cronjob:geo_sidekiq_cron_config
- cronjob:geo_scheduler_per_shard_scheduler
- cronjob:geo_scheduler_primary_per_shard_scheduler
- cronjob:geo_scheduler_secondary_per_shard_scheduler
- cronjob:geo_repository_verification_secondary_shard
- cronjob:historical_data
- cronjob:ldap_all_groups_sync
- cronjob:ldap_sync
- cronjob:update_all_mirrors
- cronjob:pseudonymizer
- geo:geo_batch_project_registry
- geo:geo_batch_project_registry_scheduler
- geo:geo_scheduler_scheduler
- geo:geo_scheduler_primary_scheduler
- geo:geo_scheduler_secondary_scheduler
- geo:geo_file_download
- geo:geo_file_removal
- geo:geo_file_registry_removal
- geo:geo_hashed_storage_attachments_migration
- geo:geo_hashed_storage_migration
- geo:geo_project_sync
- geo:geo_rename_repository
- geo:geo_repositories_clean_up
- geo:geo_repository_destroy
- geo:geo_repository_shard_sync
- geo:geo_repository_verification_primary_shard
- geo:geo_repository_verification_primary_single
- geo:geo_repository_verification_secondary_single
- pipeline_default:store_security_reports
- admin_emails
- create_github_webhook
- elastic_batch_project_indexer
- elastic_commit_indexer
- elastic_indexer
- export_csv
- ldap_group_sync
- new_epic
- project_import_schedule
- project_update_repository_storage
- rebase
- repository_update_mirror
- chat_notification
......@@ -17,8 +17,8 @@ would be `process_something`. If you're not sure what queue a worker uses,
you can find it using `SomeWorker.queue`. There is almost never a reason to
manually override the queue name using `sidekiq_options queue: :some_queue`.
You must always add any new queues to `app/workers/all_queues.yml` otherwise
your worker will not run.
You must always add any new queues to `app/workers/all_queues.yml` or `ee/app/workers/all_queues.yml`
otherwise your worker will not run.
## Queue Namespaces
......
---
- cronjob:clear_shared_runners_minutes
- cronjob:geo_file_download_dispatch
- cronjob:geo_metrics_update
- cronjob:geo_prune_event_log
- cronjob:geo_migrated_local_files_clean_up
- cronjob:geo_repository_sync
- cronjob:geo_repository_verification_primary_batch
- cronjob:geo_repository_verification_secondary_scheduler
- cronjob:geo_sidekiq_cron_config
- cronjob:geo_scheduler_per_shard_scheduler
- cronjob:geo_scheduler_primary_per_shard_scheduler
- cronjob:geo_scheduler_secondary_per_shard_scheduler
- cronjob:geo_repository_verification_secondary_shard
- cronjob:historical_data
- cronjob:ldap_all_groups_sync
- cronjob:ldap_sync
- cronjob:update_all_mirrors
- cronjob:pseudonymizer
- gcp_cluster:cluster_update_app
- gcp_cluster:cluster_wait_for_app_update
- geo:geo_batch_project_registry
- geo:geo_batch_project_registry_scheduler
- geo:geo_file_download
- geo:geo_file_removal
- geo:geo_file_registry_removal
- geo:geo_hashed_storage_attachments_migration
- geo:geo_hashed_storage_migration
- geo:geo_project_sync
- geo:geo_rename_repository
- geo:geo_repositories_clean_up
- geo:geo_repository_destroy
- geo:geo_repository_shard_sync
- geo:geo_repository_verification_primary_shard
- geo:geo_repository_verification_primary_single
- geo:geo_repository_verification_secondary_single
- geo:geo_scheduler_scheduler
- geo:geo_scheduler_primary_scheduler
- geo:geo_scheduler_secondary_scheduler
- pipeline_default:store_security_reports
- admin_emails
- chat_notification
- create_github_webhook
- elastic_batch_project_indexer
- elastic_commit_indexer
- elastic_indexer
- export_csv
- ldap_group_sync
- new_epic
- project_import_schedule
- project_update_repository_storage
- rebase
- repository_update_mirror
......@@ -5,11 +5,18 @@ require 'set'
module Gitlab
module SidekiqConfig
QUEUE_CONFIG_PATHS = %w[app/workers/all_queues.yml ee/app/workers/all_queues.yml].freeze
# This method is called by `bin/sidekiq-cluster` in EE, which runs outside
# of bundler/Rails context, so we cannot use any gem or Rails methods.
def self.worker_queues(rails_path = Rails.root.to_s)
@worker_queues ||= {}
@worker_queues[rails_path] ||= YAML.load_file(File.join(rails_path, 'app/workers/all_queues.yml'))
@worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
full_path = File.join(rails_path, path)
File.exist?(full_path) ? YAML.load_file(full_path) : []
end
end
# This method is called by `bin/sidekiq-cluster` in EE, which runs outside
......
......@@ -11,7 +11,7 @@ describe 'Every Sidekiq worker' do
expect(Gitlab::SidekiqConfig.cron_workers.map(&:queue)).to all(start_with('cronjob:'))
end
it 'has its queue in app/workers/all_queues.yml', :aggregate_failures do
it 'has its queue in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS', :aggregate_failures do
file_worker_queues = Gitlab::SidekiqConfig.worker_queues.to_set
worker_queues = Gitlab::SidekiqConfig.workers.map(&:queue).to_set
......@@ -19,10 +19,10 @@ describe 'Every Sidekiq worker' do
worker_queues << 'default'
missing_from_file = worker_queues - file_worker_queues - DEPRECATED_QUEUES
expect(missing_from_file).to be_empty, "expected #{missing_from_file.to_a.inspect} to be in app/workers/all_queues.yml"
expect(missing_from_file).to be_empty, "expected #{missing_from_file.to_a.inspect} to be in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS"
unncessarily_in_file = file_worker_queues - worker_queues - DEPRECATED_QUEUES
expect(unncessarily_in_file).to be_empty, "expected #{unncessarily_in_file.to_a.inspect} not to be in app/workers/all_queues.yml"
expect(unncessarily_in_file).to be_empty, "expected #{unncessarily_in_file.to_a.inspect} not to be in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS"
end
it 'has its queue or namespace in config/sidekiq_queues.yml', :aggregate_failures do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment