Add Rake task to generate all_queues.yml

The validity of this file is already checked by
spec/workers/every_sidekiq_worker_spec.rb. This adds a second, simple
check that the file loads as YAML matching the Rake task's generated
list (including sorting).

The files generated by the task are in broadly the same format as the
current manually-written files:

1. Namespaced queues first, then un-namespaced queues.
2. Both in alphabetical order.

They don't have the manually-inserted line breaks. This doesn't delete
the RSpec test because that also adds an important check for the
addition of two queues, mailers and default, that shouldn't be lost in
any refactor here.

app/workers/all_queues.yml

+# This file is generated automatically by
+#   bin/rake gitlab:sidekiq:all_queues_yml:generate
+#
+# Do not edit it manually!
 ---
 - auto_devops:auto_devops_disable
-
 - auto_merge:auto_merge_process
-
 - chaos:chaos_cpu_spin
 - chaos:chaos_db_spin
 - chaos:chaos_kill
 - chaos:chaos_leak_mem
 - chaos:chaos_sleep
-
+- container_repository:cleanup_container_repository
+- container_repository:delete_container_repository
 - cronjob:admin_email
+- cronjob:ci_archive_traces_cron
 - cronjob:container_expiration_policy
 - cronjob:expire_build_artifacts
 - cronjob:gitlab_usage_ping
 - cronjob:import_export_project_cleanup
-- cronjob:pages_domain_verification_cron
+- cronjob:issue_due_scheduler
+- cronjob:namespaces_prune_aggregation_schedules
 - cronjob:pages_domain_removal_cron
 - cronjob:pages_domain_ssl_renewal_cron
+- cronjob:pages_domain_verification_cron
 - cronjob:personal_access_tokens_expiring
 - cronjob:pipeline_schedule
 - cronjob:prune_old_events
+- cronjob:prune_web_hook_logs
 - cronjob:remove_expired_group_links
 - cronjob:remove_expired_members
 - cronjob:remove_unreferenced_lfs_objects
 - cronjob:repository_archive_cache
 - cronjob:repository_check_dispatch
 - cronjob:requests_profiles
+- cronjob:schedule_migrate_external_diffs
 - cronjob:stuck_ci_jobs
 - cronjob:stuck_import_jobs
 - cronjob:stuck_merge_jobs
-- cronjob:ci_archive_traces_cron
 - cronjob:trending_projects
-- cronjob:issue_due_scheduler
-- cronjob:prune_web_hook_logs
-- cronjob:schedule_migrate_external_diffs
-- cronjob:namespaces_prune_aggregation_schedules
-
+- deployment:deployments_finished
+- deployment:deployments_success
+- gcp_cluster:cluster_configure
 - gcp_cluster:cluster_install_app
 - gcp_cluster:cluster_patch_app
-- gcp_cluster:cluster_upgrade_app
+- gcp_cluster:cluster_project_configure
 - gcp_cluster:cluster_provision
-- gcp_cluster:clusters_cleanup_app
-- gcp_cluster:clusters_cleanup_project_namespace
-- gcp_cluster:clusters_cleanup_service_account
+- gcp_cluster:cluster_upgrade_app
 - gcp_cluster:cluster_wait_for_app_installation
-- gcp_cluster:wait_for_cluster_creation
 - gcp_cluster:cluster_wait_for_ingress_ip_address
-- gcp_cluster:cluster_configure
-- gcp_cluster:cluster_project_configure
-- gcp_cluster:clusters_applications_wait_for_uninstall_app
+- gcp_cluster:clusters_applications_activate_service
+- gcp_cluster:clusters_applications_deactivate_service
 - gcp_cluster:clusters_applications_uninstall
+- gcp_cluster:clusters_applications_wait_for_uninstall_app
 - gcp_cluster:clusters_cleanup_app
 - gcp_cluster:clusters_cleanup_project_namespace
 - gcp_cluster:clusters_cleanup_service_account
-- gcp_cluster:clusters_applications_activate_service
-- gcp_cluster:clusters_applications_deactivate_service
-
-- github_import_advance_stage
+- gcp_cluster:wait_for_cluster_creation
 - github_importer:github_import_import_diff_note
 - github_importer:github_import_import_issue
-- github_importer:github_import_import_note
 - github_importer:github_import_import_lfs_object
+- github_importer:github_import_import_note
 - github_importer:github_import_import_pull_request
 - github_importer:github_import_refresh_import_jid
 - github_importer:github_import_stage_finish_import
 - github_importer:github_import_stage_import_base_data
 - github_importer:github_import_stage_import_issues_and_diff_notes
-- github_importer:github_import_stage_import_notes
 - github_importer:github_import_stage_import_lfs_objects
+- github_importer:github_import_stage_import_notes
 - github_importer:github_import_stage_import_pull_requests
 - github_importer:github_import_stage_import_repository
-
 - hashed_storage:hashed_storage_migrator
-- hashed_storage:hashed_storage_rollbacker
 - hashed_storage:hashed_storage_project_migrate
 - hashed_storage:hashed_storage_project_rollback
-
+- hashed_storage:hashed_storage_rollbacker
 - mail_scheduler:mail_scheduler_issue_due
 - mail_scheduler:mail_scheduler_notification_service
-
+- notifications:new_release
+- object_pool:object_pool_create
+- object_pool:object_pool_destroy
+- object_pool:object_pool_join
+- object_pool:object_pool_schedule_join
 - object_storage:object_storage_background_move
 - object_storage:object_storage_migrate_uploads
-
+- pipeline_background:archive_trace
+- pipeline_background:ci_build_trace_chunk_flush
 - pipeline_cache:expire_job_cache
 - pipeline_cache:expire_pipeline_cache
 - pipeline_creation:create_pipeline
 - pipeline_creation:run_pipeline_schedule
-- pipeline_background:archive_trace
-- pipeline_background:ci_build_trace_chunk_flush
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
 - pipeline_default:pipeline_metrics
@@ -95,74 +95,67 @@
 - pipeline_hooks:build_hooks
 - pipeline_hooks:pipeline_hooks
 - pipeline_processing:build_finished
-- pipeline_processing:ci_build_prepare
 - pipeline_processing:build_queue
 - pipeline_processing:build_success
+- pipeline_processing:ci_build_prepare
+- pipeline_processing:ci_build_schedule
+- pipeline_processing:ci_resource_groups_assign_resource_from_resource_group
 - pipeline_processing:pipeline_process
 - pipeline_processing:pipeline_success
 - pipeline_processing:pipeline_update
 - pipeline_processing:stage_update
 - pipeline_processing:update_head_pipeline_for_merge_request
-- pipeline_processing:ci_build_schedule
-- pipeline_processing:ci_resource_groups_assign_resource_from_resource_group
-
-- deployment:deployments_success
-- deployment:deployments_finished
-
-- repository_check:repository_check_clear
 - repository_check:repository_check_batch
+- repository_check:repository_check_clear
 - repository_check:repository_check_single_repository
-
 - todos_destroyer:todos_destroyer_confidential_issue
 - todos_destroyer:todos_destroyer_entity_leave
 - todos_destroyer:todos_destroyer_group_private
-- todos_destroyer:todos_destroyer_project_private
 - todos_destroyer:todos_destroyer_private_features
-
-- update_namespace_statistics:namespaces_schedule_aggregation
+- todos_destroyer:todos_destroyer_project_private
 - update_namespace_statistics:namespaces_root_statistics
-
-- object_pool:object_pool_create
-- object_pool:object_pool_schedule_join
-- object_pool:object_pool_join
-- object_pool:object_pool_destroy
-
-- container_repository:delete_container_repository
-- container_repository:cleanup_container_repository
-
-- notifications:new_release
-
-- default
-- mailers # ActionMailer::DeliveryJob.queue_name
-
+- update_namespace_statistics:namespaces_schedule_aggregation
 - authorized_projects
 - background_migration
 - chat_notification
+- create_evidence
 - create_gpg_signature
+- create_note_diff_file
+- default
+- delete_diff_files
 - delete_merged_branches
+- delete_stored_files
 - delete_user
+- detect_repository_languages
 - email_receiver
 - emails_on_push
 - error_tracking_issue_link
 - expire_build_instance_artifacts
+- file_hook
 - git_garbage_collect
+- github_import_advance_stage
 - gitlab_shell
 - group_destroy
+- group_export
+- import_issues_csv
 - invalid_gpg_signature_update
 - irker
+- mailers
 - merge
+- merge_request_mergeability_check
 - migrate_external_diffs
 - namespaceless_project_destroy
 - new_issue
 - new_merge_request
 - new_note
 - pages
-- pages_domain_verification
 - pages_domain_ssl_renewal
-- file_hook
+- pages_domain_verification
+- phabricator_import_import_tasks
 - post_receive
 - process_commit
 - project_cache
+- project_daily_statistics
 - project_destroy
 - project_export
 - project_service
@@ -170,26 +163,16 @@
 - reactive_caching
 - rebase
 - remote_mirror_notification
+- repository_cleanup
 - repository_fork
 - repository_import
 - repository_remove_remote
+- repository_update_remote_mirror
+- self_monitoring_project_create
+- self_monitoring_project_delete
 - system_hook_push
 - update_external_pull_requests
 - update_merge_requests
 - update_project_statistics
 - upload_checksum
 - web_hook
-- repository_update_remote_mirror
-- create_note_diff_file
-- delete_diff_files
-- detect_repository_languages
-- repository_cleanup
-- delete_stored_files
-- import_issues_csv
-- project_daily_statistics
-- create_evidence
-- group_export
-- self_monitoring_project_create
-- self_monitoring_project_delete
-- merge_request_mergeability_check
-- phabricator_import_import_tasks

doc/administration/operations/extra_sidekiq_processes.md

@@ -268,8 +268,9 @@ default value can be found in `/opt/gitlab/etc/gitlab-rails/env/RAILS_ENV`.
 
 ### Using negation
 
-You're able to run all queues in `sidekiq_queues.yml` file on a single or
-multiple processes with exceptions using the `--negate` flag.
+You're able to run all queues in the `all_queues.yml` file (or the equivalent EE
+file) on a single or multiple processes with exceptions using the `--negate`
+flag.
 
 For example, say you want to run a single process for all queues,
 except `process_commit` and `post_receive`:

doc/development/sidekiq_style_guide.md

@@ -17,8 +17,11 @@ would be `process_something`. If you're not sure what queue a worker uses,
 you can find it using `SomeWorker.queue`. There is almost never a reason to
 manually override the queue name using `sidekiq_options queue: :some_queue`.
 
-You must always add any new queues to `app/workers/all_queues.yml` or `ee/app/workers/all_queues.yml`
-otherwise your worker will not run.
+After adding a new queue, run `bin/rake
+gitlab:sidekiq:all_queues_yml:generate` to regenerate
+`app/workers/all_queues.yml` or `ee/app/workers/all_queues.yml` so that
+it can be picked up by
+[`sidekiq-cluster`](../administration/operations/extra_sidekiq_processes.md).
 
 ## Queue Namespaces
 

ee/app/workers/all_queues.yml

+# This file is generated automatically by
+#   bin/rake gitlab:sidekiq:all_queues_yml:generate
+#
+# Do not edit it manually!
 ---
 - cronjob:adjourned_group_deletion
+- cronjob:adjourned_projects_deletion_cron
 - cronjob:clear_shared_runners_minutes
+- cronjob:geo_container_repository_sync_dispatch
 - cronjob:geo_file_download_dispatch
 - cronjob:geo_metrics_update
-- cronjob:geo_prune_event_log
 - cronjob:geo_migrated_local_files_clean_up
+- cronjob:geo_prune_event_log
 - cronjob:geo_repository_sync
 - cronjob:geo_repository_verification_primary_batch
 - cronjob:geo_repository_verification_secondary_scheduler
-- cronjob:geo_sidekiq_cron_config
+- cronjob:geo_repository_verification_secondary_shard
 - cronjob:geo_scheduler_per_shard_scheduler
 - cronjob:geo_scheduler_primary_per_shard_scheduler
 - cronjob:geo_scheduler_secondary_per_shard_scheduler
-- cronjob:geo_repository_verification_secondary_shard
-- cronjob:geo_container_repository_sync_dispatch
+- cronjob:geo_sidekiq_cron_config
 - cronjob:historical_data
 - cronjob:import_software_licenses
 - cronjob:ldap_all_groups_sync
 - cronjob:ldap_sync
-- cronjob:update_all_mirrors
 - cronjob:pseudonymizer
+- cronjob:update_all_mirrors
 - cronjob:update_max_seats_used_for_gitlab_com_subscriptions
-- cronjob:adjourned_projects_deletion_cron
-
+- epics:epics_update_epics_dates
 - gcp_cluster:cluster_update_app
 - gcp_cluster:cluster_wait_for_app_update
-
 - geo:geo_batch_project_registry
 - geo:geo_batch_project_registry_scheduler
+- geo:geo_container_repository_sync
+- geo:geo_design_repository_shard_sync
+- geo:geo_design_repository_sync
 - geo:geo_file_download
-- geo:geo_file_removal
 - geo:geo_file_registry_removal
+- geo:geo_file_removal
 - geo:geo_hashed_storage_attachments_migration
 - geo:geo_hashed_storage_migration
 - geo:geo_project_sync
-- geo:geo_container_repository_sync
-- geo:geo_design_repository_sync
 - geo:geo_rename_repository
-- geo:geo_secondary_repository_backfill
 - geo:geo_repositories_clean_up
 - geo:geo_repository_cleanup
 - geo:geo_repository_destroy
 - geo:geo_repository_shard_sync
-- geo:geo_design_repository_shard_sync
 - geo:geo_repository_verification_primary_shard
 - geo:geo_repository_verification_primary_single
 - geo:geo_repository_verification_secondary_single
-- geo:geo_scheduler_scheduler
 - geo:geo_scheduler_primary_scheduler
+- geo:geo_scheduler_scheduler
 - geo:geo_scheduler_secondary_scheduler
-
-- pipeline_default:store_security_reports
-- pipeline_default:sync_security_reports_to_report_approval_rules
-- pipeline_default:ci_create_cross_project_pipeline
-- pipeline_default:ci_pipeline_bridge_status
-
+- geo:geo_secondary_repository_backfill
 - incident_management:incident_management_process_alert
 - incident_management:incident_management_process_prometheus_alert
-
 - jira_connect:jira_connect_sync_branch
 - jira_connect:jira_connect_sync_merge_request
-
 - personal_access_tokens:personal_access_tokens_policy
-
+- pipeline_default:ci_create_cross_project_pipeline
+- pipeline_default:ci_pipeline_bridge_status
+- pipeline_default:store_security_reports
+- pipeline_default:sync_security_reports_to_report_approval_rules
+- adjourned_project_deletion
 - admin_emails
 - create_github_webhook
 - design_management_new_version
 - elastic_batch_project_indexer
-- elastic_namespace_indexer
 - elastic_commit_indexer
-- elastic_indexer
 - elastic_full_index
+- elastic_indexer
+- elastic_namespace_indexer
 - elastic_namespace_rollout
 - export_csv
 - ldap_group_sync
 - new_epic
 - project_import_schedule
 - project_update_repository_storage
-- adjourned_project_deletion
-- rebase
 - refresh_license_compliance_checks
-- repository_update_mirror
 - repository_push_audit_event
-- epics:epics_update_epics_dates
+- repository_update_mirror

ee/spec/lib/gitlab/sidekiq_config_spec.rb

@@ -5,10 +5,10 @@ require 'spec_helper'
 describe Gitlab::SidekiqConfig do
   describe '.workers' do
     it 'includes EE workers' do
-      workers = described_class.workers
+      worker_classes = described_class.workers.map(&:klass)
 
-      expect(workers).to include(RepositoryUpdateMirrorWorker)
-      expect(workers).to include(LdapGroupSyncWorker)
+      expect(worker_classes).to include(RepositoryUpdateMirrorWorker)
+      expect(worker_classes).to include(LdapGroupSyncWorker)
     end
   end
 
@@ -41,4 +41,42 @@ describe Gitlab::SidekiqConfig do
       expect(queues).to include('unknown')
     end
   end
+
+  describe '.workers_for_all_queues_yml' do
+    it 'returns a tuple with EE workers second' do
+      expect(described_class.workers_for_all_queues_yml.second)
+        .to include(an_object_having_attributes(queue: 'repository_update_mirror'))
+    end
+  end
+
+  describe '.all_queues_yml_outdated?' do
+    before do
+      workers = [
+        LdapGroupSyncWorker,
+        RepositoryUpdateMirrorWorker
+      ].map { |worker| described_class::Worker.new(worker, ee: true) }
+
+      allow(described_class).to receive(:workers).and_return(workers)
+
+      allow(File).to receive(:read)
+                       .with(described_class::FOSS_QUEUE_CONFIG_PATH)
+                       .and_return(YAML.dump([]))
+    end
+
+    it 'returns true if the YAML file does not match the application code' do
+      allow(File).to receive(:read)
+                       .with(described_class::EE_QUEUE_CONFIG_PATH)
+                       .and_return(YAML.dump(['ldap_group_sync']))
+
+      expect(described_class.all_queues_yml_outdated?).to be(true)
+    end
+
+    it 'returns false if the YAML file matches the application code' do
+      allow(File).to receive(:read)
+                       .with(described_class::EE_QUEUE_CONFIG_PATH)
+                       .and_return(YAML.dump(%w[ldap_group_sync repository_update_mirror]))
+
+      expect(described_class.all_queues_yml_outdated?).to be(false)
+    end
+  end
 end

lib/gitlab/sidekiq_config.rb

@@ -4,6 +4,22 @@ require 'yaml'
 
 module Gitlab
   module SidekiqConfig
+    FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml'
+    EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml'
+
+    QUEUE_CONFIG_PATHS = [
+      FOSS_QUEUE_CONFIG_PATH,
+      (EE_QUEUE_CONFIG_PATH if Gitlab.ee?)
+    ].compact.freeze
+
+    # For queues that don't have explicit workers - default and mailers
+    DummyWorker = Struct.new(:queue)
+
+    DEFAULT_WORKERS = [
+      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default'), ee: false),
+      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers'), ee: false)
+    ].freeze
+
     class << self
       include Gitlab::SidekiqConfig::CliMethods
 
@@ -25,28 +41,46 @@ module Gitlab
 
       def workers
         @workers ||= begin
-          result = find_workers(Rails.root.join('app', 'workers'))
-          result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee?
+          result = []
+          result.concat(DEFAULT_WORKERS)
+          result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false))
+
+          if Gitlab.ee?
+            result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true))
+          end
+
           result
         end
       end
 
+      def workers_for_all_queues_yml
+        workers.partition(&:ee?).reverse.map(&:sort)
+      end
+
+      def all_queues_yml_outdated?
+        foss_workers, ee_workers = workers_for_all_queues_yml
+
+        return true if foss_workers != YAML.safe_load(File.read(FOSS_QUEUE_CONFIG_PATH))
+
+        Gitlab.ee? && ee_workers != YAML.safe_load(File.read(EE_QUEUE_CONFIG_PATH))
+      end
+
       private
 
-      def find_workers(root)
+      def find_workers(root, ee:)
         concerns = root.join('concerns').to_s
 
-        workers = Dir[root.join('**', '*.rb')]
+        Dir[root.join('**', '*.rb')]
           .reject { |path| path.start_with?(concerns) }
+          .map { |path| worker_from_path(path, root) }
+          .select { |worker| worker < Sidekiq::Worker }
+          .map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: ee) }
+      end
 
-        workers.map! do |path|
-          ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
-
-          ns.camelize.constantize
-        end
+      def worker_from_path(path, root)
+        ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
 
-        # Skip things that aren't workers
-        workers.select { |w| w < Sidekiq::Worker }
+        ns.camelize.constantize
       end
     end
   end

lib/gitlab/sidekiq_config/worker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqConfig
+    class Worker
+      include Comparable
+
+      attr_reader :klass
+      delegate :feature_category_not_owned?, :get_feature_category,
+               :get_worker_resource_boundary, :latency_sensitive_worker?,
+               :queue, :worker_has_external_dependencies?,
+               to: :klass
+
+      def initialize(klass, ee:)
+        @klass = klass
+        @ee = ee
+      end
+
+      def ee?
+        @ee
+      end
+
+      def ==(other)
+        to_yaml == case other
+                   when self.class
+                     other.to_yaml
+                   else
+                     other
+                   end
+      end
+
+      def <=>(other)
+        to_sort <=> other.to_sort
+      end
+
+      # Put namespaced queues first
+      def to_sort
+        [queue.include?(':') ? 0 : 1, queue]
+      end
+
+      # YAML representation
+      def encode_with(coder)
+        coder.represent_scalar(nil, to_yaml)
+      end
+
+      def to_yaml
+        queue
+      end
+    end
+  end
+end

lib/tasks/gitlab/sidekiq.rake

+# frozen_string_literal: true
+
+return if Rails.env.production?
+
+namespace :gitlab do
+  namespace :sidekiq do
+    namespace :all_queues_yml do
+      def write_yaml(path, object)
+        banner = <<~BANNER
+          # This file is generated automatically by
+          #   bin/rake gitlab:sidekiq:all_queues_yml:generate
+          #
+          # Do not edit it manually!
+        BANNER
+
+        File.write(path, banner + YAML.dump(object))
+      end
+
+      desc 'GitLab | Generate all_queues.yml based on worker definitions'
+      task generate: :environment do
+        foss_workers, ee_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml
+
+        write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, foss_workers)
+
+        if Gitlab.ee?
+          write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, ee_workers)
+        end
+      end
+
+      desc 'GitLab | Validate that all_queues.yml matches worker definitions'
+      task check: :environment do
+        if Gitlab::SidekiqConfig.all_queues_yml_outdated?
+          raise <<~MSG
+            Changes in worker queues found, please update the metadata by running:
+
+              bin/rake gitlab:sidekiq:all_queues_yml:generate
+
+            Then commit and push the changes from:
+
+            - #{Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH}
+            - #{Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH}
+
+          MSG
+        end
+      end
+    end
+  end
+end

lib/tasks/lint.rake

@@ -34,6 +34,7 @@ unless Rails.env.production?
         scss_lint
         gettext:lint
         lint:static_verification
+        gitlab:sidekiq:all_queues_yml:check
       ]
 
       if Gitlab.ee?

spec/lib/gitlab/sidekiq_config/worker_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+describe Gitlab::SidekiqConfig::Worker do
+  def worker_with_queue(queue)
+    described_class.new(double(queue: queue), ee: false)
+  end
+
+  describe '#ee?' do
+    it 'returns the EE status set on creation' do
+      expect(described_class.new(double, ee: true)).to be_ee
+      expect(described_class.new(double, ee: false)).not_to be_ee
+    end
+  end
+
+  describe '#==' do
+    def worker_with_yaml(yaml)
+      described_class.new(double, ee: false).tap do |worker|
+        allow(worker).to receive(:to_yaml).and_return(yaml)
+      end
+    end
+
+    it 'defines two workers as equal if their YAML representations are equal' do
+      expect(worker_with_yaml('a')).to eq(worker_with_yaml('a'))
+      expect(worker_with_yaml('a')).not_to eq(worker_with_yaml('b'))
+    end
+
+    it 'returns true when a worker is compared with its YAML representation' do
+      expect(worker_with_yaml('a')).to eq('a')
+      expect(worker_with_yaml(a: 1, b: 2)).to eq(a: 1, b: 2)
+    end
+  end
+
+  describe 'delegations' do
+    [
+      :feature_category_not_owned?, :get_feature_category,
+      :get_worker_resource_boundary, :latency_sensitive_worker?, :queue,
+      :worker_has_external_dependencies?
+    ].each do |meth|
+      it "delegates #{meth} to the worker class" do
+        worker = double
+
+        expect(worker).to receive(meth)
+
+        described_class.new(worker, ee: false).send(meth)
+      end
+    end
+  end
+
+  describe 'sorting' do
+    it 'sorts queues with a namespace before those without a namespace' do
+      namespaced_worker = worker_with_queue('namespace:queue')
+      plain_worker = worker_with_queue('a_queue')
+
+      expect([plain_worker, namespaced_worker].sort)
+        .to eq([namespaced_worker, plain_worker])
+    end
+
+    it 'sorts alphabetically by queue' do
+      workers = [
+        worker_with_queue('namespace:a'),
+        worker_with_queue('namespace:b'),
+        worker_with_queue('other_namespace:a'),
+        worker_with_queue('other_namespace:b'),
+        worker_with_queue('a'),
+        worker_with_queue('b')
+      ]
+
+      expect(workers.shuffle.sort).to eq(workers)
+    end
+  end
+
+  describe 'YAML encoding' do
+    it 'encodes the worker in YAML as a string of the queue' do
+      worker_a = worker_with_queue('a')
+      worker_b = worker_with_queue('b')
+
+      expect(YAML.dump(worker_a)).to eq(YAML.dump('a'))
+      expect(YAML.dump([worker_a, worker_b]))
+        .to eq(YAML.dump(%w[a b]))
+    end
+  end
+end

spec/lib/gitlab/sidekiq_config_spec.rb

@@ -5,10 +5,10 @@ require 'spec_helper'
 describe Gitlab::SidekiqConfig do
   describe '.workers' do
     it 'includes all workers' do
-      workers = described_class.workers
+      worker_classes = described_class.workers.map(&:klass)
 
-      expect(workers).to include(PostReceive)
-      expect(workers).to include(MergeWorker)
+      expect(worker_classes).to include(PostReceive)
+      expect(worker_classes).to include(MergeWorker)
     end
   end
 
@@ -44,4 +44,40 @@ describe Gitlab::SidekiqConfig do
       expect(queues).to include('unknown')
     end
   end
+
+  describe '.workers_for_all_queues_yml' do
+    it 'returns a tuple with FOSS workers first' do
+      expect(described_class.workers_for_all_queues_yml.first)
+        .to include(an_object_having_attributes(queue: 'post_receive'))
+    end
+  end
+
+  describe '.all_queues_yml_outdated?' do
+    before do
+      workers = [
+        PostReceive,
+        MergeWorker,
+        ProcessCommitWorker
+      ].map { |worker| described_class::Worker.new(worker, ee: false) }
+
+      allow(described_class).to receive(:workers).and_return(workers)
+      allow(Gitlab).to receive(:ee?).and_return(false)
+    end
+
+    it 'returns true if the YAML file does not match the application code' do
+      allow(File).to receive(:read)
+                       .with(described_class::FOSS_QUEUE_CONFIG_PATH)
+                       .and_return(YAML.dump(%w[post_receive merge]))
+
+      expect(described_class.all_queues_yml_outdated?).to be(true)
+    end
+
+    it 'returns false if the YAML file matches the application code' do
+      allow(File).to receive(:read)
+                       .with(described_class::FOSS_QUEUE_CONFIG_PATH)
+                       .and_return(YAML.dump(%w[merge post_receive process_commit]))
+
+      expect(described_class.all_queues_yml_outdated?).to be(false)
+    end
+  end
 end

spec/workers/every_sidekiq_worker_spec.rb

@@ -3,8 +3,12 @@
 require 'spec_helper'
 
 describe 'Every Sidekiq worker' do
+  let(:workers_without_defaults) do
+    Gitlab::SidekiqConfig.workers - Gitlab::SidekiqConfig::DEFAULT_WORKERS
+  end
+
   it 'does not use the default queue' do
-    expect(Gitlab::SidekiqConfig.workers.map(&:queue)).not_to include('default')
+    expect(workers_without_defaults.map(&:queue)).not_to include('default')
   end
 
   it 'uses the cronjob queue when the worker runs as a cronjob' do
@@ -45,7 +49,7 @@ describe 'Every Sidekiq worker' do
     # or explicitly be excluded with the `feature_category_not_owned!` annotation.
     # Please see doc/development/sidekiq_style_guide.md#Feature-Categorization for more details.
     it 'has a feature_category or feature_category_not_owned! attribute', :aggregate_failures do
-      Gitlab::SidekiqConfig.workers.each do |worker|
+      workers_without_defaults.each do |worker|
         expect(worker.get_feature_category).to be_a(Symbol), "expected #{worker.inspect} to declare a feature_category or feature_category_not_owned!"
       end
     end
@@ -54,7 +58,7 @@ describe 'Every Sidekiq worker' do
     # The category should match a value in `config/feature_categories.yml`.
     # Please see doc/development/sidekiq_style_guide.md#Feature-Categorization for more details.
     it 'has a feature_category that maps to a value in feature_categories.yml', :aggregate_failures do
-      workers_with_feature_categories = Gitlab::SidekiqConfig.workers
+      workers_with_feature_categories = workers_without_defaults
                   .select(&:get_feature_category)
                   .reject(&:feature_category_not_owned?)
 
@@ -69,7 +73,7 @@ describe 'Every Sidekiq worker' do
     # rather than scaling the hardware to meet the SLO. For this reason, memory-bound,
     # latency-sensitive jobs are explicitly discouraged and disabled.
     it 'is (exclusively) memory-bound or latency-sentitive, not both', :aggregate_failures do
-      latency_sensitive_workers = Gitlab::SidekiqConfig.workers
+      latency_sensitive_workers = workers_without_defaults
                   .select(&:latency_sensitive_worker?)
 
       latency_sensitive_workers.each do |worker|
@@ -86,7 +90,7 @@ describe 'Every Sidekiq worker' do
     # Please see doc/development/sidekiq_style_guide.md#Jobs-with-External-Dependencies for more
     # details.
     it 'has (exclusively) external dependencies or is latency-sentitive, not both', :aggregate_failures do
-      latency_sensitive_workers = Gitlab::SidekiqConfig.workers
+      latency_sensitive_workers = workers_without_defaults
                   .select(&:latency_sensitive_worker?)
 
       latency_sensitive_workers.each do |worker|