Merge branch 'annotate-workers-that-need-own-queue' into 'master'

Document workers that need their own queue

See merge request gitlab-org/gitlab!70069

app/workers/all_queues.yml

@@ -2703,7 +2703,7 @@
   :worker_name: ServiceDeskEmailReceiverWorker
   :feature_category: :service_desk
   :has_external_dependencies:
-  :urgency: :low
+  :urgency: :high
   :resource_boundary: :unknown
   :weight: 1
   :idempotent:

app/workers/email_receiver_worker.rb

@@ -11,7 +11,7 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker
   urgency :high
   weight 2
 
-  # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application
+  # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263
   tags :needs_own_queue
 
   attr_accessor :raw

app/workers/hashed_storage/migrator_worker.rb

@@ -11,8 +11,7 @@ module HashedStorage
     queue_namespace :hashed_storage
     feature_category :source_code_management
 
-    # Gitlab::HashedStorage::Migrator#migration_pending? depends on the
-    # queue size of this worker.
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/340629
     tags :needs_own_queue
 
     # @param [Integer] start initial ID of the batch

app/workers/hashed_storage/project_migrate_worker.rb

@@ -11,8 +11,7 @@ module HashedStorage
     queue_namespace :hashed_storage
     loggable_arguments 1
 
-    # Gitlab::HashedStorage::Migrator#migration_pending? depends on the
-    # queue size of this worker.
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/340629
     tags :needs_own_queue
 
     attr_reader :project_id

app/workers/hashed_storage/project_rollback_worker.rb

@@ -11,8 +11,7 @@ module HashedStorage
     queue_namespace :hashed_storage
     loggable_arguments 1
 
-    # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the
-    # queue size of this worker.
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/340629
     tags :needs_own_queue
 
     attr_reader :project_id

app/workers/hashed_storage/rollbacker_worker.rb

@@ -11,8 +11,7 @@ module HashedStorage
     queue_namespace :hashed_storage
     feature_category :source_code_management
 
-    # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the
-    # queue size of this worker.
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/340629
     tags :needs_own_queue
 
     # @param [Integer] start initial ID of the batch

app/workers/service_desk_email_receiver_worker.rb

@@ -6,9 +6,10 @@ class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Sca
   data_consistency :always
 
   feature_category :service_desk
+  urgency :high
   sidekiq_options retry: 3
 
-  # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application
+  # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263
   tags :needs_own_queue
 
   def should_perform?

doc/administration/operations/extra_sidekiq_routing.md

@@ -40,6 +40,8 @@ In `/etc/gitlab/gitlab.rb`:
 
 ```ruby
 sidekiq['routing_rules'] = [
+  # Do not re-route workers that require their own queue
+  ['tags=needs_own_queue', nil],
   # Route all non-CPU-bound workers that are high urgency to `high-urgency` queue
   ['resource_boundary!=cpu&urgency=high', 'high-urgency'],
   # Route all database, gitaly and global search workers that are throttled to `throttled` queue
@@ -164,3 +166,34 @@ with the migration to avoid losing jobs entirely, especially in a system with
 long queues of jobs. The migration can be done by following the migration steps
 mentioned in [Sidekiq job
 migration](../../raketasks/sidekiq_job_migration.md)
+
+### Workers that cannot be migrated
+
+Some workers cannot share a queue with other workers - typically because
+they check the size of their own queue - and so must be excluded from
+this process. We recommend excluding these from any further worker
+routing by adding a rule to keep them in their own queue, for example:
+
+```ruby
+sidekiq['routing_rules'] = [
+  ['tags=needs_own_queue', nil],
+  # ...
+]
+```
+
+These queues will also need to be included in at least one [Sidekiq
+queue group](extra_sidekiq_processes.md#start-multiple-processes).
+
+The following table shows the workers that should have their own queue:
+
+<!-- markdownlint-disable MD044 -->
+| Worker name | Queue name | GitLab issue |
+| --- | --- | --- |
+| EmailReceiverWorker | `email_receiver` | [gitlab-com/gl-infra/scalability#1263](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263) |
+| ServiceDeskEmailReceiverWorker | `service_desk_email_receiver` | [gitlab-com/gl-infra/scalability#1263](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263) |
+| ProjectImportScheduleWorker | `project_import_schedule` | [gitlab-org/gitlab#340630](https://gitlab.com/gitlab-org/gitlab/-/issues/340630) |
+| HashedStorage::MigratorWorker | `hashed_storage:hashed_storage_migrator` | [gitlab-org/gitlab#340629](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) |
+| HashedStorage::ProjectMigrateWorker | `hashed_storage:hashed_storage_project_migrate` | [gitlab-org/gitlab#340629](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) |
+| HashedStorage::ProjectRollbackWorker | `hashed_storage:hashed_storage_project_rollback` | [gitlab-org/gitlab#340629](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) |
+| HashedStorage::RollbackerWorker | `hashed_storage:hashed_storage_rollbacker` | [gitlab-org/gitlab#340629](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) |
+<!-- markdownlint-disable MD044 -->

ee/app/workers/project_import_schedule_worker.rb

@@ -14,7 +14,7 @@ class ProjectImportScheduleWorker
   loggable_arguments 1 # For the job waiter key
 
   # UpdateAllMirrorsWorker depends on the queue size of this worker:
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/325496#note_550866012
+  # https://gitlab.com/gitlab-org/gitlab/-/issues/340630
   tags :needs_own_queue
 
   def perform(project_id)