Merge branch 'dm-application-worker' into 'master'

Add ApplicationWorker and make every worker include it

See merge request gitlab-org/gitlab-ce!15632

app/models/group.rb

@@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord'
 
 class Group < Namespace
   include Gitlab::ConfigHelper
+  include AfterCommitQueue
   include AccessRequestable
   include Avatarable
   include Referable

app/models/key.rb

@@ -2,6 +2,7 @@ require 'digest/md5'
 
 class Key < ActiveRecord::Base
   include Gitlab::CurrentSettings
+  include AfterCommitQueue
   include Sortable
 
   belongs_to :user

app/models/member.rb

 class Member < ActiveRecord::Base
+  include AfterCommitQueue
   include Sortable
   include Importable
   include Expirable

app/models/service.rb

@@ -211,7 +211,7 @@ class Service < ActiveRecord::Base
   def async_execute(data)
     return unless supported_events.include?(data[:object_kind])
 
-    Sidekiq::Client.enqueue(ProjectServiceWorker, id, data)
+    ProjectServiceWorker.perform_async(id, data)
   end
 
   def issue_tracker?

app/models/user.rb

@@ -7,6 +7,7 @@ class User < ActiveRecord::Base
   include Gitlab::ConfigHelper
   include Gitlab::CurrentSettings
   include Gitlab::SQL::Pattern
+  include AfterCommitQueue
   include Avatarable
   include Referable
   include Sortable
@@ -903,6 +904,7 @@ class User < ActiveRecord::Base
 
   def post_destroy_hook
     log_info("User \"#{name}\" (#{email})  was removed")
+
     system_hook_service.execute_hooks_for(self, :destroy)
   end
 

app/services/system_hooks_service.rb

 class SystemHooksService
   def execute_hooks_for(model, event)
-    execute_hooks(build_event_data(model, event))
+    data = build_event_data(model, event)
+
+    model.run_after_commit_or_now do
+      SystemHooksService.new.execute_hooks(data)
+    end
   end
 
   def execute_hooks(data, hooks_scope = :all)

app/services/web_hook_service.rb

@@ -63,7 +63,7 @@ class WebHookService
   end
 
   def async_execute
-    Sidekiq::Client.enqueue(WebHookWorker, hook.id, data, hook_name)
+    WebHookWorker.perform_async(hook.id, data, hook_name)
   end
 
   private

app/workers/admin_email_worker.rb

 class AdminEmailWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/authorized_projects_worker.rb

 class AuthorizedProjectsWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   # Schedules multiple jobs and waits for them to be completed.
   def self.bulk_perform_and_wait(args_list)
@@ -17,11 +16,6 @@ class AuthorizedProjectsWorker
     waiter.wait
   end
 
-  # Schedules multiple jobs to run in sidekiq without waiting for completion
-  def self.bulk_perform_async(args_list)
-    Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list)
-  end
-
   # Performs multiple jobs directly. Failed jobs will be put into sidekiq so
   # they can benefit from retries
   def self.bulk_perform_inline(args_list)

app/workers/background_migration_worker.rb

 class BackgroundMigrationWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
-
-  # Enqueues a number of jobs in bulk.
-  #
-  # The `jobs` argument should be an Array of Arrays, each sub-array must be in
-  # the form:
-  #
-  #     [migration-class, [arg1, arg2, ...]]
-  def self.perform_bulk(jobs)
-    Sidekiq::Client.push_bulk('class' => self,
-                              'queue' => sidekiq_options['queue'],
-                              'args' => jobs)
-  end
-
-  # Schedules multiple jobs in bulk, with a delay.
-  #
-  def self.perform_bulk_in(delay, jobs)
-    now = Time.now.to_i
-    schedule = now + delay.to_i
-
-    if schedule <= now
-      raise ArgumentError, 'The schedule time must be in the future!'
-    end
-
-    Sidekiq::Client.push_bulk('class' => self,
-                              'queue' => sidekiq_options['queue'],
-                              'args' => jobs,
-                              'at' => schedule)
-  end
+  include ApplicationWorker
 
   # Performs the background migration.
   #

app/workers/build_coverage_worker.rb

 class BuildCoverageWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   def perform(build_id)

app/workers/build_finished_worker.rb

 class BuildFinishedWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/build_hooks_worker.rb

 class BuildHooksWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :hooks

app/workers/build_queue_worker.rb

 class BuildQueueWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/build_success_worker.rb

 class BuildSuccessWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/build_trace_sections_worker.rb

 class BuildTraceSectionsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   def perform(build_id)

app/workers/cluster_install_app_worker.rb

 class ClusterInstallAppWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include ClusterQueue
   include ClusterApplications
 

app/workers/cluster_provision_worker.rb

 class ClusterProvisionWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include ClusterQueue
 
   def perform(cluster_id)

app/workers/cluster_wait_for_app_installation_worker.rb

 class ClusterWaitForAppInstallationWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include ClusterQueue
   include ClusterApplications
 

app/workers/concerns/application_worker.rb

+Sidekiq::Worker.extend ActiveSupport::Concern
+
+module ApplicationWorker
+  extend ActiveSupport::Concern
+
+  include Sidekiq::Worker
+
+  included do
+    sidekiq_options queue: base_queue_name
+  end
+
+  module ClassMethods
+    def base_queue_name
+      name
+        .sub(/\AGitlab::/, '')
+        .sub(/Worker\z/, '')
+        .underscore
+        .tr('/', '_')
+    end
+
+    def queue
+      get_sidekiq_options['queue'].to_s
+    end
+
+    def bulk_perform_async(args_list)
+      Sidekiq::Client.push_bulk('class' => self, 'args' => args_list)
+    end
+
+    def bulk_perform_in(delay, args_list)
+      now = Time.now.to_i
+      schedule = now + delay.to_i
+
+      if schedule <= now
+        raise ArgumentError, 'The schedule time must be in the future!'
+      end
+
+      Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule)
+    end
+  end
+end

app/workers/concerns/dedicated_sidekiq_queue.rb

-# Concern that sets the queue of a Sidekiq worker based on the worker's class
-# name/namespace.
-module DedicatedSidekiqQueue
-  extend ActiveSupport::Concern
-
-  included do
-    sidekiq_options queue: name.sub(/Worker\z/, '').underscore.tr('/', '_')
-  end
-end

app/workers/concerns/gitlab/github_import/object_importer.rb

@@ -8,7 +8,7 @@ module Gitlab
       extend ActiveSupport::Concern
 
       included do
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include ReschedulingMethods
         include NotifyUponDeath

app/workers/create_gpg_signature_worker.rb

 class CreateGpgSignatureWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(commit_sha, project_id)
     project = Project.find_by(id: project_id)

app/workers/create_pipeline_worker.rb

 class CreatePipelineWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :creation

app/workers/delete_merged_branches_worker.rb

 class DeleteMergedBranchesWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(project_id, user_id)
     begin

app/workers/delete_user_worker.rb

 class DeleteUserWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(current_user_id, delete_user_id, options = {})
     delete_user  = User.find(delete_user_id)

app/workers/email_receiver_worker.rb

 class EmailReceiverWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(raw)
     return unless Gitlab::IncomingEmail.enabled?

app/workers/emails_on_push_worker.rb

 class EmailsOnPushWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   attr_reader :email, :skip_premailer
 

app/workers/expire_build_artifacts_worker.rb

 class ExpireBuildArtifactsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform
@@ -8,6 +8,6 @@ class ExpireBuildArtifactsWorker
     build_ids = Ci::Build.with_expired_artifacts.pluck(:id)
     build_ids = build_ids.map { |build_id| [build_id] }
 
-    Sidekiq::Client.push_bulk('class' => ExpireBuildInstanceArtifactsWorker, 'args' => build_ids )
+    ExpireBuildInstanceArtifactsWorker.bulk_perform_async(build_ids)
   end
 end

app/workers/expire_build_instance_artifacts_worker.rb

 class ExpireBuildInstanceArtifactsWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(build_id)
     build = Ci::Build

app/workers/expire_job_cache_worker.rb

 class ExpireJobCacheWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :cache

app/workers/expire_pipeline_cache_worker.rb

 class ExpirePipelineCacheWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :cache

app/workers/git_garbage_collect_worker.rb

 class GitGarbageCollectWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include Gitlab::CurrentSettings
 
   sidekiq_options retry: false

app/workers/gitlab/github_import/advance_stage_worker.rb

@@ -7,7 +7,7 @@ module Gitlab
     # been completed this worker will advance the import process to the next
     # stage.
     class AdvanceStageWorker
-      include Sidekiq::Worker
+      include ApplicationWorker
 
       sidekiq_options queue: 'github_importer_advance_stage', dead: false
 

app/workers/gitlab/github_import/refresh_import_jid_worker.rb

@@ -3,7 +3,7 @@
 module Gitlab
   module GithubImport
     class RefreshImportJidWorker
-      include Sidekiq::Worker
+      include ApplicationWorker
       include GithubImport::Queue
 
       # The interval to schedule new instances of this job at.

app/workers/gitlab/github_import/stage/finish_import_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class FinishImportWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_base_data_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportBaseDataWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportIssuesAndDiffNotesWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_notes_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportNotesWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportPullRequestsWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab/github_import/stage/import_repository_worker.rb

@@ -4,7 +4,7 @@ module Gitlab
   module GithubImport
     module Stage
       class ImportRepositoryWorker
-        include Sidekiq::Worker
+        include ApplicationWorker
         include GithubImport::Queue
         include StageMethods
 

app/workers/gitlab_shell_worker.rb

 class GitlabShellWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include Gitlab::ShellAdapter
-  include DedicatedSidekiqQueue
 
   def perform(action, *arg)
     gitlab_shell.__send__(action, *arg) # rubocop:disable GitlabSecurity/PublicSend

app/workers/gitlab_usage_ping_worker.rb

 class GitlabUsagePingWorker
   LEASE_TIMEOUT = 86400
 
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/group_destroy_worker.rb

 class GroupDestroyWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
 
   def perform(group_id, user_id)

app/workers/import_export_project_cleanup_worker.rb

 class ImportExportProjectCleanupWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/invalid_gpg_signature_update_worker.rb

 class InvalidGpgSignatureUpdateWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(gpg_key_id)
     gpg_key = GpgKey.find_by(id: gpg_key_id)

app/workers/irker_worker.rb

@@ -2,8 +2,7 @@ require 'json'
 require 'socket'
 
 class IrkerWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(project_id, chans, colors, push_data, settings)
     project = Project.find(project_id)

app/workers/merge_worker.rb

 class MergeWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(merge_request_id, current_user_id, params)
     params = params.with_indifferent_access

app/workers/namespaceless_project_destroy_worker.rb

@@ -5,14 +5,9 @@
 # The worker will reject doing anything for projects that *do* have a
 # namespace. For those use ProjectDestroyWorker instead.
 class NamespacelessProjectDestroyWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
 
-  def self.bulk_perform_async(args_list)
-    Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list)
-  end
-
   def perform(project_id)
     begin
       project = Project.unscoped.find(project_id)

app/workers/new_issue_worker.rb

 class NewIssueWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include NewIssuable
 
   def perform(issue_id, user_id)

app/workers/new_merge_request_worker.rb

 class NewMergeRequestWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include NewIssuable
 
   def perform(merge_request_id, user_id)

app/workers/new_note_worker.rb

 class NewNoteWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   # Keep extra parameter to preserve backwards compatibility with
   # old `NewNoteWorker` jobs (can remove later)

app/workers/pages_worker.rb

 class PagesWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
 
   sidekiq_options queue: :pages, retry: false
 

app/workers/pipeline_hooks_worker.rb

 class PipelineHooksWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :hooks

app/workers/pipeline_metrics_worker.rb

 class PipelineMetricsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   def perform(pipeline_id)

app/workers/pipeline_notification_worker.rb

 class PipelineNotificationWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   def perform(pipeline_id, recipients = nil)

app/workers/pipeline_process_worker.rb

 class PipelineProcessWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/pipeline_schedule_worker.rb

 class PipelineScheduleWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/pipeline_success_worker.rb

 class PipelineSuccessWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/pipeline_update_worker.rb

 class PipelineUpdateWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/post_receive.rb

 class PostReceive
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(gl_repository, identifier, changes)
     project, is_wiki = Gitlab::GlRepository.parse(gl_repository)

app/workers/process_commit_worker.rb

@@ -5,8 +5,7 @@
 # Consider using an extra worker if you need to add any extra (and potentially
 # slow) processing of commits.
 class ProcessCommitWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   # project_id - The ID of the project this commit belongs to.
   # user_id - The ID of the user that pushed the commit.

app/workers/project_cache_worker.rb

 # Worker for updating any project specific caches.
 class ProjectCacheWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   LEASE_TIMEOUT = 15.minutes.to_i
 

app/workers/project_destroy_worker.rb

 class ProjectDestroyWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
 
   def perform(project_id, user_id, params)

app/workers/project_export_worker.rb

 class ProjectExportWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
 
   sidekiq_options retry: 3

app/workers/project_migrate_hashed_storage_worker.rb

 class ProjectMigrateHashedStorageWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   LEASE_TIMEOUT = 30.seconds.to_i
 

app/workers/project_service_worker.rb

 class ProjectServiceWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   sidekiq_options dead: false
 

app/workers/propagate_service_template_worker.rb

 # Worker for updating any project specific caches.
 class PropagateServiceTemplateWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   LEASE_TIMEOUT = 4.hours.to_i
 

app/workers/prune_old_events_worker.rb

 class PruneOldEventsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/reactive_caching_worker.rb

 class ReactiveCachingWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(class_name, id, *args)
     klass = begin

app/workers/remove_expired_group_links_worker.rb

 class RemoveExpiredGroupLinksWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/remove_expired_members_worker.rb

 class RemoveExpiredMembersWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/remove_old_web_hook_logs_worker.rb

 class RemoveOldWebHookLogsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   WEB_HOOK_LOG_LIFETIME = 2.days

app/workers/remove_unreferenced_lfs_objects_worker.rb

 class RemoveUnreferencedLfsObjectsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/repository_archive_cache_worker.rb

 class RepositoryArchiveCacheWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/repository_check/batch_worker.rb

 module RepositoryCheck
   class BatchWorker
-    include Sidekiq::Worker
+    include ApplicationWorker
     include CronjobQueue
 
     RUN_TIME = 3600

app/workers/repository_check/clear_worker.rb

 module RepositoryCheck
   class ClearWorker
-    include Sidekiq::Worker
+    include ApplicationWorker
     include RepositoryCheckQueue
 
     def perform

app/workers/repository_check/single_repository_worker.rb

 module RepositoryCheck
   class SingleRepositoryWorker
-    include Sidekiq::Worker
+    include ApplicationWorker
     include RepositoryCheckQueue
 
     def perform(project_id)

app/workers/repository_fork_worker.rb

 class RepositoryForkWorker
   ForkError = Class.new(StandardError)
 
-  include Sidekiq::Worker
+  include ApplicationWorker
   include Gitlab::ShellAdapter
-  include DedicatedSidekiqQueue
   include ProjectStartImport
 
   sidekiq_options status_expiration: StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION

app/workers/repository_import_worker.rb

 class RepositoryImportWorker
   ImportError = Class.new(StandardError)
 
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
   include ExceptionBacktrace
   include ProjectStartImport
 

app/workers/requests_profiles_worker.rb

 class RequestsProfilesWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/schedule_update_user_activity_worker.rb

 class ScheduleUpdateUserActivityWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform(batch_size = 500)

app/workers/stage_update_worker.rb

 class StageUpdateWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include PipelineQueue
 
   enqueue_in group: :processing

app/workers/storage_migrator_worker.rb

 class StorageMigratorWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   BATCH_SIZE = 100
 

app/workers/stuck_ci_jobs_worker.rb

 class StuckCiJobsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease'.freeze

app/workers/stuck_import_jobs_worker.rb

 class StuckImportJobsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   IMPORT_JOBS_EXPIRATION = 15.hours.to_i

app/workers/stuck_merge_jobs_worker.rb

 class StuckMergeJobsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/system_hook_push_worker.rb

 class SystemHookPushWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(push_data, hook_id)
     SystemHooksService.new.execute_hooks(push_data, hook_id)

app/workers/trending_projects_worker.rb

 class TrendingProjectsWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include CronjobQueue
 
   def perform

app/workers/update_merge_requests_worker.rb

 class UpdateMergeRequestsWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   LOG_TIME_THRESHOLD = 90 # seconds
 

app/workers/update_user_activity_worker.rb

 class UpdateUserActivityWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(pairs)
     pairs = cast_data(pairs)

app/workers/upload_checksum_worker.rb

 class UploadChecksumWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   def perform(upload_id)
     upload = Upload.find(upload_id)

app/workers/wait_for_cluster_creation_worker.rb

 class WaitForClusterCreationWorker
-  include Sidekiq::Worker
+  include ApplicationWorker
   include ClusterQueue
 
   def perform(cluster_id)

app/workers/web_hook_worker.rb

 class WebHookWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
+  include ApplicationWorker
 
   sidekiq_options retry: 4, dead: false
 

config/initializers/forbid_sidekiq_in_transactions.rb

@@ -13,13 +13,9 @@ module Sidekiq
 
     module ClassMethods
       module NoSchedulingFromTransactions
-        NESTING = ::Rails.env.test? ? 1 : 0
-
         %i(perform_async perform_at perform_in).each do |name|
           define_method(name) do |*args|
-            return super(*args) if Sidekiq::Worker.skip_transaction_check
-            return super(*args) unless ActiveRecord::Base.connection.open_transactions > NESTING
-
+            if !Sidekiq::Worker.skip_transaction_check && AfterCommitQueue.inside_transaction?
               raise <<-MSG.strip_heredoc
                 `#{self}.#{name}` cannot be called inside a transaction as this can lead to
                 race conditions when the worker runs before the transaction is committed and
@@ -28,6 +24,9 @@ module Sidekiq
                 Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead.
               MSG
             end
+
+            super(*args)
+          end
         end
       end
 

config/initializers/sidekiq.rb

@@ -64,13 +64,13 @@ end
 # The Sidekiq client API always adds the queue to the Sidekiq queue
 # list, but mail_room and gitlab-shell do not. This is only necessary
 # for monitoring.
-config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s)
-
 begin
+  queues = Gitlab::SidekiqConfig.worker_queues
+
   Sidekiq.redis do |conn|
     conn.pipelined do
-      config[:queues].each do |queue|
-        conn.sadd('queues', queue[0])
+      queues.each do |queue|
+        conn.sadd('queues', queue)
       end
     end
   end

db/post_migrate/20170627101016_schedule_event_migrations.rb

@@ -25,14 +25,14 @@ class ScheduleEventMigrations < ActiveRecord::Migration
         # We push multiple jobs at a time to reduce the time spent in
         # Sidekiq/Redis operations. We're using this buffer based approach so we
         # don't need to run additional queries for every range.
-        BackgroundMigrationWorker.perform_bulk(jobs)
+        BackgroundMigrationWorker.bulk_perform_async(jobs)
         jobs.clear
       end
 
       jobs << ['MigrateEventsToPushEventPayloads', [min, max]]
     end
 
-    BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+    BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty?
   end
 
   def down

db/post_migrate/20171005130944_schedule_create_gpg_key_subkeys_from_gpg_keys.rb

@@ -19,7 +19,7 @@ class ScheduleCreateGpgKeySubkeysFromGpgKeys < ActiveRecord::Migration
         [MIGRATION, [id]]
       end
 
-      BackgroundMigrationWorker.perform_bulk(jobs)
+      BackgroundMigrationWorker.bulk_perform_async(jobs)
     end
   end
 

doc/development/background_migrations.md

@@ -68,10 +68,10 @@ BackgroundMigrationWorker.perform_async('BackgroundMigrationClassName', [arg1, a
 ```
 
 Usually it's better to enqueue jobs in bulk, for this you can use
-`BackgroundMigrationWorker.perform_bulk`:
+`BackgroundMigrationWorker.bulk_perform_async`:
 
 ```ruby
-BackgroundMigrationWorker.perform_bulk(
+BackgroundMigrationWorker.bulk_perform_async(
   [['BackgroundMigrationClassName', [1]],
    ['BackgroundMigrationClassName', [2]]]
 )
@@ -85,13 +85,13 @@ updates. Removals in turn can be handled by simply defining foreign keys with
 cascading deletes.
 
 If you would like to schedule jobs in bulk with a delay, you can use
-`BackgroundMigrationWorker.perform_bulk_in`:
+`BackgroundMigrationWorker.bulk_perform_in`:
 
 ```ruby
 jobs = [['BackgroundMigrationClassName', [1]],
         ['BackgroundMigrationClassName', [2]]]
 
-BackgroundMigrationWorker.perform_bulk_in(5.minutes, jobs)
+BackgroundMigrationWorker.bulk_perform_in(5.minutes, jobs)
 ```
 
 ## Cleaning Up
@@ -201,7 +201,7 @@ class ScheduleExtractServicesUrl < ActiveRecord::Migration
         ['ExtractServicesUrl', [id]]
       end
 
-      BackgroundMigrationWorker.perform_bulk(jobs)
+      BackgroundMigrationWorker.bulk_perform_async(jobs)
     end
   end
 

doc/development/sidekiq_style_guide.md

@@ -3,6 +3,12 @@
 This document outlines various guidelines that should be followed when adding or
 modifying Sidekiq workers.
 
+## ApplicationWorker
+
+All workers should include `ApplicationWorker` instead of `Sidekiq::Worker`,
+which adds some convenience methods and automatically sets the queue based on
+the worker's name.
+
 ## Default Queue
 
 Use of the "default" queue is not allowed. Every worker should use a queue that
@@ -13,19 +19,10 @@ A list of all available queues can be found in `config/sidekiq_queues.yml`.
 
 ## Dedicated Queues
 
-Most workers should use their own queue. To ease this process a worker can
-include the `DedicatedSidekiqQueue` concern as follows:
-
-```ruby
-class ProcessSomethingWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
-end
-```
-
-This will set the queue name based on the class' name, minus the `Worker`
-suffix. In the above example this would lead to the queue being
-`process_something`.
+Most workers should use their own queue, which is automatically set based on the
+worker class name. For a worker named `ProcessSomethingWorker`, the queue name
+would be `process_something`. If you're not sure what a worker's queue name is,
+you can find it using `SomeWorker.queue`.
 
 In some cases multiple workers do use the same queue. For example, the various
 workers for updating CI pipelines all use the `pipeline` queue. Adding workers

lib/after_commit_queue.rb

@@ -6,12 +6,34 @@ module AfterCommitQueue
     after_rollback :_clear_after_commit_queue
   end
 
-  def run_after_commit(method = nil, &block)
-    _after_commit_queue << proc { self.send(method) } if method # rubocop:disable GitlabSecurity/PublicSend
+  def run_after_commit(&block)
     _after_commit_queue << block if block
+
+    true
+  end
+
+  def run_after_commit_or_now(&block)
+    if AfterCommitQueue.inside_transaction?
+      run_after_commit(&block)
+    else
+      instance_eval(&block)
+    end
+
     true
   end
 
+  def self.open_transactions_baseline
+    if ::Rails.env.test?
+      return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) }
+    end
+
+    0
+  end
+
+  def self.inside_transaction?
+    ActiveRecord::Base.connection.open_transactions > open_transactions_baseline
+  end
+
   protected
 
   def _run_after_commit_queue

lib/gitlab/database/migration_helpers.rb

@@ -703,14 +703,14 @@ into similar problems in the future (e.g. when new tables are created).
             # We push multiple jobs at a time to reduce the time spent in
             # Sidekiq/Redis operations. We're using this buffer based approach so we
             # don't need to run additional queries for every range.
-            BackgroundMigrationWorker.perform_bulk(jobs)
+            BackgroundMigrationWorker.bulk_perform_async(jobs)
             jobs.clear
           end
 
           jobs << [job_class_name, [start_id, end_id]]
         end
 
-        BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+        BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty?
       end
 
       # Queues background migration jobs for an entire table, batched by ID range.

lib/gitlab/sidekiq_config.rb

+require 'yaml'
+
+module Gitlab
+  module SidekiqConfig
+    def self.redis_queues
+      @redis_queues ||= Sidekiq::Queue.all.map(&:name)
+    end
+
+    # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
+    # of bundler/Rails context, so we cannot use any gem or Rails methods.
+    def self.config_queues(rails_path = Rails.root.to_s)
+      @config_queues ||= begin
+        config = YAML.load_file(File.join(rails_path, 'config', 'sidekiq_queues.yml'))
+        config[:queues].map(&:first)
+      end
+    end
+
+    def self.cron_workers
+      @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
+    end
+
+    def self.workers
+      @workers ||= find_workers(Rails.root.join('app', 'workers'))
+    end
+
+    def self.default_queues
+      [ActionMailer::DeliveryJob.queue_name, 'default']
+    end
+
+    def self.worker_queues
+      @worker_queues ||= (workers.map(&:queue) + default_queues).uniq
+    end
+
+    def self.find_workers(root)
+      concerns = root.join('concerns').to_s
+
+      workers = Dir[root.join('**', '*.rb')]
+        .reject { |path| path.start_with?(concerns) }
+
+      workers.map! do |path|
+        ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
+
+        ns.camelize.constantize
+      end
+
+      # Skip concerns
+      workers.select { |w| w < Sidekiq::Worker }
+    end
+  end
+end

spec/lib/gitlab/database/migration_helpers_spec.rb

@@ -942,8 +942,8 @@ describe Gitlab::Database::MigrationHelpers do
         end
 
         it 'queues jobs in groups of buffer size 1' do
-          expect(BackgroundMigrationWorker).to receive(:perform_bulk).with([['FooJob', [id1, id2]]])
-          expect(BackgroundMigrationWorker).to receive(:perform_bulk).with([['FooJob', [id3, id3]]])
+          expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]]])
+          expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id3, id3]]])
 
           model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
         end
@@ -960,7 +960,7 @@ describe Gitlab::Database::MigrationHelpers do
         end
 
         it 'queues jobs in bulk all at once (big buffer size)' do
-          expect(BackgroundMigrationWorker).to receive(:perform_bulk).with([['FooJob', [id1, id2]],
+          expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]],
                                                                                   ['FooJob', [id3, id3]]])
 
           model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)

spec/lib/gitlab/sidekiq_config_spec.rb

+require 'rails_helper'
+
+describe Gitlab::SidekiqConfig do
+  describe '.workers' do
+    it 'includes all workers' do
+      workers = described_class.workers
+
+      expect(workers).to include(PostReceive)
+      expect(workers).to include(MergeWorker)
+    end
+  end
+
+  describe '.worker_queues' do
+    it 'includes all queues' do
+      queues = described_class.worker_queues
+
+      expect(queues).to include('post_receive')
+      expect(queues).to include('merge')
+      expect(queues).to include('cronjob')
+      expect(queues).to include('mailers')
+      expect(queues).to include('default')
+    end
+  end
+end

spec/services/web_hook_service_spec.rb

@@ -146,7 +146,7 @@ describe WebHookService do
     let(:system_hook) { create(:system_hook) }
 
     it 'enqueue WebHookWorker' do
-      expect(Sidekiq::Client).to receive(:enqueue).with(WebHookWorker, project_hook.id, data, 'push_hooks')
+      expect(WebHookWorker).to receive(:perform_async).with(project_hook.id, data, 'push_hooks')
 
       described_class.new(project_hook, data, 'push_hooks').async_execute
     end

spec/workers/authorized_projects_worker_spec.rb

@@ -65,7 +65,6 @@ describe AuthorizedProjectsWorker do
       args_list = build_args_list(project.owner.id)
       push_bulk_args = {
         'class' => described_class,
-        'queue' => described_class.sidekiq_options['queue'],
         'args' => args_list
       }
 

spec/workers/background_migration_worker_spec.rb

@@ -10,35 +10,4 @@ describe BackgroundMigrationWorker, :sidekiq do
       described_class.new.perform('Foo', [10, 20])
     end
   end
-
-  describe '.perform_bulk' do
-    it 'enqueues background migrations in bulk' do
-      Sidekiq::Testing.fake! do
-        described_class.perform_bulk([['Foo', [1]], ['Foo', [2]]])
-
-        expect(described_class.jobs.count).to eq 2
-        expect(described_class.jobs).to all(include('enqueued_at'))
-      end
-    end
-  end
-
-  describe '.perform_bulk_in' do
-    context 'when delay is valid' do
-      it 'correctly schedules background migrations' do
-        Sidekiq::Testing.fake! do
-          described_class.perform_bulk_in(1.minute, [['Foo', [1]], ['Foo', [2]]])
-
-          expect(described_class.jobs.count).to eq 2
-          expect(described_class.jobs).to all(include('at'))
-        end
-      end
-    end
-
-    context 'when delay is invalid' do
-      it 'raises an ArgumentError exception' do
-        expect { described_class.perform_bulk_in(-60, [['Foo']]) }
-          .to raise_error(ArgumentError)
-      end
-    end
-  end
 end

spec/workers/concerns/application_worker_spec.rb

+require 'spec_helper'
+
+describe ApplicationWorker do
+  let(:worker) do
+    Class.new do
+      def self.name
+        'Gitlab::Foo::Bar::DummyWorker'
+      end
+
+      include ApplicationWorker
+    end
+  end
+
+  describe 'Sidekiq options' do
+    it 'sets the queue name based on the class name' do
+      expect(worker.sidekiq_options['queue']).to eq('foo_bar_dummy')
+    end
+  end
+
+  describe '.queue' do
+    it 'returns the queue name' do
+      worker.sidekiq_options queue: :some_queue
+
+      expect(worker.queue).to eq('some_queue')
+    end
+  end
+
+  describe '.bulk_perform_async' do
+    it 'enqueues jobs in bulk' do
+      Sidekiq::Testing.fake! do
+        worker.bulk_perform_async([['Foo', [1]], ['Foo', [2]]])
+
+        expect(worker.jobs.count).to eq 2
+        expect(worker.jobs).to all(include('enqueued_at'))
+      end
+    end
+  end
+
+  describe '.bulk_perform_in' do
+    context 'when delay is valid' do
+      it 'correctly schedules jobs' do
+        Sidekiq::Testing.fake! do
+          worker.bulk_perform_in(1.minute, [['Foo', [1]], ['Foo', [2]]])
+
+          expect(worker.jobs.count).to eq 2
+          expect(worker.jobs).to all(include('at'))
+        end
+      end
+    end
+
+    context 'when delay is invalid' do
+      it 'raises an ArgumentError exception' do
+        expect { worker.bulk_perform_in(-60, [['Foo']]) }
+          .to raise_error(ArgumentError)
+      end
+    end
+  end
+end

spec/workers/concerns/cluster_queue_spec.rb

@@ -3,7 +3,11 @@ require 'spec_helper'
 describe ClusterQueue do
   let(:worker) do
     Class.new do
-      include Sidekiq::Worker
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
       include ClusterQueue
     end
   end

spec/workers/concerns/cronjob_queue_spec.rb

@@ -3,7 +3,11 @@ require 'spec_helper'
 describe CronjobQueue do
   let(:worker) do
     Class.new do
-      include Sidekiq::Worker
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
       include CronjobQueue
     end
   end

spec/workers/concerns/dedicated_sidekiq_queue_spec.rb

-require 'spec_helper'
-
-describe DedicatedSidekiqQueue do
-  let(:worker) do
-    Class.new do
-      def self.name
-        'Foo::Bar::DummyWorker'
-      end
-
-      include Sidekiq::Worker
-      include DedicatedSidekiqQueue
-    end
-  end
-
-  describe 'queue names' do
-    it 'sets the queue name based on the class name' do
-      expect(worker.sidekiq_options['queue']).to eq('foo_bar_dummy')
-    end
-  end
-end

spec/workers/concerns/gitlab/github_import/object_importer_spec.rb

@@ -3,6 +3,10 @@ require 'spec_helper'
 describe Gitlab::GithubImport::ObjectImporter do
   let(:worker) do
     Class.new do
+      def self.name
+        'DummyWorker'
+      end
+
       include(Gitlab::GithubImport::ObjectImporter)
 
       def counter_name

spec/workers/concerns/gitlab/github_import/queue_spec.rb

@@ -3,7 +3,11 @@ require 'spec_helper'
 describe Gitlab::GithubImport::Queue do
   it 'sets the Sidekiq options for the worker' do
     worker = Class.new do
-      include Sidekiq::Worker
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
       include Gitlab::GithubImport::Queue
     end
 

spec/workers/concerns/pipeline_queue_spec.rb

@@ -3,7 +3,11 @@ require 'spec_helper'
 describe PipelineQueue do
   let(:worker) do
     Class.new do
-      include Sidekiq::Worker
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
       include PipelineQueue
     end
   end

spec/workers/concerns/repository_check_queue_spec.rb

@@ -3,7 +3,11 @@ require 'spec_helper'
 describe RepositoryCheckQueue do
   let(:worker) do
     Class.new do
-      include Sidekiq::Worker
+      def self.name
+        'DummyWorker'
+      end
+
+      include ApplicationWorker
       include RepositoryCheckQueue
     end
   end

spec/workers/every_sidekiq_worker_spec.rb

 require 'spec_helper'
 
 describe 'Every Sidekiq worker' do
-  let(:workers) do
-    root = Rails.root.join('app', 'workers')
-    concerns = root.join('concerns').to_s
-
-    workers = Dir[root.join('**', '*.rb')]
-      .reject { |path| path.start_with?(concerns) }
-
-    workers.map do |path|
-      ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
-
-      ns.camelize.constantize
-    end
+  it 'includes ApplicationWorker' do
+    expect(Gitlab::SidekiqConfig.workers).to all(include(ApplicationWorker))
   end
 
   it 'does not use the default queue' do
-    workers.each do |worker|
-      expect(worker.sidekiq_options['queue'].to_s).not_to eq('default')
-    end
+    expect(Gitlab::SidekiqConfig.workers.map(&:queue)).not_to include('default')
   end
 
   it 'uses the cronjob queue when the worker runs as a cronjob' do
-    cron_workers = Settings.cron_jobs
-      .map { |job_name, options| options['job_class'].constantize }
-      .to_set
-
-    workers.each do |worker|
-      next unless cron_workers.include?(worker)
-
-      expect(worker.sidekiq_options['queue'].to_s).to eq('cronjob')
-    end
+    expect(Gitlab::SidekiqConfig.cron_workers.map(&:queue)).to all(eq('cronjob'))
   end
 
   it 'defines the queue in the Sidekiq configuration file' do
-    config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s)
-    queue_names = config[:queues].map { |(queue, _)| queue }.to_set
+    config_queue_names = Gitlab::SidekiqConfig.config_queues.to_set
 
-    workers.each do |worker|
-      expect(queue_names).to include(worker.sidekiq_options['queue'].to_s)
-    end
+    expect(Gitlab::SidekiqConfig.worker_queues).to all(be_in(config_queue_names))
   end
 end