Merge branch 'stackprof-sidekiq' into 'master'

Defer stackprof signal trap when running in sidekiq

See merge request gitlab-org/gitlab!37988

config/initializers/stackprof.rb

@@ -8,12 +8,35 @@
 # * timeout profile after 30 seconds
 # * write to $TMPDIR/stackprof.$PID.$RAND.profile
 
-if Gitlab::Utils.to_boolean(ENV['STACKPROF_ENABLED'].to_s)
-  Gitlab::Cluster::LifecycleEvents.on_worker_start do
+module Gitlab
+  class StackProf
+    # this is a workaround for sidekiq, which defines its own SIGUSR2 handler.
+    # by defering to the sidekiq startup event, we get to set up our own
+    # handler late enough.
+    # see also: https://github.com/mperham/sidekiq/pull/4653
+    def self.install
       require 'stackprof'
       require 'tmpdir'
 
-    Gitlab::AppJsonLogger.info "stackprof: listening on SIGUSR2 signal"
+      if Gitlab::Runtime.sidekiq?
+        Sidekiq.configure_server do |config|
+          config.on :startup do
+            on_worker_start
+          end
+        end
+      else
+        Gitlab::Cluster::LifecycleEvents.on_worker_start do
+          on_worker_start
+        end
+      end
+    end
+
+    def self.on_worker_start
+      Gitlab::AppJsonLogger.info(
+        event: "stackprof",
+        message: "listening on SIGUSR2 signal",
+        pid: Process.pid
+      )
 
       # create a pipe in order to propagate signal out of the signal handler
       # see also: https://cr.yp.to/docs/selfpipe.html
@@ -38,7 +61,7 @@ if Gitlab::Utils.to_boolean(ENV['STACKPROF_ENABLED'].to_s)
           got_value = IO.select([read], nil, nil, current_timeout_s)
           read.getbyte if got_value
 
-        if StackProf.running?
+          if ::StackProf.running?
             stackprof_file_prefix = ENV['STACKPROF_FILE_PREFIX'] || Dir.tmpdir
             stackprof_out_file = "#{stackprof_file_prefix}/stackprof.#{Process.pid}.#{SecureRandom.hex(6)}.profile"
 
@@ -51,8 +74,8 @@ if Gitlab::Utils.to_boolean(ENV['STACKPROF_ENABLED'].to_s)
               timed_out: got_value.nil?
             )
 
-          StackProf.stop
-          StackProf.results(stackprof_out_file)
+            ::StackProf.stop
+            ::StackProf.results(stackprof_out_file)
             current_timeout_s = nil
           else
             Gitlab::AppJsonLogger.info(
@@ -61,7 +84,7 @@ if Gitlab::Utils.to_boolean(ENV['STACKPROF_ENABLED'].to_s)
               pid: Process.pid
             )
 
-          StackProf.start(
+            ::StackProf.start(
               mode: :cpu,
               raw: Gitlab::Utils.to_boolean(ENV['STACKPROF_RAW'] || 'true'),
               interval: ENV['STACKPROF_INTERVAL_US']&.to_i || 10_000
@@ -98,4 +121,9 @@ if Gitlab::Utils.to_boolean(ENV['STACKPROF_ENABLED'].to_s)
         write.write('.')
       end
     end
+  end
+end
+
+if Gitlab::Utils.to_boolean(ENV['STACKPROF_ENABLED'].to_s)
+  Gitlab::StackProf.install
 end

doc/development/performance.md

@@ -281,6 +281,10 @@ This can be done via `pkill -USR2 puma:`. The `:` disambiguates between `puma
 4.3.3.gitlab.2 ...` (the master process) from `puma: cluster worker 0: ...` (the
 worker processes), selecting the latter.
 
+For Sidekiq, the signal can be sent to the `sidekiq-cluster` process via `pkill
+-USR2 bin/sidekiq-cluster`, which will forward the signal to all Sidekiq
+children. Alternatively, you can also select a specific pid of interest.
+
 Production profiles can be especially noisy. It can be helpful to visualize them
 as a [flamegraph](https://github.com/brendangregg/FlameGraph). This can be done
 via: