Merge branch 'speedscope' into 'master'

Adds flamegraph generation using Speedscope

See merge request gitlab-org/gitlab!53288

app/controllers/concerns/with_performance_bar.rb

@@ -20,12 +20,12 @@ module WithPerformanceBar
   end
 
   def cookie_or_default_value
-    return false unless Gitlab::PerformanceBar.enabled_for_user?(current_user)
-
-    if cookies[:perf_bar_enabled].present?
+    cookie_enabled = if cookies[:perf_bar_enabled].present?
                        cookies[:perf_bar_enabled] == 'true'
                      else
                        cookies[:perf_bar_enabled] = 'true' if Rails.env.development?
                      end
+
+    cookie_enabled && Gitlab::PerformanceBar.allowed_for_user?(current_user)
   end
 end

changelogs/unreleased/speedscope.yml

+---
+title: Add generating Speedscope flamegraphs for a request
+merge_request: 53288
+author:
+type: added

config/application.rb

@@ -173,6 +173,8 @@ module Gitlab
     config.assets.paths << Gemojione.images_path
     config.assets.paths << "#{config.root}/vendor/assets/fonts"
 
+    config.assets.paths << "#{config.root}/vendor/speedscope"
+
     config.assets.precompile << "application_utilities.css"
     config.assets.precompile << "application_utilities_dark.css"
     config.assets.precompile << "application_dark.css"

config/initializers/request_profiler.rb

@@ -2,4 +2,5 @@
 
 Rails.application.configure do |config|
   config.middleware.use(Gitlab::RequestProfiler::Middleware)
+  config.middleware.use(Gitlab::Middleware::Speedscope)
 end

doc/development/profiling.md

@@ -96,6 +96,16 @@ that builds on this to add some additional niceties, such as allowing
 configuration with a single YAML file for multiple URLs, and uploading of the
 profile and log output to S3.
 
+## Speedscope flamegraphs
+
+You can generate a flamegraph for a particular URL by adding the `performance_bar=flamegraph` parameter to the request.
+
+![Speedscope](img/speedscope_v13_12.png)
+
+More information about the views can be found in the [Speedscope docs](https://github.com/jlfwong/speedscope#views)
+
+This is enabled for all users that can access the performance bar.
+
 ## Sherlock
 
 Sherlock is a custom profiling tool built into GitLab. Sherlock is _only_

lib/api/helpers/performance_bar_helpers.rb

@@ -4,17 +4,17 @@ module API
   module Helpers
     module PerformanceBarHelpers
       def set_peek_enabled_for_current_request
-        Gitlab::SafeRequestStore.fetch(:peek_enabled) { perf_bar_cookie_enabled? && perf_bar_enabled_for_user? }
+        Gitlab::SafeRequestStore.fetch(:peek_enabled) { perf_bar_cookie_enabled? && perf_bar_allowed_for_user? }
       end
 
       def perf_bar_cookie_enabled?
         cookies[:perf_bar_enabled] == 'true'
       end
 
-      def perf_bar_enabled_for_user?
+      def perf_bar_allowed_for_user?
         # We cannot use `current_user` here because that method raises an exception when the user
         # is unauthorized and some API endpoints require that `current_user` is not called.
-        Gitlab::PerformanceBar.enabled_for_user?(find_user_from_sources)
+        Gitlab::PerformanceBar.allowed_for_user?(find_user_from_sources)
       end
     end
   end

lib/gitlab/middleware/speedscope.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Middleware
+    class Speedscope
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        request = Rack::Request.new(env)
+
+        if request.params['performance_bar'] == 'flamegraph' && Gitlab::PerformanceBar.allowed_for_user?(request.env['warden'].user)
+          body = nil
+
+          Gitlab::SafeRequestStore[:capturing_flamegraph] = true
+
+          require 'stackprof'
+
+          flamegraph = ::StackProf.run(
+            mode: :wall,
+            raw: true,
+            aggregate: false,
+            interval: (0.5 * 1000).to_i
+          ) do
+            _, _, body = @app.call(env)
+          end
+
+          path = env['PATH_INFO'].sub('//', '/')
+          body.close if body.respond_to?(:close)
+
+          return flamegraph(flamegraph, path)
+        end
+
+        @app.call(env)
+      end
+
+      def flamegraph(graph, path)
+        headers = { 'Content-Type' => 'text/html' }
+
+        html = <<~HTML
+          <!DOCTYPE html>
+          <html>
+            <head>
+              <style>
+                body { margin: 0; height: 100vh; }
+                #speedscope-iframe { width: 100%; height: 100%; border: none; }
+              </style>
+            </head>
+            <body>
+              <script type="text/javascript">
+                var graph = #{Gitlab::Json.generate(graph)};
+                var json = JSON.stringify(graph);
+                var blob = new Blob([json], { type: 'text/plain' });
+                var objUrl = encodeURIComponent(URL.createObjectURL(blob));
+                var iframe = document.createElement('IFRAME');
+                iframe.setAttribute('id', 'speedscope-iframe');
+                document.body.appendChild(iframe);
+                var iframeUrl = '#{Gitlab.config.gitlab.relative_url_root}/assets/speedscope/index.html#profileURL=' + objUrl + '&title=' + 'Flamegraph for #{CGI.escape(path)}';
+                iframe.setAttribute('src', iframeUrl);
+              </script>
+            </body>
+          </html>
+        HTML
+
+        [200, headers, [html]]
+      end
+    end
+  end
+end

lib/gitlab/performance_bar.rb

@@ -7,10 +7,10 @@ module Gitlab
     EXPIRY_TIME_L2_CACHE = 5.minutes
 
     def self.enabled_for_request?
-      Gitlab::SafeRequestStore[:peek_enabled]
+      !Gitlab::SafeRequestStore[:capturing_flamegraph] && Gitlab::SafeRequestStore[:peek_enabled]
     end
 
-    def self.enabled_for_user?(user = nil)
+    def self.allowed_for_user?(user = nil)
       return true if Rails.env.development?
       return true if user&.admin?
       return false unless user && allowed_group_id

spec/lib/gitlab/middleware/speedscope_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'stackprof'
+
+RSpec.describe Gitlab::Middleware::Speedscope do
+  let(:app) { proc { |env| [200, { 'Content-Type' => 'text/plain' }, ['Hello world!']] } }
+  let(:middleware) { described_class.new(app) }
+
+  describe '#call' do
+    shared_examples 'returns original response' do
+      it 'returns original response' do
+        expect(StackProf).not_to receive(:run)
+
+        status, headers, body = middleware.call(env)
+
+        expect(status).to eq(200)
+        expect(headers).to eq({ 'Content-Type' => 'text/plain' })
+        expect(body.first).to eq('Hello world!')
+      end
+    end
+
+    context 'when flamegraph is not requested' do
+      let(:env) { Rack::MockRequest.env_for('/') }
+
+      it_behaves_like 'returns original response'
+    end
+
+    context 'when flamegraph requested' do
+      let(:env) { Rack::MockRequest.env_for('/', params: { 'performance_bar' => 'flamegraph' }) }
+
+      before do
+        allow(env).to receive(:[]).and_call_original
+      end
+
+      context 'when user is not allowed' do
+        before do
+          allow(env).to receive(:[]).with('warden').and_return(double('Warden', user: create(:user)))
+        end
+
+        it_behaves_like 'returns original response'
+      end
+
+      context 'when user is allowed' do
+        before do
+          allow(env).to receive(:[]).with('warden').and_return(double('Warden', user: create(:admin)))
+        end
+
+        it 'runs StackProf and returns a flamegraph' do
+          expect(StackProf).to receive(:run).and_call_original
+
+          status, headers, body = middleware.call(env)
+
+          expect(status).to eq(200)
+          expect(headers).to eq({ 'Content-Type' => 'text/html' })
+          expect(body.first).to include('speedscope-iframe')
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/performance_bar_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe Gitlab::PerformanceBar do
   it { expect(described_class.l1_cache_backend).to eq(Gitlab::ProcessMemoryCache.cache_backend) }
   it { expect(described_class.l2_cache_backend).to eq(Rails.cache) }
 
-  describe '.enabled_for_user?' do
+  describe '.allowed_for_user?' do
     let(:user) { create(:user) }
 
     before do
@@ -14,24 +14,24 @@ RSpec.describe Gitlab::PerformanceBar do
     end
 
     it 'returns false when given user is nil' do
-      expect(described_class.enabled_for_user?(nil)).to be_falsy
+      expect(described_class.allowed_for_user?(nil)).to be_falsy
     end
 
     it 'returns true when given user is an admin' do
       user = build_stubbed(:user, :admin)
 
-      expect(described_class.enabled_for_user?(user)).to be_truthy
+      expect(described_class.allowed_for_user?(user)).to be_truthy
     end
 
     it 'returns false when allowed_group_id is nil' do
       expect(described_class).to receive(:allowed_group_id).and_return(nil)
 
-      expect(described_class.enabled_for_user?(user)).to be_falsy
+      expect(described_class.allowed_for_user?(user)).to be_falsy
     end
 
     context 'when allowed group ID does not exist' do
       it 'returns false' do
-        expect(described_class.enabled_for_user?(user)).to be_falsy
+        expect(described_class.allowed_for_user?(user)).to be_falsy
       end
     end
 
@@ -44,15 +44,15 @@ RSpec.describe Gitlab::PerformanceBar do
 
       context 'when user is not a member of the allowed group' do
         it 'returns false' do
-          expect(described_class.enabled_for_user?(user)).to be_falsy
+          expect(described_class.allowed_for_user?(user)).to be_falsy
         end
 
         context 'caching of allowed user IDs' do
-          subject { described_class.enabled_for_user?(user) }
+          subject { described_class.allowed_for_user?(user) }
 
           before do
             # Warm the caches
-            described_class.enabled_for_user?(user)
+            described_class.allowed_for_user?(user)
           end
 
           it_behaves_like 'allowed user IDs are cached'
@@ -65,15 +65,15 @@ RSpec.describe Gitlab::PerformanceBar do
         end
 
         it 'returns true' do
-          expect(described_class.enabled_for_user?(user)).to be_truthy
+          expect(described_class.allowed_for_user?(user)).to be_truthy
         end
 
         context 'caching of allowed user IDs' do
-          subject { described_class.enabled_for_user?(user) }
+          subject { described_class.allowed_for_user?(user) }
 
           before do
             # Warm the caches
-            described_class.enabled_for_user?(user)
+            described_class.allowed_for_user?(user)
           end
 
           it_behaves_like 'allowed user IDs are cached'
@@ -91,7 +91,7 @@ RSpec.describe Gitlab::PerformanceBar do
       end
 
       it 'returns the nested group' do
-        expect(described_class.enabled_for_user?(user)).to be_truthy
+        expect(described_class.allowed_for_user?(user)).to be_truthy
       end
     end
 
@@ -101,7 +101,7 @@ RSpec.describe Gitlab::PerformanceBar do
       end
 
       it 'returns false' do
-        expect(described_class.enabled_for_user?(user)).to be_falsy
+        expect(described_class.allowed_for_user?(user)).to be_falsy
       end
     end
   end

vendor/speedscope/speedscope/LICENSE

+MIT License
+
+Copyright (c) 2018 Jamie Wong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/speedscope/speedscope/README

+This is a self-contained release of https://github.com/jlfwong/speedscope.
+To use it, open index.html in Chrome or Firefox.

vendor/speedscope/speedscope/file-format-schema.json

+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "definitions": {
+        "CloseFrameEvent": {
+            "properties": {
+                "at": {
+                    "title": "at",
+                    "type": "number"
+                },
+                "frame": {
+                    "title": "frame",
+                    "type": "number"
+                },
+                "type": {
+                    "enum": [
+                        "C"
+                    ],
+                    "title": "type",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "at",
+                "frame",
+                "type"
+            ],
+            "title": "CloseFrameEvent",
+            "type": "object"
+        },
+        "FileFormat.EventType": {
+            "enum": [
+                "C",
+                "O"
+            ],
+            "title": "FileFormat.EventType",
+            "type": "string"
+        },
+        "FileFormat.EventedProfile": {
+            "properties": {
+                "endValue": {
+                    "title": "endValue",
+                    "type": "number"
+                },
+                "events": {
+                    "items": {
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/OpenFrameEvent"
+                            },
+                            {
+                                "$ref": "#/definitions/CloseFrameEvent"
+                            }
+                        ]
+                    },
+                    "title": "events",
+                    "type": "array"
+                },
+                "name": {
+                    "title": "name",
+                    "type": "string"
+                },
+                "startValue": {
+                    "title": "startValue",
+                    "type": "number"
+                },
+                "type": {
+                    "enum": [
+                        "evented"
+                    ],
+                    "title": "type",
+                    "type": "string"
+                },
+                "unit": {
+                    "$ref": "#/definitions/FileFormat.ValueUnit",
+                    "title": "unit"
+                }
+            },
+            "required": [
+                "endValue",
+                "events",
+                "name",
+                "startValue",
+                "type",
+                "unit"
+            ],
+            "title": "FileFormat.EventedProfile",
+            "type": "object"
+        },
+        "FileFormat.File": {
+            "properties": {
+                "$schema": {
+                    "enum": [
+                        "https://www.speedscope.app/file-format-schema.json"
+                    ],
+                    "title": "$schema",
+                    "type": "string"
+                },
+                "activeProfileIndex": {
+                    "title": "activeProfileIndex",
+                    "type": "number"
+                },
+                "exporter": {
+                    "title": "exporter",
+                    "type": "string"
+                },
+                "name": {
+                    "title": "name",
+                    "type": "string"
+                },
+                "profiles": {
+                    "items": {
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/FileFormat.EventedProfile"
+                            },
+                            {
+                                "$ref": "#/definitions/FileFormat.SampledProfile"
+                            }
+                        ]
+                    },
+                    "title": "profiles",
+                    "type": "array"
+                },
+                "shared": {
+                    "properties": {
+                        "frames": {
+                            "items": {
+                                "$ref": "#/definitions/FileFormat.Frame"
+                            },
+                            "title": "frames",
+                            "type": "array"
+                        }
+                    },
+                    "required": [
+                        "frames"
+                    ],
+                    "title": "shared",
+                    "type": "object"
+                }
+            },
+            "required": [
+                "$schema",
+                "profiles",
+                "shared"
+            ],
+            "title": "FileFormat.File",
+            "type": "object"
+        },
+        "FileFormat.Frame": {
+            "properties": {
+                "col": {
+                    "title": "col",
+                    "type": "number"
+                },
+                "file": {
+                    "title": "file",
+                    "type": "string"
+                },
+                "line": {
+                    "title": "line",
+                    "type": "number"
+                },
+                "name": {
+                    "title": "name",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "name"
+            ],
+            "title": "FileFormat.Frame",
+            "type": "object"
+        },
+        "FileFormat.IProfile": {
+            "properties": {
+                "type": {
+                    "$ref": "#/definitions/FileFormat.ProfileType",
+                    "title": "type"
+                }
+            },
+            "required": [
+                "type"
+            ],
+            "title": "FileFormat.IProfile",
+            "type": "object"
+        },
+        "FileFormat.Profile": {
+            "anyOf": [
+                {
+                    "$ref": "#/definitions/FileFormat.EventedProfile"
+                },
+                {
+                    "$ref": "#/definitions/FileFormat.SampledProfile"
+                }
+            ]
+        },
+        "FileFormat.ProfileType": {
+            "enum": [
+                "evented",
+                "sampled"
+            ],
+            "title": "FileFormat.ProfileType",
+            "type": "string"
+        },
+        "FileFormat.SampledProfile": {
+            "properties": {
+                "endValue": {
+                    "title": "endValue",
+                    "type": "number"
+                },
+                "name": {
+                    "title": "name",
+                    "type": "string"
+                },
+                "samples": {
+                    "items": {
+                        "items": {
+                            "type": "number"
+                        },
+                        "type": "array"
+                    },
+                    "title": "samples",
+                    "type": "array"
+                },
+                "startValue": {
+                    "title": "startValue",
+                    "type": "number"
+                },
+                "type": {
+                    "enum": [
+                        "sampled"
+                    ],
+                    "title": "type",
+                    "type": "string"
+                },
+                "unit": {
+                    "$ref": "#/definitions/FileFormat.ValueUnit",
+                    "title": "unit"
+                },
+                "weights": {
+                    "items": {
+                        "type": "number"
+                    },
+                    "title": "weights",
+                    "type": "array"
+                }
+            },
+            "required": [
+                "endValue",
+                "name",
+                "samples",
+                "startValue",
+                "type",
+                "unit",
+                "weights"
+            ],
+            "title": "FileFormat.SampledProfile",
+            "type": "object"
+        },
+        "FileFormat.ValueUnit": {
+            "enum": [
+                "bytes",
+                "microseconds",
+                "milliseconds",
+                "nanoseconds",
+                "none",
+                "seconds"
+            ],
+            "title": "FileFormat.ValueUnit",
+            "type": "string"
+        },
+        "IEvent": {
+            "properties": {
+                "at": {
+                    "title": "at",
+                    "type": "number"
+                },
+                "type": {
+                    "$ref": "#/definitions/FileFormat.EventType",
+                    "title": "type"
+                }
+            },
+            "required": [
+                "at",
+                "type"
+            ],
+            "title": "IEvent",
+            "type": "object"
+        },
+        "OpenFrameEvent": {
+            "properties": {
+                "at": {
+                    "title": "at",
+                    "type": "number"
+                },
+                "frame": {
+                    "title": "frame",
+                    "type": "number"
+                },
+                "type": {
+                    "enum": [
+                        "O"
+                    ],
+                    "title": "type",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "at",
+                "frame",
+                "type"
+            ],
+            "title": "OpenFrameEvent",
+            "type": "object"
+        },
+        "SampledStack": {
+            "items": {
+                "type": "number"
+            },
+            "type": "array"
+        }
+    },
+    "$ref": "#/definitions/FileFormat.File"
+}

vendor/speedscope/speedscope/index.html

+<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta http-equiv="X-UA-Compatible" content="ie=edge"><title>speedscope</title><link href="https://fonts.googleapis.com/css?family=Source+Code+Pro" rel="stylesheet"><script></script><link rel="stylesheet" href="reset.8c46b7a1.css"><link rel="icon" type="image/png" sizes="32x32" href="favicon-32x32.bc503437.png"><link rel="icon" type="image/png" sizes="16x16" href="favicon-16x16.f74b3187.png"></head><body> <script src="speedscope.026f36b0.js"></script>
+</body></html>
\ No newline at end of file

vendor/speedscope/speedscope/release.txt

+speedscope@1.13.0
+Sun Feb 14 23:36:53 PST 2021
+03a5104317e030e07303de006a85464ed1fcdea0

vendor/speedscope/speedscope/reset.8c46b7a1.css

+a,abbr,acronym,address,applet,article,aside,audio,b,big,blockquote,body,canvas,caption,center,cite,code,dd,del,details,dfn,div,dl,dt,em,embed,fieldset,figcaption,figure,footer,form,h1,h2,h3,h4,h5,h6,header,hgroup,html,i,iframe,img,ins,kbd,label,legend,li,mark,menu,nav,object,ol,output,p,pre,q,ruby,s,samp,section,small,span,strike,strong,sub,summary,sup,table,tbody,td,tfoot,th,thead,time,tr,tt,u,ul,var,video{margin:0;padding:0;border:0;font-size:100%;font:inherit;vertical-align:initial}article,aside,details,figcaption,figure,footer,header,hgroup,menu,nav,section{display:block}body{line-height:1}ol,ul{list-style:none}blockquote,q{quotes:none}blockquote:after,blockquote:before,q:after,q:before{content:"";content:none}table{border-collapse:collapse;border-spacing:0}html{overflow:hidden}body,html{height:100%}body{overflow:auto}
+/*# sourceMappingURL=reset.8c46b7a1.css.map */
\ No newline at end of file

vendor/speedscope/speedscope/reset.8c46b7a1.css.map

+{"version":3,"sources":["reset.css"],"names":[],"mappings":"AAIA,2ZAaC,QAAS,CACT,SAAU,CACV,QAAS,CACT,cAAe,CACf,YAAa,CACb,sBACD,CAEA,8EAEC,aACD,CACA,KACC,aACD,CACA,MACC,eACD,CACA,aACC,WACD,CACA,oDAEC,UAAW,CACX,YACD,CACA,MACC,wBAAyB,CACzB,gBACD,CAIA,KACI,eAEJ,CACA,UAFI,WAKJ,CAHA,KAEI,aACJ","file":"reset.8c46b7a1.css","sourceRoot":"../../assets","sourcesContent":["/* http://meyerweb.com/eric/tools/css/reset/\n   v2.0 | 20110126\n   License: none (public domain)\n*/\nhtml, body, div, span, applet, object, iframe,\nh1, h2, h3, h4, h5, h6, p, blockquote, pre,\na, abbr, acronym, address, big, cite, code,\ndel, dfn, em, img, ins, kbd, q, s, samp,\nsmall, strike, strong, sub, sup, tt, var,\nb, u, i, center,\ndl, dt, dd, ol, ul, li,\nfieldset, form, label, legend,\ntable, caption, tbody, tfoot, thead, tr, th, td,\narticle, aside, canvas, details, embed,\nfigure, figcaption, footer, header, hgroup,\nmenu, nav, output, ruby, section, summary,\ntime, mark, audio, video {\n\tmargin: 0;\n\tpadding: 0;\n\tborder: 0;\n\tfont-size: 100%;\n\tfont: inherit;\n\tvertical-align: baseline;\n}\n/* HTML5 display-role reset for older browsers */\narticle, aside, details, figcaption, figure,\nfooter, header, hgroup, menu, nav, section {\n\tdisplay: block;\n}\nbody {\n\tline-height: 1;\n}\nol, ul {\n\tlist-style: none;\n}\nblockquote, q {\n\tquotes: none;\n}\nblockquote:before, blockquote:after,\nq:before, q:after {\n\tcontent: '';\n\tcontent: none;\n}\ntable {\n\tborder-collapse: collapse;\n\tborder-spacing: 0;\n}\n\n/* Prevent overscrolling */\n/* https://stackoverflow.com/a/17899813 */\nhtml {\n    overflow: hidden;\n    height: 100%;\n}\nbody {\n    height: 100%;\n    overflow: auto;\n}"]}
\ No newline at end of file