Merge branch 'acunskis-spec-runtimes' into 'master'

Add task to report on long running spec files

See merge request gitlab-org/gitlab!78221

qa/Rakefile

 # frozen_string_literal: true
 # rubocop:disable Rails/RakeEnvironment
 
-Dir['tasks/*.rake'].each { |file| load file }
+require_relative "qa"
 
-require_relative 'qa/tools/revoke_all_personal_access_tokens'
-require_relative 'qa/tools/delete_subgroups'
-require_relative 'qa/tools/generate_perf_testdata'
-require_relative 'qa/tools/delete_test_ssh_keys'
-require_relative 'qa/tools/initialize_gitlab_auth'
-require_relative 'qa/tools/delete_projects'
+Dir['tasks/*.rake'].each { |file| load file }
 
 desc "Revokes all personal access tokens"
 task :revoke_personal_access_tokens do

qa/qa/tools/delete_projects.rb

 # frozen_string_literal: true
 
-require_relative '../../qa'
-
 # This script deletes all projects directly under a group specified by ENV['TOP_LEVEL_GROUP_NAME']
 # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
 # Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group')

qa/qa/tools/delete_subgroups.rb

 # frozen_string_literal: true
 
-require_relative '../../qa'
-
 # This script deletes all subgroups of a group specified by ENV['TOP_LEVEL_GROUP_NAME']
 # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
 # Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group')

qa/qa/tools/delete_test_ssh_keys.rb

 # frozen_string_literal: true
 
-require_relative '../../qa'
-
 # This script deletes all selected test ssh keys for a specific user
 # Keys can be selected by a string matching part of the key's title and by created date
 #   - Specify `title_portion` to delete only keys that include the string provided

qa/qa/tools/generate_perf_testdata.rb

 # frozen_string_literal: true
 
 require 'yaml'
-require_relative '../../qa'
+
 # This script generates testdata for Performance Testing.
 # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
 # This job creates a urls.txt which contains a hash of all the URLs needed for Performance Testing

qa/qa/tools/initialize_gitlab_auth.rb

 # frozen_string_literal: true
 
-require_relative '../../qa'
-
 module QA
   module Tools
     # Task to set default password from Runtime::Env.default_password if not set already

qa/qa/tools/long_running_spec_reporter.rb

+# frozen_string_literal: true
+
+require "fog/google"
+require "slack-notifier"
+
+module QA
+  module Tools
+    class LongRunningSpecReporter
+      extend SingleForwardable
+
+      SLACK_CHANNEL = "#quality-reports"
+      PROJECT = "gitlab-qa-resources"
+      BUCKET = "knapsack-reports"
+      REPORT_NAME = "ee-instance-parallel.json"
+      RUNTIME_THRESHOLD = 300
+
+      def_delegator :new, :execute
+
+      # Find and report specs exceeding runtime threshold
+      #
+      # @return [void]
+      def execute
+        return puts("No long running specs detected, all good!") if long_running_specs.empty?
+
+        specs = long_running_specs.map { |k, v| "#{k}: #{(v / 60).round(2)} minutes" }.join("\n")
+        average = mean_runtime < 60 ? "#{mean_runtime.round(0)} seconds" : "#{(mean_runtime / 60).round(2)} minutes"
+        msg = <<~MSG
+          Following spec files are exceeding #{RUNTIME_THRESHOLD / 60} minute runtime threshold!
+          Current average spec runtime: #{average}.
+        MSG
+
+        puts("#{msg}\n#{specs}")
+        notifier.post(icon_emoji: ":time-out:", text: "#{msg}\n```#{specs}```")
+      end
+
+      private
+
+      # Average runtime of spec files
+      #
+      # @return [Number]
+      def mean_runtime
+        @mean_runtime ||= latest_report.values
+          .select { |v| v < RUNTIME_THRESHOLD }
+          .yield_self { |runtimes| runtimes.sum(0.0) / runtimes.length }
+      end
+
+      # Spec files exceeding runtime threshold
+      #
+      # @return [Hash]
+      def long_running_specs
+        @long_running_specs ||= latest_report.select { |k, v| v > RUNTIME_THRESHOLD }
+      end
+
+      # Latest knapsack report
+      #
+      # @return [Hash]
+      def latest_report
+        @latest_report ||= JSON.parse(client.get_object(BUCKET, REPORT_NAME)[:body])
+      end
+
+      # Slack notifier
+      #
+      # @return [Slack::Notifier]
+      def notifier
+        @notifier ||= Slack::Notifier.new(
+          slack_webhook_url,
+          channel: SLACK_CHANNEL,
+          username: "Spec Runtime Report"
+        )
+      end
+
+      # GCS client
+      #
+      # @return [Fog::Storage::GoogleJSON]
+      def client
+        @client ||= Fog::Storage::Google.new(
+          google_project: PROJECT,
+          **(File.exist?(gcs_json) ? { google_json_key_location: gcs_json } : { google_json_key_string: gcs_json })
+        )
+      end
+
+      # Slack webhook url
+      #
+      # @return [String]
+      def slack_webhook_url
+        @slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable")
+      end
+
+      # GCS credentials json
+      #
+      # @return [Hash]
+      def gcs_json
+        ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("Missing QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable!")
+      end
+    end
+  end
+end

qa/qa/tools/reliable_report.rb

 # frozen_string_literal: true
 
-require_relative "../../qa"
-
 require "influxdb-client"
 require "terminal-table"
 require "slack-notifier"

qa/qa/tools/revoke_all_personal_access_tokens.rb

 # frozen_string_literal: true
 
-require_relative '../../qa'
 require 'net/protocol'
+
 # This script revokes all personal access tokens with the name of 'api-test-token' on the host specified by GITLAB_ADDRESS
 # Required environment variables: GITLAB_USERNAME, GITLAB_PASSWORD and GITLAB_ADDRESS
 # Run `rake revoke_personal_access_tokens`

qa/spec/tools/long_running_spec_reporter_spec.rb

+# frozen_string_literal: true
+
+RSpec.describe QA::Tools::LongRunningSpecReporter do
+  include QA::Support::Helpers::StubEnv
+
+  subject(:reporter) { described_class.execute }
+
+  let(:gcs_client) { double("Fog::Storage::GoogleJSON", get_object: report) }
+  let(:slack_notifier) { double("Slack::Notifier", post: nil) }
+
+  before do
+    stub_env("SLACK_WEBHOOK", "slack_url")
+    stub_env("QA_KNAPSACK_REPORT_GCS_CREDENTIALS", "gcs_json")
+
+    allow(Fog::Storage::Google).to receive(:new)
+      .with(google_project: "gitlab-qa-resources", google_json_key_string: "gcs_json")
+      .and_return(gcs_client)
+    allow(Slack::Notifier).to receive(:new)
+      .with("slack_url", channel: "#quality-reports", username: "Spec Runtime Report")
+      .and_return(slack_notifier)
+  end
+
+  context "without specs exceeding runtime" do
+    let(:report) do
+      {
+        body: <<~JSON
+          {
+            "spec.rb": 5,
+            "spec_2.rb": 10
+          }
+        JSON
+      }
+    end
+
+    it "returns all good message" do
+      expect { reporter }.to output("No long running specs detected, all good!\n").to_stdout
+    end
+  end
+
+  context "with specs exceeding runtime" do
+    let(:report) do
+      {
+        body: <<~JSON
+          {
+            "spec.rb": 5.0,
+            "spec_2.rb": 320.0
+          }
+        JSON
+      }
+    end
+
+    let(:spec) { "spec_2.rb: 5.33 minutes" }
+
+    let(:message) do
+      <<~MSG
+        Following spec files are exceeding 5 minute runtime threshold!
+        Current average spec runtime: 5 seconds.
+      MSG
+    end
+
+    it "notifies on long running specs" do
+      expect { reporter }.to output("#{message}\n#{spec}\n").to_stdout
+      expect(slack_notifier).to have_received(:post).with(
+        icon_emoji: ":time-out:",
+        text: "#{message}\n```#{spec}```"
+      )
+    end
+  end
+end

qa/tasks/knapsack.rake

 # frozen_string_literal: true
 # rubocop:disable Rails/RakeEnvironment
 
-require_relative "../qa/tools/knapsack_report"
-
 namespace :knapsack do
   desc "Download latest knapsack report"
   task :download do
@@ -13,5 +11,10 @@ namespace :knapsack do
   task :upload, [:glob] do |_task, args|
     QA::Tools::KnapsackReport.upload_report(args[:glob])
   end
+
+  desc "Report long running spec files"
+  task :notify_long_running_specs do
+    QA::Tools::LongRunningSpecReporter.execute
+  end
 end
 # rubocop:enable Rails/RakeEnvironment

qa/tasks/reliable_report.rake

 # frozen_string_literal: true
 # rubocop:disable Rails/RakeEnvironment
 
-require_relative "../qa/tools/reliable_report"
-
 desc "Fetch reliable and unreliable spec data and create report"
 task :reliable_spec_report, [:range, :report_in_issue_and_slack] do |_task, args|
   QA::Tools::ReliableReport.run(**args)