Merge branch 'acunskis-spec-metadata-formatter' into 'master'

E2E: Custom rspec formatter to track QA test suite health

See merge request gitlab-org/gitlab!69273

.gitlab/ci/qa.gitlab-ci.yml

@@ -7,6 +7,7 @@
   variables:
     USE_BUNDLE_INSTALL: "false"
     SETUP_DB: "false"
+    QA_EXPORT_TEST_METRICS: "false"
   before_script:
     - !reference [.default-before_script, before_script]
     - cd qa/

qa/Gemfile

@@ -24,6 +24,7 @@ gem 'rspec-parameterized', '~> 0.4.2'
 gem 'octokit', '~> 4.21'
 gem 'webdrivers', '~> 4.6'
 gem 'zeitwerk', '~> 2.4'
+gem 'influxdb-client', '~> 1.17'
 
 gem 'chemlab', '~> 0.7'
 gem 'chemlab-library-www-gitlab-com', '~> 0.1'

qa/Gemfile.lock

@@ -88,6 +88,7 @@ GEM
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
+    influxdb-client (1.17.0)
     knapsack (1.17.1)
       rake
     launchy (2.4.3)
@@ -220,6 +221,7 @@ DEPENDENCIES
   deprecation_toolkit (~> 1.5.1)
   faker (~> 2.19, >= 2.19.0)
   gitlab-qa
+  influxdb-client (~> 1.17)
   knapsack (~> 1.17)
   octokit (~> 4.21)
   parallel (~> 1.19)

qa/qa/runtime/allure_report.rb

@@ -73,7 +73,7 @@ module QA
         def configure_rspec
           RSpec.configure do |config|
             config.add_formatter(AllureRspecFormatter)
-            config.add_formatter(QA::Support::AllureMetadataFormatter)
+            config.add_formatter(QA::Support::Formatters::AllureMetadataFormatter)
           end
         end
 

qa/qa/runtime/env.rb

@@ -403,6 +403,10 @@ module QA
         ENV['GITLAB_TLS_CERTIFICATE']
       end
 
+      def export_metrics?
+        running_in_ci? && enabled?(ENV['QA_EXPORT_TEST_METRICS'], default: true)
+      end
+
       private
 
       def remote_grid_credentials

qa/qa/specs/helpers/rspec.rb

@@ -19,13 +19,22 @@ module QA
         # expanding into the global state
         # See: https://github.com/rspec/rspec-core/issues/2603
         def describe_successfully(*args, &describe_body)
-          reporter = ::RSpec.configuration.reporter
-
-          example_group = RSpec.describe(*args, &describe_body)
+          example_group = ::RSpec.describe(*args, &describe_body)
           ran_successfully = example_group.run reporter
           expect(ran_successfully).to eq true
           example_group
         end
+
+        def send_stop_notification
+          reporter.notify(
+            :stop,
+            ::RSpec::Core::Notifications::ExamplesNotification.new(reporter)
+          )
+        end
+
+        def reporter
+          ::RSpec.configuration.reporter
+        end
       end
     end
   end

qa/qa/support/allure_metadata_formatter.rb

-# frozen_string_literal: true
-
-require 'rspec/core'
-require "rspec/core/formatters/base_formatter"
-
-module QA
-  module Support
-    class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter
-      ::RSpec::Core::Formatters.register(
-        self,
-        :example_started
-      )
-
-      # Starts example
-      # @param [RSpec::Core::Notifications::ExampleNotification] example_notification
-      # @return [void]
-      def example_started(example_notification)
-        example = example_notification.example
-
-        quarantine_issue = example.metadata.dig(:quarantine, :issue)
-        example.issue('Quarantine issue', quarantine_issue) if quarantine_issue
-
-        spec_file = example.file_path.split('/').last
-        example.issue(
-          'Failure issues',
-          "https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=#{spec_file}"
-        )
-        return unless Runtime::Env.running_in_ci?
-
-        example.add_link(name: "Job(#{Runtime::Env.ci_job_name})", url: Runtime::Env.ci_job_url)
-      end
-    end
-  end
-end

qa/qa/support/formatters/allure_metadata_formatter.rb

+# frozen_string_literal: true
+
+module QA
+  module Support
+    module Formatters
+      class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter
+        ::RSpec::Core::Formatters.register(
+          self,
+          :example_started
+        )
+
+        # Starts example
+        # @param [RSpec::Core::Notifications::ExampleNotification] example_notification
+        # @return [void]
+        def example_started(example_notification)
+          example = example_notification.example
+
+          quarantine_issue = example.metadata.dig(:quarantine, :issue)
+          example.issue('Quarantine issue', quarantine_issue) if quarantine_issue
+
+          spec_file = example.file_path.split('/').last
+          example.issue(
+            'Failure issues',
+            "https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=#{spec_file}"
+          )
+          return unless Runtime::Env.running_in_ci?
+
+          example.add_link(name: "Job(#{Runtime::Env.ci_job_name})", url: Runtime::Env.ci_job_url)
+        end
+      end
+    end
+  end
+end

qa/qa/specs/helpers/context_formatter.rb → qa/qa/support/formatters/context_formatter.rb

 # frozen_string_literal: true
 
-require 'rspec/core'
-require "rspec/core/formatters/base_formatter"
-
 module QA
-  module Specs
-    module Helpers
+  module Support
+    module Formatters
       class ContextFormatter < ::RSpec::Core::Formatters::BaseFormatter
-        include ContextSelector
+        include Specs::Helpers::ContextSelector
 
         ::RSpec::Core::Formatters.register(
           self,

qa/qa/support/formatters/formatters.rb

+# frozen_string_literal: true
+
+require 'rspec/core'
+require 'rspec/core/formatters/base_formatter'
+
+module QA
+  module Support
+    module Formatters
+    end
+  end
+end

qa/qa/specs/helpers/quarantine_formatter.rb → qa/qa/support/formatters/quarantine_formatter.rb

 # frozen_string_literal: true
 
-require 'rspec/core'
-require "rspec/core/formatters/base_formatter"
-
 module QA
-  module Specs
-    module Helpers
+  module Support
+    module Formatters
       class QuarantineFormatter < ::RSpec::Core::Formatters::BaseFormatter
-        include Quarantine
+        include Specs::Helpers::Quarantine
 
         ::RSpec::Core::Formatters.register(
           self,

qa/qa/support/formatters/test_stats_formatter.rb

+# frozen_string_literal: true
+
+module QA
+  module Support
+    module Formatters
+      class TestStatsFormatter < RSpec::Core::Formatters::BaseFormatter
+        RSpec::Core::Formatters.register(self, :stop)
+
+        # Finish test execution
+        #
+        # @param [RSpec::Core::Notifications::ExamplesNotification] notification
+        # @return [void]
+        def stop(notification)
+          return log(:warn, 'Missing QA_INFLUXDB_URL, skipping metrics export!') unless influxdb_url
+          return log(:warn, 'Missing QA_INFLUXDB_TOKEN, skipping metrics export!') unless influxdb_token
+
+          data = notification.examples.map { |example| test_stats(example) }.compact
+          influx_client.create_write_api.write(data: data)
+          log(:info, "Pushed #{data.length} entries to influxdb")
+        rescue StandardError => e
+          log(:error, "Failed to push data to influxdb, error: #{e}")
+        end
+
+        private
+
+        # InfluxDb client
+        #
+        # @return [InfluxDB2::Client]
+        def influx_client
+          @influx_client ||= InfluxDB2::Client.new(
+            influxdb_url,
+            influxdb_token,
+            bucket: 'e2e-test-stats',
+            org: 'gitlab-qa',
+            use_ssl: false,
+            precision: InfluxDB2::WritePrecision::NANOSECOND
+          )
+        end
+
+        # InfluxDb instance url
+        #
+        # @return [String]
+        def influxdb_url
+          @influxdb_url ||= ENV['QA_INFLUXDB_URL']
+        end
+
+        # Influxdb token
+        #
+        # @return [String]
+        def influxdb_token
+          @influxdb_token ||= ENV['QA_INFLUXDB_TOKEN']
+        end
+
+        # Transform example to influxdb compatible metrics data
+        # https://github.com/influxdata/influxdb-client-ruby#data-format
+        #
+        # @param [RSpec::Core::Example] example
+        # @return [Hash]
+        def test_stats(example)
+          {
+            name: 'test-stats',
+            time: time,
+            tags: {
+              name: example.full_description,
+              file_path: example.metadata[:file_path].gsub('./qa/specs/features', ''),
+              status: example.execution_result.status,
+              reliable: example.metadata.key?(:reliable).to_s,
+              quarantined: example.metadata.key?(:quarantine).to_s,
+              retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s,
+              job_name: job_name,
+              merge_request: merge_request,
+              run_type: ENV['QA_RUN_TYPE']
+            },
+            fields: {
+              id: example.id,
+              run_time: (example.execution_result.run_time * 1000).round,
+              retry_attempts: example.metadata[:retry_attempts] || 0,
+              job_url: QA::Runtime::Env.ci_job_url,
+              pipeline_id: ENV['CI_PIPELINE_ID']
+            }
+          }
+        rescue StandardError => e
+          log(:error, "Failed to transform example '#{example.id}', error: #{e}")
+          nil
+        end
+
+        # Single common timestamp for all exported example metrics to keep data points consistently grouped
+        #
+        # @return [Time]
+        def time
+          @time ||= DateTime.strptime(ENV['CI_PIPELINE_CREATED_AT']).to_time
+        end
+
+        # Is a merge request execution
+        #
+        # @return [String]
+        def merge_request
+          @merge_request ||= (!!ENV['CI_MERGE_REQUEST_IID'] || !!ENV['TOP_UPSTREAM_MERGE_REQUEST_IID']).to_s
+        end
+
+        # Base ci job name
+        #
+        # @return [String]
+        def job_name
+          @job_name ||= QA::Runtime::Env.ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        end
+
+        # Print log message
+        #
+        # @param [Symbol] level
+        # @param [String] message
+        # @return [void]
+        def log(level, message)
+          QA::Runtime::Logger.public_send(level, "influxdb exporter: #{message}")
+        end
+      end
+    end
+  end
+end

qa/spec/spec_helper.rb

@@ -16,17 +16,15 @@ QA::Runtime::Browser.configure!
 QA::Runtime::AllureReport.configure!
 QA::Runtime::Scenario.from_env(QA::Runtime::Env.runtime_scenario_attributes)
 
-Dir[::File.join(__dir__, "support/helpers/*.rb")].sort.each { |f| require f }
-Dir[::File.join(__dir__, "support/matchers/*.rb")].sort.each { |f| require f }
-Dir[::File.join(__dir__, "support/shared_contexts/*.rb")].sort.each { |f| require f }
 Dir[::File.join(__dir__, "support/shared_examples/*.rb")].sort.each { |f| require f }
 
 RSpec.configure do |config|
   config.include QA::Support::Matchers::EventuallyMatcher
   config.include QA::Support::Matchers::HaveMatcher
 
-  config.add_formatter QA::Specs::Helpers::ContextFormatter
-  config.add_formatter QA::Specs::Helpers::QuarantineFormatter
+  config.add_formatter QA::Support::Formatters::ContextFormatter
+  config.add_formatter QA::Support::Formatters::QuarantineFormatter
+  config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.export_metrics?
 
   config.before do |example|
     QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n")

qa/spec/specs/allure_report_spec.rb

@@ -68,7 +68,8 @@ describe QA::Runtime::AllureReport do
 
     it 'adds rspec and metadata formatter' do
       expect(rspec_config).to have_received(:add_formatter).with(AllureRspecFormatter).ordered
-      expect(rspec_config).to have_received(:add_formatter).with(QA::Support::AllureMetadataFormatter).ordered
+      expect(rspec_config).to have_received(:add_formatter)
+        .with(QA::Support::Formatters::AllureMetadataFormatter).ordered
     end
 
     it 'configures screenshot saving' do

qa/spec/specs/helpers/context_selector_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe QA::Specs::Helpers::ContextSelector do
     QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com')
 
     RSpec::Core::Sandbox.sandboxed do |config|
-      config.formatter = QA::Specs::Helpers::ContextFormatter
+      config.formatter = QA::Support::Formatters::ContextFormatter
 
       # If there is an example-within-an-example, we want to make sure the inner example
       # does not get a reference to the outer example (the real spec) if it calls

qa/spec/specs/helpers/quarantine_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe QA::Specs::Helpers::Quarantine do
 
   around do |ex|
     RSpec::Core::Sandbox.sandboxed do |config|
-      config.formatter = QA::Specs::Helpers::QuarantineFormatter
+      config.formatter = QA::Support::Formatters::QuarantineFormatter
 
       # If there is an example-within-an-example, we want to make sure the inner example
       # does not get a reference to the outer example (the real spec) if it calls

qa/spec/support/allure_metadata_formatter_spec.rb → qa/spec/support/formatters/allure_metadata_formatter_spec.rb

 # frozen_string_literal: true
 
-describe QA::Support::AllureMetadataFormatter do
+describe QA::Support::Formatters::AllureMetadataFormatter do
   include QA::Support::Helpers::StubEnv
 
   let(:formatter) { described_class.new(StringIO.new) }

qa/spec/support/formatters/test_stats_formatter_spec.rb

+# frozen_string_literal: true
+
+require 'rspec/core/sandbox'
+
+describe QA::Support::Formatters::TestStatsFormatter do
+  include QA::Support::Helpers::StubEnv
+  include QA::Specs::Helpers::RSpec
+
+  let(:url) { "http://influxdb.net" }
+  let(:token) { "token" }
+  let(:ci_timestamp) { "2021-02-23T20:58:41Z" }
+  let(:ci_job_name) { "test-job 1/5" }
+  let(:ci_job_url) { "url" }
+  let(:ci_pipeline_id) { "123" }
+  let(:run_type) { 'staging-full' }
+  let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) }
+  let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) }
+
+  let(:influx_client_args) do
+    {
+      bucket: 'e2e-test-stats',
+      org: 'gitlab-qa',
+      use_ssl: false,
+      precision: InfluxDB2::WritePrecision::NANOSECOND
+    }
+  end
+
+  let(:data) do
+    {
+      name: 'test-stats',
+      time: DateTime.strptime(ci_timestamp).to_time,
+      tags: {
+        name: "stats export #{spec_name}",
+        file_path: './spec/support/formatters/test_stats_formatter_spec.rb',
+        status: :passed,
+        reliable: reliable,
+        quarantined: quarantined,
+        retried: "false",
+        job_name: "test-job",
+        merge_request: "false",
+        run_type: run_type
+      },
+      fields: {
+        id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',
+        run_time: 0,
+        retry_attempts: 0,
+        job_url: ci_job_url,
+        pipeline_id: ci_pipeline_id
+      }
+    }
+  end
+
+  def run_spec(&spec)
+    describe_successfully('stats export', &spec)
+    send_stop_notification
+  end
+
+  around do |example|
+    RSpec::Core::Sandbox.sandboxed do |config|
+      config.formatter = QA::Support::Formatters::TestStatsFormatter
+
+      config.before(:context) { RSpec.current_example = nil }
+
+      example.run
+    end
+  end
+
+  before do
+    allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client }
+  end
+
+  context "without influxdb variables configured" do
+    it "skips export without influxdb url" do
+      stub_env('QA_INFLUXDB_URL', nil)
+      stub_env('QA_INFLUXDB_TOKEN', nil)
+
+      run_spec do
+        it('skips export') {}
+      end
+
+      expect(influx_client).not_to have_received(:create_write_api)
+    end
+
+    it "skips export without influxdb token" do
+      stub_env('QA_INFLUXDB_URL', url)
+      stub_env('QA_INFLUXDB_TOKEN', nil)
+
+      run_spec do
+        it('skips export') {}
+      end
+
+      expect(influx_client).not_to have_received(:create_write_api)
+    end
+  end
+
+  context 'with influxdb variables configured' do
+    let(:spec_name) { 'exports data' }
+    let(:run_type) { ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '') }
+
+    before do
+      stub_env('QA_INFLUXDB_URL', url)
+      stub_env('QA_INFLUXDB_TOKEN', token)
+      stub_env('CI_PIPELINE_CREATED_AT', ci_timestamp)
+      stub_env('CI_JOB_URL', ci_job_url)
+      stub_env('CI_JOB_NAME', ci_job_name)
+      stub_env('CI_PIPELINE_ID', ci_pipeline_id)
+      stub_env('CI_MERGE_REQUEST_IID', nil)
+      stub_env('TOP_UPSTREAM_MERGE_REQUEST_IID', nil)
+      stub_env('QA_RUN_TYPE', run_type)
+    end
+
+    context 'with reliable spec' do
+      let(:reliable) { 'true' }
+      let(:quarantined) { 'false' }
+
+      it 'exports data to influxdb' do
+        run_spec do
+          it('exports data', :reliable) {}
+        end
+
+        expect(influx_write_api).to have_received(:write).with(data: [data])
+      end
+    end
+
+    context 'with quarantined spec' do
+      let(:reliable) { 'false' }
+      let(:quarantined) { 'true' }
+
+      it 'exports data to influxdb' do
+        run_spec do
+          it('exports data', :quarantine) {}
+        end
+
+        expect(influx_write_api).to have_received(:write).with(data: [data])
+      end
+    end
+  end
+end