Merge branch 'acunskis-knapsack-download' into 'master'

E2E: Integrate knapsack report setup in to QA framework

See merge request gitlab-org/gitlab!77316

.gitlab/ci/review-apps/qa.gitlab-ci.yml

@@ -26,35 +26,22 @@
     - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
     - cd qa
-    - if [ -n "$KNAPSACK_REPORT_PATH" ]; then
-        bundle exec rake knapsack:download;
-      fi
-  artifacts:
-    paths:
-      - qa/tmp
-    expire_in: 7 days
-    when: always
-
-.parallel-qa-base:
-  parallel: 5
-  variables:
-    KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb"
   script:
     - |
       bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \
         -- \
         --color --format documentation \
         --format RspecJunitFormatter --out tmp/rspec.xml
-  after_script:
-    - if [ -n "$KNAPSACK_GENERATE_REPORT" ]; then
-        mv qa/${KNAPSACK_REPORT_PATH} qa/knapsack/gcs/regenerated-${CI_NODE_INDEX}.json;
-      fi
   artifacts:
     paths:
-      - qa/tmp # we can't merge list so need to include explicitly once more
-      - qa/knapsack/gcs/regenerated-*.json
+      - qa/tmp
     reports:
       junit: qa/tmp/rspec.xml
+    expire_in: 7 days
+    when: always
+
+.parallel-qa-base:
+  parallel: 5
 
 .allure-report-base:
   image:
@@ -79,17 +66,6 @@
         --ignore-missing-results \
         --color
 
-.knapsack-upload-base:
-  image:
-    name: ${QA_IMAGE}
-    entrypoint: [""]
-  stage: post-qa
-  allow_failure: true
-  before_script:
-    - cd qa
-  script:
-    - bundle exec rake 'knapsack:upload[knapsack/gcs/regenerated-*.json]'
-
 review-qa-smoke:
   extends:
     - .review-qa-base
@@ -97,8 +73,8 @@ review-qa-smoke:
   retry: 1  # This is confusing but this means "2 runs at max".
   variables:
     QA_RUN_TYPE: review-qa-smoke
-  script:
-    - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}"
+    QA_SCENARIO: Test::Instance::Smoke
+
 
 review-qa-reliable:
   extends:
@@ -109,7 +85,6 @@ review-qa-reliable:
   variables:
     QA_RUN_TYPE: review-qa-reliable
     QA_SCENARIO: Test::Instance::Reliable
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
 
 review-qa-all:
   extends:
@@ -119,7 +94,6 @@ review-qa-all:
   variables:
     QA_RUN_TYPE: review-qa-all
     QA_SCENARIO: Test::Instance::All
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
 
 review-performance:
   extends:
@@ -174,18 +148,15 @@ allure-report-qa-all:
     ALLURE_REPORT_PATH_PREFIX: gitlab-review-all
     ALLURE_JOB_NAME: review-qa-all
 
-knapsack-report-qa-all:
-  extends:
-    - .knapsack-upload-base
-    - .review:rules:knapsack-report-qa-all
-  needs: ["review-qa-all"]
-  variables:
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
-
-knapsack-report-qa-reliable:
+knapsack-report:
   extends:
-    - .knapsack-upload-base
-    - .review:rules:knapsack-report-qa-reliable
-  needs: ["review-qa-reliable"]
-  variables:
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
+    - .review:rules:knapsack-report
+  image:
+    name: ${QA_IMAGE}
+    entrypoint: [""]
+  stage: post-qa
+  allow_failure: true
+  before_script:
+    - cd qa
+  script:
+    - bundle exec rake 'knapsack:upload[tmp/knapsack/*/*.json]'

.gitlab/ci/rules.gitlab-ci.yml

@@ -1689,20 +1689,10 @@
     - when: on_success
     - when: on_failure
 
-# Generate knapsack report on successful runs only
-# Reliable suite will pass most of the time so this should yield best distribution
-.review:rules:knapsack-report-qa-reliable:
+.review:rules:knapsack-report:
   rules:
     - if: '$KNAPSACK_GENERATE_REPORT == "true"'
-      when: on_success
-
-# Since `review-qa-all` is allowed to fail (and potentially manual), we need to use `when: on_success` and `when: on_failure` for `knapsack-report-qa-all`.
-.review:rules:knapsack-report-qa-all:
-  rules:
-    - if: '$KNAPSACK_GENERATE_REPORT != "true"'
-      when: never
-    - when: on_success
-    - when: on_failure
+      when: always
 
 .review:rules:review-cleanup:
   rules:

qa/knapsack/gcs/.gitignore → qa/knapsack/.gitignore

 **
 
 !.gitignore
+!master_report.json

qa/qa/runtime/env.rb

@@ -281,9 +281,7 @@ module QA
       end
 
       def knapsack?
-        return false unless ENV['CI_NODE_TOTAL'].to_i > 1
-
-        !!(ENV['KNAPSACK_GENERATE_REPORT'] || ENV['KNAPSACK_REPORT_PATH'] || ENV['KNAPSACK_TEST_FILE_PATTERN'])
+        ENV['CI_NODE_TOTAL'].to_i > 1 && ENV['NO_KNAPSACK'] != "true"
       end
 
       def ldap_username

qa/qa/scenario/template.rb

@@ -32,6 +32,11 @@ module QA
         #   Given *gitlab_address* = 'http://gitlab-abc123.test/' #=> http://about.gitlab-abc123.test/
         Runtime::Scenario.define(:about_address, URI(-> { gitlab_address.host = "about.#{gitlab_address.host}"; gitlab_address }.call).to_s) # rubocop:disable Style/Semicolon
 
+        ##
+        # Setup knapsack and download latest report
+        #
+        Tools::KnapsackReport.configure! if Runtime::Env.knapsack?
+
         ##
         # Perform before hooks, which are different for CE and EE
         #

qa/qa/specs/runner.rb

@@ -19,13 +19,11 @@ module QA
       def paths_from_knapsack
         allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
 
-        QA::Runtime::Logger.info ''
+        QA::Runtime::Logger.info '==== Knapsack specs to execute ====='
         QA::Runtime::Logger.info 'Report specs:'
         QA::Runtime::Logger.info allocator.report_node_tests.join(', ')
-        QA::Runtime::Logger.info ''
         QA::Runtime::Logger.info 'Leftover specs:'
         QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ')
-        QA::Runtime::Logger.info ''
 
         ['--', allocator.node_tests]
       end

qa/qa/tools/knapsack_report.rb

@@ -5,50 +5,81 @@ require "fog/google"
 module QA
   module Tools
     class KnapsackReport
+      extend SingleForwardable
+
       PROJECT = "gitlab-qa-resources"
       BUCKET = "knapsack-reports"
 
-      class << self
-        def download
-          new.download_report
-        end
+      def_delegators :new, :configure!, :move_regenerated_report, :download_report, :upload_report
 
-        def upload(glob)
-          new.upload_report(glob)
-        end
-      end
+      # Configure knapsack report
+      #
+      # * Setup variables
+      # * Fetch latest report
+      #
+      # @return [void]
+      def configure!
+        ENV["KNAPSACK_TEST_FILE_PATTERN"] ||= "qa/specs/features/**/*_spec.rb"
+        ENV["KNAPSACK_REPORT_PATH"] = report_path
 
-      def initialize
-        ENV["KNAPSACK_REPORT_PATH"] || raise("KNAPSACK_REPORT_PATH env var is required!")
-        ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("QA_KNAPSACK_REPORT_GCS_CREDENTIALS env var is required!")
+        Knapsack.logger = QA::Runtime::Logger.logger
+
+        download_report
       end
 
       # Download knapsack report from gcs bucket
       #
       # @return [void]
       def download_report
-        logger.info("Downloading latest knapsack report '#{report_file}'")
+        logger.debug("Downloading latest knapsack report for '#{report_name}' to '#{report_path}'")
         file = client.get_object(BUCKET, report_file)
-
-        logger.info("Saving latest knapsack report to '#{report_path}'")
         File.write(report_path, file[:body])
+      rescue StandardError => e
+        ENV["KNAPSACK_REPORT_PATH"] = "knapsack/master_report.json"
+        logger.warn("Failed to fetch latest knapsack report: #{e}")
+        logger.warn("Falling back to 'knapsack/master_report.json'")
+      end
+
+      # Rename and move new regenerated report to a separate folder used to indicate report name
+      #
+      # @return [void]
+      def move_regenerated_report
+        return unless ENV["KNAPSACK_GENERATE_REPORT"] == "true"
+
+        path = "tmp/knapsack/#{report_name}"
+        FileUtils.mkdir_p(path)
+
+        # Use path from knapsack config in case of fallback to master_report.json
+        FileUtils.cp(Knapsack.report.report_path, "#{path}/#{ENV['CI_NODE_INDEX']}.json")
       end
 
       # Merge and upload knapsack report to gcs bucket
       #
+      # Fetches all files defined in glob and uses parent folder as report name
+      #
       # @param [String] glob
       # @return [void]
       def upload_report(glob)
-        reports = Dir[glob]
-        return logger.error("Pattern '#{glob}' did not match any files!") if reports.empty?
+        reports = Pathname.glob(glob).each_with_object(Hash.new { |hsh, key| hsh[key] = [] }) do |report, hash|
+          next unless report.extname == ".json"
+
+          hash[report.parent.basename.to_s].push(report)
+        end
+        return logger.error("Glob '#{glob}' did not contain any valid report files!") if reports.empty?
+
+        reports.each do |name, jsons|
+          file = "#{name}.json"
 
-        report = reports
-          .map { |path| JSON.parse(File.read(path)) }
+          report = jsons
+            .map { |json| JSON.parse(File.read(json)) }
             .reduce({}, :merge)
-        return logger.error("Knapsack generated empty report, skipping upload!") if report.empty?
+          next logger.warn("Knapsack generated empty report for '#{name}', skipping upload!") if report.empty?
 
-        logger.info("Uploading latest knapsack report '#{report_file}'")
-        client.put_object(BUCKET, report_file, JSON.pretty_generate(report))
+          logger.info("Uploading latest knapsack report '#{file}'")
+          client.put_object(BUCKET, file, JSON.pretty_generate(report))
+        rescue StandardError => e
+          logger.error("Failed to upload knapsack report for '#{name}'. Error: #{e}")
+        end
       end
 
       private
@@ -66,22 +97,48 @@ module QA
       def client
         @client ||= Fog::Storage::Google.new(
           google_project: PROJECT,
-          google_json_key_location: ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"]
+          google_json_key_location: gcs_credentials
         )
       end
 
+      # Base path of knapsack report
+      #
+      # @return [String]
+      def report_base_path
+        @report_base_path ||= "knapsack"
+      end
+
       # Knapsack report path
       #
       # @return [String]
       def report_path
-        @report_path ||= ENV["KNAPSACK_REPORT_PATH"]
+        @report_path ||= "#{report_base_path}/#{report_file}"
       end
 
       # Knapsack report name
       #
       # @return [String]
       def report_file
-        @report_name ||= report_path.split("/").last
+        @report_file ||= "#{report_name}.json"
+      end
+
+      # Report name
+      #
+      # Infer report name from ci job name
+      # Remove characters incompatible with gcs bucket naming from job names like ee:instance-parallel
+      #
+      # @return [String]
+      def report_name
+        @report_name ||= ENV["CI_JOB_NAME"].split(" ").first.tr(":", "-")
+      end
+
+      # Path to GCS credentials json
+      #
+      # @return [String]
+      def gcs_credentials
+        @gcs_credentials ||= ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise(
+          "QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable is required!"
+        )
       end
     end
   end

qa/spec/runtime/env_spec.rb

@@ -173,31 +173,23 @@ RSpec.describe QA::Runtime::Env do
       stub_env('CI_NODE_TOTAL', '2')
     end
 
-    it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do
-      stub_env('KNAPSACK_GENERATE_REPORT', 'true')
-
+    it 'returns true if running in parallel CI run' do
       expect(described_class.knapsack?).to be_truthy
     end
 
-    it 'returns true if KNAPSACK_REPORT_PATH is defined' do
-      stub_env('KNAPSACK_REPORT_PATH', '/a/path')
-
-      expect(described_class.knapsack?).to be_truthy
+    it 'returns false if knapsack disabled' do
+      stub_env('NO_KNAPSACK', 'true')
+      expect(described_class.knapsack?).to be_falsey
     end
 
-    it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do
-      stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern')
-
-      expect(described_class.knapsack?).to be_truthy
-    end
+    it 'returns false if not running in a parallel job' do
+      stub_env('CI_NODE_TOTAL', '1')
 
-    it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do
       expect(described_class.knapsack?).to be_falsey
     end
 
-    it 'returns false if not running in parallel job' do
-      stub_env('CI_NODE_TOTAL', '1')
-      stub_env('KNAPSACK_GENERATE_REPORT', 'true')
+    it 'returns false if not running in ci' do
+      stub_env('CI_NODE_TOTAL', nil)
 
       expect(described_class.knapsack?).to be_falsey
     end

qa/spec/scenario/test/integration/github_spec.rb

@@ -2,7 +2,7 @@
 
 RSpec.describe QA::Scenario::Test::Integration::Github do
   describe '#perform' do
-    let(:env) { spy('Runtime::Env') }
+    let(:env) { spy('Runtime::Env', knapsack?: false) }
 
     before do
       stub_const('QA::Runtime::Env', env)

qa/spec/spec_helper.rb

@@ -10,7 +10,7 @@ require 'active_support/core_ext/object/blank'
 require_relative 'qa_deprecation_toolkit_env'
 QaDeprecationToolkitEnv.configure!
 
-Knapsack::Adapters::RSpecAdapter.bind if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK']
+Knapsack::Adapters::RSpecAdapter.bind if QA::Runtime::Env.knapsack?
 
 QA::Runtime::Browser.configure!
 QA::Runtime::AllureReport.configure!
@@ -65,10 +65,10 @@ RSpec.configure do |config|
   end
 
   config.after(:suite) do |suite|
-    # If any tests failed, leave the resources behind to help troubleshoot
-    next if suite.reporter.failed_examples.present?
+    QA::Tools::KnapsackReport.move_regenerated_report if QA::Runtime::Env.knapsack?
 
-    QA::Resource::ReusableProject.remove_all_via_api!
+    # If any tests failed, leave the resources behind to help troubleshoot
+    QA::Resource::ReusableProject.remove_all_via_api! unless suite.reporter.failed_examples.present?
   end
 
   config.expect_with :rspec do |expectations|

qa/tasks/knapsack.rake

@@ -10,8 +10,8 @@ namespace :knapsack do
   end
 
   desc "Merge and upload knapsack report"
-  task :upload, [:glob_pattern] do |_task, args|
-    QA::Tools::KnapsackReport.upload(args[:glob_pattern])
+  task :upload, [:glob] do |_task, args|
+    QA::Tools::KnapsackReport.upload_report(args[:glob])
   end
 end
 # rubocop:enable Rails/RakeEnvironment