Commit 064131ae authored by Lin Jen-Shin's avatar Lin Jen-Shin

Merge branch 'acunskis-knapsack-download' into 'master'

E2E: Integrate knapsack report setup in to QA framework

See merge request gitlab-org/gitlab!77316
parents 65f51f7d b2cb15ed
......@@ -26,35 +26,22 @@
- export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
- echo "${CI_ENVIRONMENT_URL}"
- cd qa
- if [ -n "$KNAPSACK_REPORT_PATH" ]; then
bundle exec rake knapsack:download;
fi
artifacts:
paths:
- qa/tmp
expire_in: 7 days
when: always
.parallel-qa-base:
parallel: 5
variables:
KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb"
script:
- |
bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \
-- \
--color --format documentation \
--format RspecJunitFormatter --out tmp/rspec.xml
after_script:
- if [ -n "$KNAPSACK_GENERATE_REPORT" ]; then
mv qa/${KNAPSACK_REPORT_PATH} qa/knapsack/gcs/regenerated-${CI_NODE_INDEX}.json;
fi
artifacts:
paths:
- qa/tmp # we can't merge list so need to include explicitly once more
- qa/knapsack/gcs/regenerated-*.json
- qa/tmp
reports:
junit: qa/tmp/rspec.xml
expire_in: 7 days
when: always
.parallel-qa-base:
parallel: 5
.allure-report-base:
image:
......@@ -79,17 +66,6 @@
--ignore-missing-results \
--color
.knapsack-upload-base:
image:
name: ${QA_IMAGE}
entrypoint: [""]
stage: post-qa
allow_failure: true
before_script:
- cd qa
script:
- bundle exec rake 'knapsack:upload[knapsack/gcs/regenerated-*.json]'
review-qa-smoke:
extends:
- .review-qa-base
......@@ -97,8 +73,8 @@ review-qa-smoke:
retry: 1 # This is confusing but this means "2 runs at max".
variables:
QA_RUN_TYPE: review-qa-smoke
script:
- bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}"
QA_SCENARIO: Test::Instance::Smoke
review-qa-reliable:
extends:
......@@ -109,7 +85,6 @@ review-qa-reliable:
variables:
QA_RUN_TYPE: review-qa-reliable
QA_SCENARIO: Test::Instance::Reliable
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
review-qa-all:
extends:
......@@ -119,7 +94,6 @@ review-qa-all:
variables:
QA_RUN_TYPE: review-qa-all
QA_SCENARIO: Test::Instance::All
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
review-performance:
extends:
......@@ -174,18 +148,15 @@ allure-report-qa-all:
ALLURE_REPORT_PATH_PREFIX: gitlab-review-all
ALLURE_JOB_NAME: review-qa-all
knapsack-report-qa-all:
extends:
- .knapsack-upload-base
- .review:rules:knapsack-report-qa-all
needs: ["review-qa-all"]
variables:
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
knapsack-report-qa-reliable:
knapsack-report:
extends:
- .knapsack-upload-base
- .review:rules:knapsack-report-qa-reliable
needs: ["review-qa-reliable"]
variables:
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
- .review:rules:knapsack-report
image:
name: ${QA_IMAGE}
entrypoint: [""]
stage: post-qa
allow_failure: true
before_script:
- cd qa
script:
- bundle exec rake 'knapsack:upload[tmp/knapsack/*/*.json]'
......@@ -1689,20 +1689,10 @@
- when: on_success
- when: on_failure
# Generate knapsack report on successful runs only
# Reliable suite will pass most of the time so this should yield best distribution
.review:rules:knapsack-report-qa-reliable:
.review:rules:knapsack-report:
rules:
- if: '$KNAPSACK_GENERATE_REPORT == "true"'
when: on_success
# Since `review-qa-all` is allowed to fail (and potentially manual), we need to use `when: on_success` and `when: on_failure` for `knapsack-report-qa-all`.
.review:rules:knapsack-report-qa-all:
rules:
- if: '$KNAPSACK_GENERATE_REPORT != "true"'
when: never
- when: on_success
- when: on_failure
when: always
.review:rules:review-cleanup:
rules:
......
......@@ -281,9 +281,7 @@ module QA
end
def knapsack?
return false unless ENV['CI_NODE_TOTAL'].to_i > 1
!!(ENV['KNAPSACK_GENERATE_REPORT'] || ENV['KNAPSACK_REPORT_PATH'] || ENV['KNAPSACK_TEST_FILE_PATTERN'])
ENV['CI_NODE_TOTAL'].to_i > 1 && ENV['NO_KNAPSACK'] != "true"
end
def ldap_username
......
......@@ -32,6 +32,11 @@ module QA
# Given *gitlab_address* = 'http://gitlab-abc123.test/' #=> http://about.gitlab-abc123.test/
Runtime::Scenario.define(:about_address, URI(-> { gitlab_address.host = "about.#{gitlab_address.host}"; gitlab_address }.call).to_s) # rubocop:disable Style/Semicolon
##
# Setup knapsack and download latest report
#
Tools::KnapsackReport.configure! if Runtime::Env.knapsack?
##
# Perform before hooks, which are different for CE and EE
#
......
......@@ -19,13 +19,11 @@ module QA
def paths_from_knapsack
allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
QA::Runtime::Logger.info ''
QA::Runtime::Logger.info '==== Knapsack specs to execute ====='
QA::Runtime::Logger.info 'Report specs:'
QA::Runtime::Logger.info allocator.report_node_tests.join(', ')
QA::Runtime::Logger.info ''
QA::Runtime::Logger.info 'Leftover specs:'
QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ')
QA::Runtime::Logger.info ''
['--', allocator.node_tests]
end
......
......@@ -5,50 +5,81 @@ require "fog/google"
module QA
module Tools
class KnapsackReport
extend SingleForwardable
PROJECT = "gitlab-qa-resources"
BUCKET = "knapsack-reports"
class << self
def download
new.download_report
end
def_delegators :new, :configure!, :move_regenerated_report, :download_report, :upload_report
def upload(glob)
new.upload_report(glob)
end
end
# Configure knapsack report
#
# * Setup variables
# * Fetch latest report
#
# @return [void]
def configure!
ENV["KNAPSACK_TEST_FILE_PATTERN"] ||= "qa/specs/features/**/*_spec.rb"
ENV["KNAPSACK_REPORT_PATH"] = report_path
def initialize
ENV["KNAPSACK_REPORT_PATH"] || raise("KNAPSACK_REPORT_PATH env var is required!")
ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("QA_KNAPSACK_REPORT_GCS_CREDENTIALS env var is required!")
Knapsack.logger = QA::Runtime::Logger.logger
download_report
end
# Download knapsack report from gcs bucket
#
# @return [void]
def download_report
logger.info("Downloading latest knapsack report '#{report_file}'")
logger.debug("Downloading latest knapsack report for '#{report_name}' to '#{report_path}'")
file = client.get_object(BUCKET, report_file)
logger.info("Saving latest knapsack report to '#{report_path}'")
File.write(report_path, file[:body])
rescue StandardError => e
ENV["KNAPSACK_REPORT_PATH"] = "knapsack/master_report.json"
logger.warn("Failed to fetch latest knapsack report: #{e}")
logger.warn("Falling back to 'knapsack/master_report.json'")
end
# Rename and move new regenerated report to a separate folder used to indicate report name
#
# @return [void]
def move_regenerated_report
return unless ENV["KNAPSACK_GENERATE_REPORT"] == "true"
path = "tmp/knapsack/#{report_name}"
FileUtils.mkdir_p(path)
# Use path from knapsack config in case of fallback to master_report.json
FileUtils.cp(Knapsack.report.report_path, "#{path}/#{ENV['CI_NODE_INDEX']}.json")
end
# Merge and upload knapsack report to gcs bucket
#
# Fetches all files defined in glob and uses parent folder as report name
#
# @param [String] glob
# @return [void]
def upload_report(glob)
reports = Dir[glob]
return logger.error("Pattern '#{glob}' did not match any files!") if reports.empty?
reports = Pathname.glob(glob).each_with_object(Hash.new { |hsh, key| hsh[key] = [] }) do |report, hash|
next unless report.extname == ".json"
hash[report.parent.basename.to_s].push(report)
end
return logger.error("Glob '#{glob}' did not contain any valid report files!") if reports.empty?
reports.each do |name, jsons|
file = "#{name}.json"
report = reports
.map { |path| JSON.parse(File.read(path)) }
report = jsons
.map { |json| JSON.parse(File.read(json)) }
.reduce({}, :merge)
return logger.error("Knapsack generated empty report, skipping upload!") if report.empty?
next logger.warn("Knapsack generated empty report for '#{name}', skipping upload!") if report.empty?
logger.info("Uploading latest knapsack report '#{report_file}'")
client.put_object(BUCKET, report_file, JSON.pretty_generate(report))
logger.info("Uploading latest knapsack report '#{file}'")
client.put_object(BUCKET, file, JSON.pretty_generate(report))
rescue StandardError => e
logger.error("Failed to upload knapsack report for '#{name}'. Error: #{e}")
end
end
private
......@@ -66,22 +97,48 @@ module QA
def client
@client ||= Fog::Storage::Google.new(
google_project: PROJECT,
google_json_key_location: ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"]
google_json_key_location: gcs_credentials
)
end
# Base path of knapsack report
#
# @return [String]
def report_base_path
@report_base_path ||= "knapsack"
end
# Knapsack report path
#
# @return [String]
def report_path
@report_path ||= ENV["KNAPSACK_REPORT_PATH"]
@report_path ||= "#{report_base_path}/#{report_file}"
end
# Knapsack report name
#
# @return [String]
def report_file
@report_name ||= report_path.split("/").last
@report_file ||= "#{report_name}.json"
end
# Report name
#
# Infer report name from ci job name
# Remove characters incompatible with gcs bucket naming from job names like ee:instance-parallel
#
# @return [String]
def report_name
@report_name ||= ENV["CI_JOB_NAME"].split(" ").first.tr(":", "-")
end
# Path to GCS credentials json
#
# @return [String]
def gcs_credentials
@gcs_credentials ||= ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise(
"QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable is required!"
)
end
end
end
......
......@@ -173,31 +173,23 @@ RSpec.describe QA::Runtime::Env do
stub_env('CI_NODE_TOTAL', '2')
end
it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do
stub_env('KNAPSACK_GENERATE_REPORT', 'true')
it 'returns true if running in parallel CI run' do
expect(described_class.knapsack?).to be_truthy
end
it 'returns true if KNAPSACK_REPORT_PATH is defined' do
stub_env('KNAPSACK_REPORT_PATH', '/a/path')
expect(described_class.knapsack?).to be_truthy
it 'returns false if knapsack disabled' do
stub_env('NO_KNAPSACK', 'true')
expect(described_class.knapsack?).to be_falsey
end
it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do
stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern')
expect(described_class.knapsack?).to be_truthy
end
it 'returns false if not running in a parallel job' do
stub_env('CI_NODE_TOTAL', '1')
it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do
expect(described_class.knapsack?).to be_falsey
end
it 'returns false if not running in parallel job' do
stub_env('CI_NODE_TOTAL', '1')
stub_env('KNAPSACK_GENERATE_REPORT', 'true')
it 'returns false if not running in ci' do
stub_env('CI_NODE_TOTAL', nil)
expect(described_class.knapsack?).to be_falsey
end
......
......@@ -2,7 +2,7 @@
RSpec.describe QA::Scenario::Test::Integration::Github do
describe '#perform' do
let(:env) { spy('Runtime::Env') }
let(:env) { spy('Runtime::Env', knapsack?: false) }
before do
stub_const('QA::Runtime::Env', env)
......
......@@ -10,7 +10,7 @@ require 'active_support/core_ext/object/blank'
require_relative 'qa_deprecation_toolkit_env'
QaDeprecationToolkitEnv.configure!
Knapsack::Adapters::RSpecAdapter.bind if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK']
Knapsack::Adapters::RSpecAdapter.bind if QA::Runtime::Env.knapsack?
QA::Runtime::Browser.configure!
QA::Runtime::AllureReport.configure!
......@@ -65,10 +65,10 @@ RSpec.configure do |config|
end
config.after(:suite) do |suite|
# If any tests failed, leave the resources behind to help troubleshoot
next if suite.reporter.failed_examples.present?
QA::Tools::KnapsackReport.move_regenerated_report if QA::Runtime::Env.knapsack?
QA::Resource::ReusableProject.remove_all_via_api!
# If any tests failed, leave the resources behind to help troubleshoot
QA::Resource::ReusableProject.remove_all_via_api! unless suite.reporter.failed_examples.present?
end
config.expect_with :rspec do |expectations|
......
......@@ -10,8 +10,8 @@ namespace :knapsack do
end
desc "Merge and upload knapsack report"
task :upload, [:glob_pattern] do |_task, args|
QA::Tools::KnapsackReport.upload(args[:glob_pattern])
task :upload, [:glob] do |_task, args|
QA::Tools::KnapsackReport.upload_report(args[:glob])
end
end
# rubocop:enable Rails/RakeEnvironment
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment