Skip to content

G
gitlab-ce
Commit 25122cfd authored by Andrejs Cunskis's avatar Andrejs Cunskis Committed by Rémy Coutable
Browse files
Options

E2E: Generate knapsack report on scheduled runs

parent ab4e3d10
Hide whitespace changes
Inline Side-by-side
Showing with 253 additions and 7 deletions
.gitlab/ci/review-apps/qa.gitlab-ci.yml
View file @ 25122cfd
......@@ -26,6 +26,9 @@
- export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
- echo "${CI_ENVIRONMENT_URL}"
- cd qa
- if [ -n "$KNAPSACK_REPORT_PATH" ]; then
bundle exec rake knapsack:download;
fi
artifacts:
paths:
- qa/tmp
......@@ -34,15 +37,22 @@
.parallel-qa-base:
parallel: 5
variables:
KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb"
script:
- export KNAPSACK_REPORT_PATH=knapsack/master_report.json
- export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
- |
bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \
-- \
--color --format documentation \
--format RspecJunitFormatter --out tmp/rspec.xml
after_script:
- if [ -n "$KNAPSACK_GENERATE_REPORT" ]; then
mv qa/${KNAPSACK_REPORT_PATH} qa/knapsack/gcs/regenerated-${CI_NODE_INDEX}.json;
fi
artifacts:
paths:
- qa/tmp # we can't merge list so need to include explicitly once more
- qa/knapsack/gcs/regenerated-*.json
reports:
junit: qa/tmp/rspec.xml
......@@ -69,6 +79,16 @@
--ignore-missing-results \
--color
.knapsack-upload-base:
image:
name: ${QA_IMAGE}
entrypoint: [""]
stage: post-qa
before_script:
- cd qa
script:
- bundle exec rake 'knapsack:upload[knapsack/gcs/regenerated-*.json]'
review-qa-smoke:
extends:
- .review-qa-base
......@@ -88,6 +108,7 @@ review-qa-reliable:
variables:
QA_RUN_TYPE: review-qa-reliable
QA_SCENARIO: Test::Instance::Reliable
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
review-qa-all:
extends:
......@@ -97,6 +118,7 @@ review-qa-all:
variables:
QA_RUN_TYPE: review-qa-all
QA_SCENARIO: Test::Instance::All
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
review-performance:
extends:
......@@ -141,3 +163,19 @@ allure-report-qa-all:
variables:
ALLURE_REPORT_PATH_PREFIX: gitlab-review-all
ALLURE_JOB_NAME: review-qa-all
knapsack-report-qa-all:
extends:
- .knapsack-upload-base
- .review:rules:knapsack-report-qa-all
needs: ["review-qa-all"]
variables:
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
knapsack-report-qa-reliable:
extends:
- .knapsack-upload-base
- .review:rules:knapsack-report-qa-reliable
needs: ["review-qa-reliable"]
variables:
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
.gitlab/ci/rules.gitlab-ci.yml
View file @ 25122cfd
......@@ -1621,6 +1621,10 @@
.review:rules:review-qa-reliable:
rules:
- <<: *if-dot-com-ee-2-hourly-schedule
allow_failure: true
variables:
KNAPSACK_GENERATE_REPORT: "true"
- when: on_success
allow_failure: true
......@@ -1631,6 +1635,10 @@
changes: *code-patterns
when: manual
allow_failure: true # manual jobs needs to be allowd to fail, otherwise they block the pipeline
- <<: *if-dot-com-ee-2-hourly-schedule
allow_failure: true
variables:
KNAPSACK_GENERATE_REPORT: "true"
- when: on_success
allow_failure: true
......@@ -1646,6 +1654,20 @@
- when: on_failure
allow_failure: true
# Generate knapsack report on successful runs only
# Reliable suite will pass most of the time so this should yield best distribution
.review:rules:knapsack-report-qa-reliable:
rules:
- <<: *if-dot-com-ee-2-hourly-schedule
when: on_success
allow_failure: true
.review:rules:knapsack-report-qa-all:
rules:
- <<: *if-dot-com-ee-2-hourly-schedule
when: always
allow_failure: true
.review:rules:review-cleanup:
rules:
- <<: *if-not-ee
......
qa/Gemfile
View file @ 25122cfd
......@@ -15,7 +15,7 @@ gem 'rest-client', '~> 2.1.0'
gem 'rspec-retry', '~> 0.6.1', require: 'rspec/retry'
gem 'rspec_junit_formatter', '~> 0.4.1'
gem 'faker', '~> 2.19', '>= 2.19.0'
gem 'knapsack', '~> 1.17'
gem 'knapsack', '~> 4.0'
gem 'parallel_tests', '~> 2.29'
gem 'rotp', '~> 3.1.0'
gem 'timecop', '~> 0.9.1'
......@@ -27,6 +27,7 @@ gem 'zeitwerk', '~> 2.4'
gem 'influxdb-client', '~> 1.17'
gem 'terminal-table', '~> 1.8', require: false
gem 'slack-notifier', '~> 2.4', require: false
gem 'fog-google', '~> 1.17', require: false
gem 'chemlab', '~> 0.9'
gem 'chemlab-library-www-gitlab-com', '~> 0.1'
......
qa/Gemfile.lock
View file @ 25122cfd
......@@ -29,6 +29,7 @@ GEM
uuid (>= 2.3, < 3)
ast (2.4.2)
binding_ninja (0.2.3)
builder (3.2.4)
byebug (9.1.0)
capybara (3.35.3)
addressable
......@@ -56,12 +57,14 @@ GEM
adamantium (~> 0.2.0)
equalizer (~> 0.0.9)
concurrent-ruby (1.1.9)
declarative (0.0.20)
deprecation_toolkit (1.5.1)
activesupport (>= 4.2)
diff-lcs (1.3)
domain_name (0.5.20190701)
unf (>= 0.0.5, < 1.0.0)
equalizer (0.0.11)
excon (0.88.0)
faker (2.19.0)
i18n (>= 1.6, < 2)
faraday (1.5.1)
......@@ -85,6 +88,30 @@ GEM
ffi-compiler (1.0.1)
ffi (>= 1.0.0)
rake
fog-core (2.1.0)
builder
excon (~> 0.58)
formatador (~> 0.2)
mime-types
fog-google (1.17.0)
fog-core (<= 2.1.0)
fog-json (~> 1.2)
fog-xml (~> 0.1.0)
google-apis-compute_v1 (~> 0.14)
google-apis-dns_v1 (~> 0.12)
google-apis-iamcredentials_v1 (~> 0.6)
google-apis-monitoring_v3 (~> 0.12)
google-apis-pubsub_v1 (~> 0.7)
google-apis-sqladmin_v1beta4 (~> 0.13)
google-apis-storage_v1 (~> 0.6)
google-cloud-env (~> 1.2)
fog-json (1.2.0)
fog-core
multi_json (~> 1.10)
fog-xml (0.1.4)
fog-core
nokogiri (>= 1.5.11, < 2.0.0)
formatador (0.3.0)
gitlab (4.16.1)
httparty (~> 0.14, >= 0.14.0)
terminal-table (~> 1.5, >= 1.5.1)
......@@ -94,6 +121,38 @@ GEM
http (~> 5.0)
nokogiri (~> 1.10)
table_print (= 1.5.7)
google-apis-compute_v1 (0.21.0)
google-apis-core (>= 0.4, < 2.a)
google-apis-core (0.4.1)
addressable (~> 2.5, >= 2.5.1)
googleauth (>= 0.16.2, < 2.a)
httpclient (>= 2.8.1, < 3.a)
mini_mime (~> 1.0)
representable (~> 3.0)
retriable (>= 2.0, < 4.a)
rexml
webrick
google-apis-dns_v1 (0.16.0)
google-apis-core (>= 0.4, < 2.a)
google-apis-iamcredentials_v1 (0.8.0)
google-apis-core (>= 0.4, < 2.a)
google-apis-monitoring_v3 (0.18.0)
google-apis-core (>= 0.4, < 2.a)
google-apis-pubsub_v1 (0.10.0)
google-apis-core (>= 0.4, < 2.a)
google-apis-sqladmin_v1beta4 (0.21.0)
google-apis-core (>= 0.4, < 2.a)
google-apis-storage_v1 (0.9.0)
google-apis-core (>= 0.4, < 2.a)
google-cloud-env (1.5.0)
faraday (>= 0.17.3, < 2.0)
googleauth (1.1.0)
faraday (>= 0.17.3, < 2.0)
jwt (>= 1.4, < 3.0)
memoist (~> 0.16)
multi_json (~> 1.11)
os (>= 0.9, < 2.0)
signet (>= 0.16, < 2.a)
http (5.0.4)
addressable (~> 2.8)
http-cookie (~> 1.0)
......@@ -106,11 +165,13 @@ GEM
httparty (0.20.0)
mime-types (~> 3.0)
multi_xml (>= 0.5.2)
httpclient (2.8.3)
i18n (1.8.11)
concurrent-ruby (~> 1.0)
ice_nine (0.11.2)
influxdb-client (1.17.0)
knapsack (1.17.1)
jwt (2.3.0)
knapsack (4.0.0)
rake
launchy (2.4.3)
addressable (~> 2.3)
......@@ -119,6 +180,7 @@ GEM
rake (~> 13.0)
macaddr (1.7.2)
systemu (~> 2.6.5)
memoist (0.16.2)
memoizable (0.4.2)
thread_safe (~> 0.3, >= 0.3.1)
method_source (0.9.0)
......@@ -128,6 +190,7 @@ GEM
mini_mime (1.1.0)
mini_portile2 (2.6.1)
minitest (5.14.4)
multi_json (1.15.0)
multi_xml (0.6.0)
multipart-post (2.1.1)
netrc (0.11.0)
......@@ -138,6 +201,7 @@ GEM
faraday (>= 0.9)
sawyer (~> 0.8.0, >= 0.5.3)
oj (3.13.8)
os (1.1.4)
parallel (1.19.2)
parallel_tests (2.29.0)
parallel
......@@ -161,12 +225,17 @@ GEM
rack (>= 1.0, < 3)
rake (13.0.6)
regexp_parser (2.1.1)
representable (3.1.1)
declarative (< 0.1.0)
trailblazer-option (>= 0.1.1, < 0.2.0)
uber (< 0.2.0)
require_all (3.0.0)
rest-client (2.1.0)
http-accept (>= 1.7.0, < 2.0)
http-cookie (>= 1.0.2, < 2.0)
mime-types (>= 1.16, < 4.0)
netrc (~> 0.8)
retriable (3.1.2)
rexml (3.2.5)
rotp (3.1.0)
rspec (3.10.0)
......@@ -203,6 +272,11 @@ GEM
childprocess (>= 0.5, < 5.0)
rexml (~> 3.2, >= 3.2.5)
rubyzip (>= 1.2.2)
signet (0.16.0)
addressable (~> 2.8)
faraday (>= 0.17.3, < 2.0)
jwt (>= 1.5, < 3.0)
multi_json (~> 1.10)
slack-notifier (2.4.0)
systemu (2.6.5)
table_print (1.5.7)
......@@ -210,8 +284,10 @@ GEM
unicode-display_width (~> 1.1, >= 1.1.1)
thread_safe (0.3.6)
timecop (0.9.1)
trailblazer-option (0.1.2)
tzinfo (2.0.4)
concurrent-ruby (~> 1.0)
uber (0.1.0)
unf (0.1.4)
unf_ext
unf_ext (0.0.8)
......@@ -233,6 +309,7 @@ GEM
nokogiri (~> 1.6)
rubyzip (>= 1.3.0)
selenium-webdriver (~> 4.0)
webrick (1.7.0)
xpath (3.2.0)
nokogiri (~> 1.8)
zeitwerk (2.5.1)
......@@ -250,9 +327,10 @@ DEPENDENCIES
chemlab-library-www-gitlab-com (~> 0.1)
deprecation_toolkit (~> 1.5.1)
faker (~> 2.19, >= 2.19.0)
fog-google (~> 1.17)
gitlab-qa
influxdb-client (~> 1.17)
knapsack (~> 1.17)
knapsack (~> 4.0)
octokit (~> 4.21)
parallel (~> 1.19)
parallel_tests (~> 2.29)
......
qa/Rakefile
View file @ 25122cfd
# frozen_string_literal: true
# rubocop:disable Rails/RakeEnvironment
load 'tasks/webdrivers.rake'
load 'tasks/reliable_report.rake'
Dir['tasks/*.rake'].each { |file| load file }
require_relative 'qa/tools/revoke_all_personal_access_tokens'
require_relative 'qa/tools/delete_subgroups'
......
qa/qa/tools/knapsack_report.rb 0 → 100644
View file @ 25122cfd
# frozen_string_literal: true
require "fog/google"
module QA
module Tools
class KnapsackReport
PROJECT = "gitlab-qa-resources"
BUCKET = "knapsack-reports"
class << self
def download
new.download_report
end
def upload(glob)
new.upload_report(glob)
end
end
def initialize
ENV["KNAPSACK_REPORT_PATH"] || raise("KNAPSACK_REPORT_PATH env var is required!")
ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("QA_KNAPSACK_REPORT_GCS_CREDENTIALS env var is required!")
end
# Download knapsack report from gcs bucket
#
# @return [void]
def download_report
logger.info("Downloading latest knapsack report '#{report_file}'")
file = client.get_object(BUCKET, report_file)
logger.info("Saving latest knapsack report to '#{report_path}'")
File.write(report_path, file[:body])
end
# Merge and upload knapsack report to gcs bucket
#
# @param [String] glob
# @return [void]
def upload_report(glob)
reports = Dir[glob]
return logger.error("Pattern '#{glob}' did not match any files!") if reports.empty?
report = reports
.map { |path| JSON.parse(File.read(path)) }
.reduce({}, :merge)
return logger.error("Knapsack generated empty report, skipping upload!") if report.empty?
logger.info("Uploading latest knapsack report '#{report_file}'")
client.put_object(BUCKET, report_file, JSON.pretty_generate(report))
end
private
# Logger instance
#
# @return [Logger]
def logger
@logger ||= Logger.new($stdout)
end
# GCS client
#
# @return [Fog::Storage::GoogleJSON]
def client
@client ||= Fog::Storage::Google.new(
google_project: PROJECT,
google_json_key_location: ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"]
)
end
# Knapsack report path
#
# @return [String]
def report_path
@report_path ||= ENV["KNAPSACK_REPORT_PATH"]
end
# Knapsack report name
#
# @return [String]
def report_file
@report_name ||= report_path.split("/").last
end
end
end
end
qa/tasks/knapsack.rake 0 → 100644
View file @ 25122cfd
# frozen_string_literal: true
# rubocop:disable Rails/RakeEnvironment
require_relative "../qa/tools/knapsack_report"
namespace :knapsack do
desc "Download latest knapsack report"
task :download do
QA::Tools::KnapsackReport.download
end
desc "Merge and upload knapsack report"
task :upload, [:glob_pattern] do |_task, args|
QA::Tools::KnapsackReport.upload(args[:glob_pattern])
end
end
# rubocop:enable Rails/RakeEnvironment
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7