Skip to content

G
gitlab-ce
Commit 7c0af33d authored by Andrejs Cunskis's avatar Andrejs Cunskis Committed by Sanad Liaquat
Browse files
Options

E2E: Update reliable spec report creation

parent d56e9bc5
Hide whitespace changes
Inline Side-by-side
Showing with 328 additions and 179 deletions
qa/qa/tools/reliable_report.rb
View file @ 7c0af33d
# frozen_string_literal: true
require_relative "../../qa"
require "influxdb-client"
require "terminal-table"
require "slack-notifier"
require "colorize"
module QA
module Tools
class ReliableReport
def initialize(run_type, range = 30)
@results = 2
@slack_channel = "#quality-reports"
include Support::API
# Project for report creation: https://gitlab.com/gitlab-org/gitlab
PROJECT_ID = 278964
def initialize(range)
@range = range
@run_type = run_type
@stable_title = "Top #{results} stable specs for past #{@range} days in '#{run_type}' runs"
@unstable_title = "Top #{results} unstable reliable specs for past #{@range} days in '#{run_type}' runs"
@influxdb_bucket = "e2e-test-stats"
@slack_channel = "#quality-reports"
@influxdb_url = ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable")
@influxdb_token = ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN env variable")
end
# Print top stable specs
# Run reliable reporter
#
# @param [Integer] range amount of days for results range
# @param [String] report_in_issue_and_slack
# @return [void]
def show_top_stable
results_table(:stable).each { |table| puts "#{table}\n\n" }
def self.run(range: 14, report_in_issue_and_slack: "false")
reporter = new(range)
reporter.print_report
reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true"
rescue StandardError => e
puts "Report creation failed! Error: '#{e}'".colorize(:red)
reporter.notify_failure(e)
exit(1)
end
# Post top stable spec report to slack
# Print top stable specs
#
# @return [void]
def notify_top_stable
puts "\nSending top stable spec report to #{slack_channel} slack channel"
slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
notifier.post(text: "*#{stable_title}*", **slack_args)
results_table(:stable).each { |table| notifier.post(text: "```#{table}```", **slack_args) }
def print_report
puts "#{stable_summary_table}\n\n"
stable_results_tables.each { |stage, table| puts "#{table}\n\n" }
return puts("No unstable reliable tests present!".colorize(:yellow)) if unstable_reliable_test_runs.empty?
puts "#{unstable_summary_table}\n\n"
unstable_reliable_results_tables.each { |stage, table| puts "#{table}\n\n" }
end
# Print top unstable specs
# Create report issue
#
# @return [void]
def show_top_unstable
return puts("No unstable tests present!") if top_unstable_reliable.empty?
def report_in_issue_and_slack
puts "Creating report".colorize(:green)
response = post(
"#{gitlab_api_url}/projects/#{PROJECT_ID}/issues",
{ title: "Reliable spec report", description: report_issue_body, labels: "Quality,test" },
headers: { "PRIVATE-TOKEN" => gitlab_access_token }
)
web_url = parse_body(response)[:web_url]
puts "Created report issue: #{web_url}"
results_table(:unstable).each { |table| puts "#{table}\n\n" }
puts "Sending slack notification".colorize(:green)
notifier.post(
icon_emoji: ":tanuki-protect:",
text: <<~TEXT
```#{stable_summary_table}```
```#{unstable_summary_table}```
#{web_url}
TEXT
)
puts "Done!"
end
# Post top unstable reliable spec report to slack
# Notify failure
#
# @param [StandardError] error
# @return [void]
def notify_top_unstable
return puts("No unstable tests present!") if top_unstable_reliable.empty?
puts "\nSending top unstable reliable spec report to #{slack_channel} slack channel"
slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
notifier.post(text: "*#{unstable_title}*", **slack_args)
results_table(:unstable).each { |table| notifier.post(text: "```#{table}```", **slack_args) }
def notify_failure(error)
notifier.post(
text: "Reliable reporter failed to create report. Error: ```#{error}```",
icon_emoji: ":sadpanda:"
)
end
private
attr_reader :results,
:slack_channel,
:range,
:run_type,
:stable_title,
:unstable_title
attr_reader :range, :influxdb_bucket, :slack_channel, :influxdb_url, :influxdb_token
# Markdown formatted report issue body
#
# @return [String]
def report_issue_body
issue = []
issue << "[[_TOC_]]"
issue << "# Candidates for promotion to reliable\n\n```\n#{stable_summary_table}\n```"
issue << results_markdown(stable_results_tables)
return issue.join("\n\n") if unstable_reliable_test_runs.empty?
issue << "# Reliable specs with failures\n\n```\n#{unstable_summary_table}\n```"
issue << results_markdown(unstable_reliable_results_tables)
issue.join("\n\n")
end
# Stable spec summary table
#
# @return [Terminal::Table]
def stable_summary_table
@stable_summary_table ||= terminal_table(
rows: stable_test_runs.map { |stage, specs| [stage, specs.length] },
title: "Stable spec summary for past #{range} days".ljust(50),
headings: %w[STAGE COUNT]
)
end
# Unstable reliable summary table
#
# @return [Terminal::Table]
def unstable_summary_table
@unstable_summary_table ||= terminal_table(
rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] },
title: "Unstable spec summary for past #{range} days".ljust(50),
headings: %w[STAGE COUNT]
)
end
# Result tables for stable specs
#
# @return [Hash]
def stable_results_tables
@stable_results ||= results_tables(:stable)
end
# Result table for unstable specs
#
# @return [Hash]
def unstable_reliable_results_tables
@unstable_results ||= results_tables(:unstable)
end
# Markdown formatted tables
#
# @param [Hash] results
# @return [String]
def results_markdown(results)
results.map do |stage, table|
<<~STAGE.strip
## #{stage}
<details>
<summary>Executions table</summary>
```
#{table}
```
</details>
STAGE
end.join("\n\n")
end
# Results table
#
# @param [Symbol] type result type - :stable, :unstable
# @return [Hash]
def results_table(type)
(type == :stable ? top_stable : top_unstable_reliable).map do |stage, specs|
terminal_table(
# @return [Hash<Symbol, Terminal::Table>]
def results_tables(type)
(type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs|
headings = ["name", "runs", "failures", "failure rate"]
[stage, terminal_table(
rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
title: "Top #{type} specs in '#{stage}' stage"
)
title: "Top #{type} specs in '#{stage}' stage for past #{range} days",
headings: headings.map(&:upcase)
)]
end
end
# Top stable specs
# Stable specs
#
# @return [Hash]
def top_stable
@top_stable ||= runs(reliable: false).transform_values do |specs|
specs.sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h
def stable_test_runs
@top_stable ||= begin
stable_specs = test_runs(reliable: false).transform_values do |specs|
specs
.reject { |k, v| v[:failure_rate] != 0 }
.sort_by { |k, v| -v[:runs] }
.to_h
end
stable_specs.reject { |k, v| v.empty? }
end
end
# Top unstable reliable specs
# Unstable reliable specs
#
# @return [Hash]
def top_unstable_reliable
def unstable_reliable_test_runs
@top_unstable_reliable ||= begin
unstable = runs(reliable: true).transform_values do |specs|
unstable = test_runs(reliable: true).transform_values do |specs|
specs
.reject { |k, v| v[:failure_rate] == 0 }
.sort_by { |k, v| -v[:failure_rate] }[0..results - 1]
.sort_by { |k, v| -v[:failure_rate] }
.to_h
end
......@@ -104,9 +215,9 @@ module QA
# Terminal table for result formatting
#
# @return [Terminal::Table]
def terminal_table(rows:, title: nil)
def terminal_table(rows:, headings:, title: nil)
Terminal::Table.new(
headings: ["name", "runs", "failed", "failure rate"],
headings: headings,
style: { all_separators: true },
title: title,
rows: rows
......@@ -127,20 +238,19 @@ module QA
# @param [String] file
# @return [String]
def name_column(name, file)
spec_name = name.length > 100 ? "#{name} ".scan(/.{1,100} /).map(&:strip).join("\n") : name
spec_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name
name_line = "name: '#{spec_name}'"
file_line = "file: '#{file}'"
"#{name_line}\n#{file_line.ljust(110)}"
"#{name_line}\n#{file_line.ljust(160)}"
end
# Test executions grouped by name
#
# @param [Boolean] reliable
# @return [Hash<String, Hash>]
def runs(reliable:)
puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past 30 days in '#{run_type}' runs")
puts
def test_runs(reliable:)
puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past #{range} days\n".colorize(:green))
all_runs = query_api.query(query: query(reliable)).values
all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
......@@ -168,17 +278,24 @@ module QA
# @return [String]
def query(reliable)
<<~QUERY
from(bucket: "e2e-test-stats")
|> range(start: -#{range}d)
|> filter(fn: (r) => r._measurement == "test-stats" and
r.run_type == "#{run_type}" and
r.status != "pending" and
r.merge_request == "false" and
r.quarantined == "false" and
r.reliable == "#{reliable}" and
r._field == "id"
)
|> group(columns: ["name"])
from(bucket: "#{influxdb_bucket}")
|> range(start: -#{range}d)
|> filter(fn: (r) => r._measurement == "test-stats")
|> filter(fn: (r) => r.run_type == "staging-full" or
r.run_type == "staging-sanity" or
r.run_type == "staging-sanity-no-admin" or
r.run_type == "production-full" or
r.run_type == "production-sanity" or
r.run_type == "package-and-qa" or
r.run_type == "nightly"
)
|> filter(fn: (r) => r.status != "pending" and
r.merge_request == "false" and
r.quarantined == "false" and
r.reliable == "#{reliable}" and
r._field == "id"
)
|> group(columns: ["name"])
QUERY
end
......@@ -196,7 +313,7 @@ module QA
@influx_client ||= InfluxDB2::Client.new(
influxdb_url,
influxdb_token,
bucket: "e2e-test-stats",
bucket: influxdb_bucket,
org: "gitlab-qa",
precision: InfluxDB2::WritePrecision::NANOSECOND
)
......@@ -209,29 +326,29 @@ module QA
@notifier ||= Slack::Notifier.new(
slack_webhook_url,
channel: slack_channel,
username: "Reliable spec reporter"
username: "Reliable Spec Report"
)
end
# InfluxDb instance url
# Gitlab access token
#
# @return [String]
def influxdb_url
@influxdb_url ||= ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL environment variable")
def gitlab_access_token
@gitlab_access_token ||= ENV["GITLAB_ACCESS_TOKEN"] || raise("Missing GITLAB_ACCESS_TOKEN env variable")
end
# Influxdb token
# Gitlab api url
#
# @return [String]
def influxdb_token
@influxdb_token ||= ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN environment variable")
def gitlab_api_url
@gitlab_api_url ||= ENV["CI_API_V4_URL"] || raise("Missing CI_API_V4_URL env variable")
end
# Slack webhook url
#
# @return [String]
def slack_webhook_url
@slack_webhook_url ||= ENV["CI_SLACK_WEBHOOK_URL"] || raise("Missing CI_SLACK_WEBHOOK_URL environment variable")
@slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable")
end
end
end
......
qa/spec/tools/reliable_report_spec.rb
View file @ 7c0af33d
......@@ -3,63 +3,94 @@
describe QA::Tools::ReliableReport do
include QA::Support::Helpers::StubEnv
subject(:reporter) { described_class.new(run_type, range) }
subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) }
let(:gitlab_response) { instance_double("RestClient::Response", code: 200, body: { web_url: issue_url }.to_json) }
let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) }
let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) }
let(:query_api) { instance_double("InfluxDB2::QueryApi") }
let(:slack_channel) { "#quality-reports" }
let(:run_type) { "package-and-qa" }
let(:range) { 30 }
let(:results) { 2 }
let(:runs) { { 0 => stable_spec, 1 => unstable_spec } }
let(:spec_values) { { "file_path" => "some/spec.rb", "stage" => "manage" } }
let(:stable_spec) do
values = { "name" => "stable spec", "status" => "passed", **spec_values }
instance_double(
"InfluxDB2::FluxTable",
records: [
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values)
]
)
let(:range) { 14 }
let(:issue_url) { "https://gitlab.com/issue/1" }
let(:runs) do
values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb", "stage" => "manage" }
{
0 => instance_double(
"InfluxDB2::FluxTable",
records: [
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values)
]
)
}
end
let(:unstable_spec) do
values = { "name" => "unstable spec", "status" => "failed", **spec_values }
instance_double(
"InfluxDB2::FluxTable",
records: [
instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }),
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values)
]
)
let(:reliable_runs) do
values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb", "stage" => "create" }
{
0 => instance_double(
"InfluxDB2::FluxTable",
records: [
instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }),
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values)
]
)
}
end
def flux_query(reliable)
def flux_query(reliable:)
<<~QUERY
from(bucket: "e2e-test-stats")
|> range(start: -#{range}d)
|> filter(fn: (r) => r._measurement == "test-stats" and
r.run_type == "#{run_type}" and
r.status != "pending" and
r.merge_request == "false" and
r.quarantined == "false" and
r.reliable == "#{reliable}" and
r._field == "id"
)
|> group(columns: ["name"])
from(bucket: "e2e-test-stats")
|> range(start: -#{range}d)
|> filter(fn: (r) => r._measurement == "test-stats")
|> filter(fn: (r) => r.run_type == "staging-full" or
r.run_type == "staging-sanity" or
r.run_type == "staging-sanity-no-admin" or
r.run_type == "production-full" or
r.run_type == "production-sanity" or
r.run_type == "package-and-qa" or
r.run_type == "nightly"
)
|> filter(fn: (r) => r.status != "pending" and
r.merge_request == "false" and
r.quarantined == "false" and
r.reliable == "#{reliable}" and
r._field == "id"
)
|> group(columns: ["name"])
QUERY
end
def table(rows, title = nil)
def markdown_section(summary, result, stage, type)
<<~SECTION.strip
```
#{summary_table(summary, type)}
```
## #{stage}
<details>
<summary>Executions table</summary>
```
#{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days")}
```
</details>
SECTION
end
def summary_table(summary, type)
table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50))
end
def table(rows, headings, title)
Terminal::Table.new(
headings: ["name", "runs", "failed", "failure rate"],
headings: headings,
style: { all_separators: true },
title: title,
rows: rows
......@@ -68,7 +99,7 @@ describe QA::Tools::ReliableReport do
def name_column(spec_name)
name = "name: '#{spec_name}'"
file = "file: 'spec.rb'".ljust(110)
file = "file: 'spec.rb'".ljust(160)
"#{name}\n#{file}"
end
......@@ -76,73 +107,85 @@ describe QA::Tools::ReliableReport do
before do
stub_env("QA_INFLUXDB_URL", "url")
stub_env("QA_INFLUXDB_TOKEN", "token")
stub_env("CI_SLACK_WEBHOOK_URL", "slack_url")
stub_env("SLACK_WEBHOOK", "slack_url")
stub_env("CI_API_V4_URL", "gitlab_api_url")
stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token")
allow(RestClient::Request).to receive(:execute).and_return(gitlab_response)
allow(Slack::Notifier).to receive(:new).and_return(slack_notifier)
allow(InfluxDB2::Client).to receive(:new).and_return(influx_client)
allow(query_api).to receive(:query).with(query: query).and_return(runs)
end
context "with stable spec report" do
let(:query) { flux_query(false) }
let(:fetch_message) { "Fetching data on test execution for past #{range} days in '#{run_type}' runs" }
let(:slack_send_message) { "Sending top stable spec report to #{slack_channel} slack channel" }
let(:message_title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" }
let(:table_title) { "Top stable specs in 'manage' stage" }
let(:rows) do
[
[name_column("stable spec"), 3, 0, "0%"],
[name_column("unstable spec"), 3, 2, "66.67%"]
]
end
allow(query_api).to receive(:query).with(query: flux_query(reliable: false)).and_return(runs)
allow(query_api).to receive(:query).with(query: flux_query(reliable: true)).and_return(reliable_runs)
end
it "prints top stable spec report to console" do
expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, table_title)}\n\n").to_stdout
end
context "without report creation" do
let(:create_issue) { "false" }
it "sends top stable spec report to slack" do
slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
it "does not create report issue", :aggregate_failures do
expect { run }.to output.to_stdout
expect { reporter.notify_top_stable }.to output("\n#{slack_send_message}\n#{fetch_message}\n\n").to_stdout
expect(slack_notifier).to have_received(:post).with(text: "*#{message_title}*", **slack_args)
expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows, table_title)}```", **slack_args)
expect(RestClient::Request).not_to have_received(:execute)
expect(slack_notifier).not_to have_received(:post)
end
end
context "with unstable spec report" do
let(:query) { flux_query(true) }
let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" }
let(:slack_send_message) { "Sending top unstable reliable spec report to #{slack_channel} slack channel" }
let(:message_title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" }
let(:table_title) { "Top unstable specs in 'manage' stage" }
let(:rows) { [[name_column("unstable spec"), 3, 2, "66.67%"]] }
context "with report creation" do
let(:create_issue) { "true" }
let(:issue_body) do
<<~TXT.strip
[[_TOC_]]
it "prints top unstable spec report to console" do
expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, table_title)}\n\n").to_stdout
end
# Candidates for promotion to reliable
it "sends top unstable reliable spec report to slack" do
slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
#{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout
expect(slack_notifier).to have_received(:post).with(text: "*#{message_title}*", **slack_args)
expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows, table_title)}```", **slack_args)
# Reliable specs with failures
#{markdown_section([['create', 1]], [[name_column('unstable spec'), 3, 2, '66.67%']], 'create', 'unstable')}
TXT
end
it "creates report issue", :aggregate_failures do
expect { run }.to output.to_stdout
expect(RestClient::Request).to have_received(:execute).with(
method: :post,
url: "gitlab_api_url/projects/278964/issues",
verify_ssl: false,
headers: { "PRIVATE-TOKEN" => "gitlab_token" },
payload: {
title: "Reliable spec report",
description: issue_body,
labels: "Quality,test"
}
)
expect(slack_notifier).to have_received(:post).with(
icon_emoji: ":tanuki-protect:",
text: <<~TEXT
```#{summary_table([['manage', 1]], 'stable')}```
```#{summary_table([['create', 1]], 'unstable')}```
#{issue_url}
TEXT
)
end
end
context "without unstable reliable specs" do
let(:query) { flux_query(true) }
let(:runs) { { 0 => stable_spec } }
let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" }
let(:no_result_message) { "No unstable tests present!" }
context "with failure" do
let(:create_issue) { "true" }
it "prints no result message to console" do
expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout
before do
allow(query_api).to receive(:query).and_raise("Connection error!")
end
it "skips slack notification" do
expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout
expect(slack_notifier).not_to have_received(:post)
it "notifies failure", :aggregate_failures do
expect { expect { run }.to raise_error(SystemExit) }.to output.to_stdout
expect(slack_notifier).to have_received(:post).with(
icon_emoji: ":sadpanda:",
text: "Reliable reporter failed to create report. Error: ```Connection error!```"
)
end
end
end
qa/tasks/reliable_report.rake
View file @ 7c0af33d
......@@ -3,19 +3,8 @@
require_relative "../qa/tools/reliable_report"
desc "Fetch top most reliable specs"
task :reliable_spec_report, [:run_type, :range, :create_slack_report] do |_task, args|
report = QA::Tools::ReliableReport.new(args[:run_type] || "package-and-qa", args[:range])
report.show_top_stable
report.notify_top_stable if args[:create_slack_report] == 'true'
end
desc "Fetch top most unstable reliable specs"
task :unreliable_spec_report, [:run_type, :range, :create_slack_report] do |_task, args|
report = QA::Tools::ReliableReport.new(args[:run_type] || "package-and-qa", args[:range])
report.show_top_unstable
report.notify_top_unstable if args[:create_slack_report] == 'true'
desc "Fetch reliable and unreliable spec data and create report"
task :reliable_spec_report, [:range, :report_in_issue_and_slack] do |_task, args|
QA::Tools::ReliableReport.run(**args)
end
# rubocop:enable Rails/RakeEnvironment
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7