Commit 1d205e81 authored by Andrejs Cunskis's avatar Andrejs Cunskis Committed by Mark Lapierre

E2E: Improve reliable|unreliable spec report

parent 64cf6829
......@@ -167,7 +167,7 @@ module QA
# @param [String] location
# @return [String, nil]
def devops_stage(file_path)
file_path.match(%r{(\d{1,2}_\w+)/})&.captures&.first
file_path.match(%r{\d{1,2}_(\w+)/})&.captures&.first
end
end
end
......
......@@ -8,7 +8,7 @@ module QA
module Tools
class ReliableReport
def initialize(run_type, range = 30)
@results = 10
@results = 2
@slack_channel = "#quality-reports"
@range = range
@run_type = run_type
......@@ -20,27 +20,17 @@ module QA
#
# @return [void]
def show_top_stable
puts terminal_table(
rows: top_stable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
title: stable_title
)
results_table(:stable).each { |table| puts "#{table}\n\n" }
end
# Post top stable spec report to slack
# Slice table in to multiple messages due to max char limitation
#
# @return [void]
def notify_top_stable
tables = top_stable.each_slice(5).map do |slice|
terminal_table(
rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] }
)
end
puts "\nSending top stable spec report to #{slack_channel} slack channel"
slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
notifier.post(text: "*#{stable_title}*", **slack_args)
tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) }
results_table(:stable).each { |table| notifier.post(text: "```#{table}```", **slack_args) }
end
# Print top unstable specs
......@@ -49,29 +39,19 @@ module QA
def show_top_unstable
return puts("No unstable tests present!") if top_unstable_reliable.empty?
puts terminal_table(
rows: top_unstable_reliable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
title: unstable_title
)
results_table(:unstable).each { |table| puts "#{table}\n\n" }
end
# Post top unstable reliable spec report to slack
# Slice table in to multiple messages due to max char limitation
#
# @return [void]
def notify_top_unstable
return puts("No unstable tests present!") if top_unstable_reliable.empty?
tables = top_unstable_reliable.each_slice(5).map do |slice|
terminal_table(
rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] }
)
end
puts "\nSending top unstable reliable spec report to #{slack_channel} slack channel"
slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
notifier.post(text: "*#{unstable_title}*", **slack_args)
tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) }
results_table(:unstable).each { |table| notifier.post(text: "```#{table}```", **slack_args) }
end
private
......@@ -83,21 +63,42 @@ module QA
:stable_title,
:unstable_title
# Results table
#
# @param [Symbol] type result type - :stable, :unstable
# @return [Hash]
def results_table(type)
(type == :stable ? top_stable : top_unstable_reliable).map do |stage, specs|
terminal_table(
rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
title: "Top #{type} specs in '#{stage}' stage"
)
end
end
# Top stable specs
#
# @return [Hash]
def top_stable
@top_stable ||= runs(reliable: false).sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h
@top_stable ||= runs(reliable: false).transform_values do |specs|
specs.sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h
end
end
# Top unstable reliable specs
#
# @return [Hash]
def top_unstable_reliable
@top_unstable_reliable ||= runs(reliable: true)
.reject { |k, v| v[:failure_rate] == 0 }
.sort_by { |k, v| -v[:failure_rate] }[0..results - 1]
.to_h
@top_unstable_reliable ||= begin
unstable = runs(reliable: true).transform_values do |specs|
specs
.reject { |k, v| v[:failure_rate] == 0 }
.sort_by { |k, v| -v[:failure_rate] }[0..results - 1]
.to_h
end
unstable.reject { |k, v| v.empty? }
end
end
# Terminal table for result formatting
......@@ -136,20 +137,23 @@ module QA
# Test executions grouped by name
#
# @param [Boolean] reliable
# @return [Hash]
# @return [Hash<String, Hash>]
def runs(reliable:)
puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past 30 days in '#{run_type}' runs")
puts
query_api.query(query: query(reliable)).values.each_with_object({}) do |table, result|
all_runs = query_api.query(query: query(reliable)).values
all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
records = table.records
name = records.last.values["name"]
file = records.last.values["file_path"].split("/").last
stage = records.last.values["stage"] || "unknown"
runs = records.count
failed = records.count { |r| r.values["status"] == "failed" }
failure_rate = (failed.to_f / runs.to_f) * 100
result[name] = {
result[stage][name] = {
file: file,
runs: runs,
failed: failed,
......
......@@ -45,7 +45,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
job_name: "test-job",
merge_request: "false",
run_type: run_type,
stage: stage
stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first
},
fields: {
id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',
......
......@@ -12,30 +12,31 @@ describe QA::Tools::ReliableReport do
let(:slack_channel) { "#quality-reports" }
let(:run_type) { "package-and-qa" }
let(:range) { 30 }
let(:results) { 10 }
let(:results) { 2 }
let(:runs) { { 0 => stable_spec, 1 => unstable_spec } }
let(:spec_values) { { "file_path" => "some/spec.rb", "stage" => "manage" } }
let(:stable_spec) do
spec_values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb" }
values = { "name" => "stable spec", "status" => "passed", **spec_values }
instance_double(
"InfluxDB2::FluxTable",
records: [
instance_double("InfluxDB2::FluxRecord", values: spec_values),
instance_double("InfluxDB2::FluxRecord", values: spec_values),
instance_double("InfluxDB2::FluxRecord", values: spec_values)
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values)
]
)
end
let(:unstable_spec) do
spec_values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb" }
values = { "name" => "unstable spec", "status" => "failed", **spec_values }
instance_double(
"InfluxDB2::FluxTable",
records: [
instance_double("InfluxDB2::FluxRecord", values: { **spec_values, "status" => "passed" }),
instance_double("InfluxDB2::FluxRecord", values: spec_values),
instance_double("InfluxDB2::FluxRecord", values: spec_values)
instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }),
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values)
]
)
end
......@@ -86,7 +87,8 @@ describe QA::Tools::ReliableReport do
let(:query) { flux_query(false) }
let(:fetch_message) { "Fetching data on test execution for past #{range} days in '#{run_type}' runs" }
let(:slack_send_message) { "Sending top stable spec report to #{slack_channel} slack channel" }
let(:title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" }
let(:message_title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" }
let(:table_title) { "Top stable specs in 'manage' stage" }
let(:rows) do
[
[name_column("stable spec"), 3, 0, "0%"],
......@@ -95,15 +97,15 @@ describe QA::Tools::ReliableReport do
end
it "prints top stable spec report to console" do
expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout
expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, table_title)}\n\n").to_stdout
end
it "sends top stable spec report to slack" do
slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
expect { reporter.notify_top_stable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout
expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args)
expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args)
expect { reporter.notify_top_stable }.to output("\n#{slack_send_message}\n#{fetch_message}\n\n").to_stdout
expect(slack_notifier).to have_received(:post).with(text: "*#{message_title}*", **slack_args)
expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows, table_title)}```", **slack_args)
end
end
......@@ -111,19 +113,20 @@ describe QA::Tools::ReliableReport do
let(:query) { flux_query(true) }
let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" }
let(:slack_send_message) { "Sending top unstable reliable spec report to #{slack_channel} slack channel" }
let(:title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" }
let(:message_title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" }
let(:table_title) { "Top unstable specs in 'manage' stage" }
let(:rows) { [[name_column("unstable spec"), 3, 2, "66.67%"]] }
it "prints top unstable spec report to console" do
expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout
expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, table_title)}\n\n").to_stdout
end
it "sends top unstable reliable spec report to slack" do
slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout
expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args)
expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args)
expect(slack_notifier).to have_received(:post).with(text: "*#{message_title}*", **slack_args)
expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows, table_title)}```", **slack_args)
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment