Skip to content

G
gitlab-ce
Commit e5e80b3e authored by Andrejs Cunskis's avatar Andrejs Cunskis
Browse files
Options

E2E: Reliable spec report improvements

parent 9fe7ca49
Show whitespace changes
Inline Side-by-side
Showing with 82 additions and 69 deletions
qa/Gemfile
View file @ e5e80b3e
......@@ -25,7 +25,7 @@ gem 'octokit', '~> 4.21'
gem 'webdrivers', '~> 5.0'
gem 'zeitwerk', '~> 2.4'
gem 'influxdb-client', '~> 1.17'
gem 'terminal-table', '~> 1.8', require: false
gem 'terminal-table', '~> 3.0.0', require: false
gem 'slack-notifier', '~> 2.4', require: false
gem 'fog-google', '~> 1.17', require: false
......
qa/Gemfile.lock
View file @ e5e80b3e
......@@ -112,12 +112,12 @@ GEM
fog-core
nokogiri (>= 1.5.11, < 2.0.0)
formatador (0.3.0)
gitlab (4.16.1)
httparty (~> 0.14, >= 0.14.0)
terminal-table (~> 1.5, >= 1.5.1)
gitlab-qa (7.14.0)
gitlab (4.18.0)
httparty (~> 0.18)
terminal-table (>= 1.5.1)
gitlab-qa (7.17.1)
activesupport (~> 6.1)
gitlab (~> 4.16.1)
gitlab (~> 4.18.0)
http (~> 5.0)
nokogiri (~> 1.10)
table_print (= 1.5.7)
......@@ -184,12 +184,12 @@ GEM
memoizable (0.4.2)
thread_safe (~> 0.3, >= 0.3.1)
method_source (0.9.0)
mime-types (3.4.0)
mime-types (3.4.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2021.1115)
mini_mime (1.1.0)
mini_portile2 (2.6.1)
minitest (5.14.4)
minitest (5.15.0)
multi_json (1.15.0)
multi_xml (0.6.0)
multipart-post (2.1.1)
......@@ -280,8 +280,8 @@ GEM
slack-notifier (2.4.0)
systemu (2.6.5)
table_print (1.5.7)
terminal-table (1.8.0)
unicode-display_width (~> 1.1, >= 1.1.1)
terminal-table (3.0.2)
unicode-display_width (>= 1.1.1, < 3)
thread_safe (0.3.6)
timecop (0.9.1)
trailblazer-option (0.1.2)
......@@ -291,7 +291,7 @@ GEM
unf (0.1.4)
unf_ext
unf_ext (0.0.8)
unicode-display_width (1.8.0)
unicode-display_width (2.1.0)
unparser (0.4.7)
abstract_type (~> 0.0.7)
adamantium (~> 0.2.0)
......@@ -312,7 +312,7 @@ GEM
webrick (1.7.0)
xpath (3.2.0)
nokogiri (~> 1.8)
zeitwerk (2.5.1)
zeitwerk (2.5.2)
PLATFORMS
ruby
......@@ -345,10 +345,10 @@ DEPENDENCIES
ruby-debug-ide (~> 0.7.0)
selenium-webdriver (~> 4.0)
slack-notifier (~> 2.4)
terminal-table (~> 1.8)
terminal-table (~> 3.0.0)
timecop (~> 0.9.1)
webdrivers (~> 5.0)
zeitwerk (~> 2.4)
BUNDLED WITH
2.2.30
2.2.33
qa/qa/tools/reliable_report.rb
View file @ e5e80b3e
......@@ -16,7 +16,7 @@ module QA
PROJECT_ID = 278964
def initialize(range)
@range = range
@range = range.to_i
@influxdb_bucket = "e2e-test-stats"
@slack_channel = "#quality-reports"
@influxdb_url = ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable")
......@@ -34,7 +34,7 @@ module QA
reporter.print_report
reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true"
rescue StandardError => e
reporter.notify_failure(e)
reporter&.notify_failure(e)
raise(e)
end
......@@ -100,67 +100,74 @@ module QA
issue = []
issue << "[[_TOC_]]"
issue << "# Candidates for promotion to reliable #{execution_interval}"
issue << "```\n#{stable_summary_table}\n```"
issue << results_markdown(stable_results_tables)
issue << "Total amount: **#{stable_test_runs.sum { |_k, v| v.count }}**"
issue << stable_summary_table(markdown: true).to_s
issue << results_markdown(:stable)
return issue.join("\n\n") if unstable_reliable_test_runs.empty?
issue << "# Reliable specs with failures #{execution_interval}"
issue << "```\n#{unstable_summary_table}\n```"
issue << results_markdown(unstable_reliable_results_tables)
issue << "Total amount: **#{unstable_reliable_test_runs.sum { |_k, v| v.count }}**"
issue << unstable_summary_table(markdown: true).to_s
issue << results_markdown(:unstable)
issue.join("\n\n")
end
# Stable spec summary table
#
# @param [Boolean] markdown
# @return [Terminal::Table]
def stable_summary_table
@stable_summary_table ||= terminal_table(
def stable_summary_table(markdown: false)
terminal_table(
rows: stable_test_runs.map { |stage, specs| [stage, specs.length] },
title: "Stable spec summary for past #{range} days".ljust(50),
headings: %w[STAGE COUNT]
headings: %w[STAGE COUNT],
markdown: markdown
)
end
# Unstable reliable summary table
#
# @param [Boolean] markdown
# @return [Terminal::Table]
def unstable_summary_table
@unstable_summary_table ||= terminal_table(
def unstable_summary_table(markdown: false)
terminal_table(
rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] },
title: "Unstable spec summary for past #{range} days".ljust(50),
headings: %w[STAGE COUNT]
headings: %w[STAGE COUNT],
markdown: markdown
)
end
# Result tables for stable specs
#
# @param [Boolean] markdown
# @return [Hash]
def stable_results_tables
@stable_results ||= results_tables(:stable)
def stable_results_tables(markdown: false)
results_tables(:stable, markdown: markdown)
end
# Result table for unstable specs
#
# @param [Boolean] markdown
# @return [Hash]
def unstable_reliable_results_tables
@unstable_results ||= results_tables(:unstable)
def unstable_reliable_results_tables(markdown: false)
results_tables(:unstable, markdown: markdown)
end
# Markdown formatted tables
#
# @param [Hash] results
# @param [Symbol] type result type - :stable, :unstable
# @return [String]
def results_markdown(results)
results.map do |stage, table|
def results_markdown(type)
runs = type == :stable ? stable_test_runs : unstable_reliable_test_runs
results_tables(type, markdown: true).map do |stage, table|
<<~STAGE.strip
## #{stage}
## #{stage} (#{runs[stage].count})
<details>
<summary>Executions table</summary>
```
#{table}
```
</details>
STAGE
......@@ -170,15 +177,19 @@ module QA
# Results table
#
# @param [Symbol] type result type - :stable, :unstable
# @param [Boolean] markdown
# @return [Hash<Symbol, Terminal::Table>]
def results_tables(type)
def results_tables(type, markdown: false)
(type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs|
headings = ["name", "runs", "failures", "failure rate"]
[stage, terminal_table(
rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
title: "Top #{type} specs in '#{stage}' stage for past #{range} days",
headings: headings.map(&:upcase)
headings: headings.map(&:upcase),
markdown: markdown,
rows: specs.map do |k, v|
[name_column(name: k, file: v[:file], markdown: markdown), *table_params(v.values)]
end
)]
end
end
......@@ -217,13 +228,17 @@ module QA
# Terminal table for result formatting
#
# @param [Array] rows
# @param [Array] headings
# @param [String] title
# @param [Boolean] markdown
# @return [Terminal::Table]
def terminal_table(rows:, headings:, title: nil)
def terminal_table(rows:, headings:, title:, markdown:)
Terminal::Table.new(
headings: headings,
style: { all_separators: true },
title: title,
rows: rows
title: markdown ? nil : title,
rows: rows,
style: markdown ? { border: :markdown } : { all_separators: true }
)
end
......@@ -235,17 +250,17 @@ module QA
[*parameters[1..2], "#{parameters.last}%"]
end
# Name column value
# Name column content
#
# @param [String] name
# @param [String] file
# @param [Boolean] markdown
# @return [String]
def name_column(name, file)
spec_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name
name_line = "name: '#{spec_name}'"
file_line = "file: '#{file}'"
def name_column(name:, file:, markdown: false)
return "**name**: #{name}<br>**file**: #{file}" if markdown
"#{name_line}\n#{file_line.ljust(160)}"
wrapped_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name
"name: '#{wrapped_name}'\nfile: #{file.ljust(160)}"
end
# Test executions grouped by name
......@@ -258,9 +273,10 @@ module QA
all_runs = query_api.query(query: query(reliable)).values
all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
records = table.records
# skip specs that executed less time than defined by range
# skip specs that executed less time than defined by range or stopped executing before report date
# offset 1 day due to how schedulers are configured and first run can be 1 day later
next if (Date.today - Date.parse(records.first.values["_time"])).to_i < (range - 1)
next if (Date.today - Date.parse(records.last.values["_time"])).to_i > 1
last_record = records.last.values
name = last_record["name"]
......
qa/spec/tools/reliable_report_spec.rb
View file @ e5e80b3e
......@@ -29,7 +29,7 @@ describe QA::Tools::ReliableReport do
records: [
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values)
instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s }))
]
)
}
......@@ -49,7 +49,7 @@ describe QA::Tools::ReliableReport do
records: [
instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }),
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values)
instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s }))
]
)
}
......@@ -80,41 +80,34 @@ describe QA::Tools::ReliableReport do
def markdown_section(summary, result, stage, type)
<<~SECTION.strip
```
#{summary_table(summary, type)}
```
#{summary_table(summary, type, true)}
## #{stage}
## #{stage} (1)
<details>
<summary>Executions table</summary>
```
#{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days")}
```
#{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)}
</details>
SECTION
end
def summary_table(summary, type)
table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50))
def summary_table(summary, type, markdown = false)
table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown)
end
def table(rows, headings, title)
def table(rows, headings, title, markdown = false)
Terminal::Table.new(
headings: headings,
style: { all_separators: true },
title: title,
rows: rows
title: markdown ? nil : title,
rows: rows,
style: markdown ? { border: :markdown } : { all_separators: true }
)
end
def name_column(spec_name)
name = "name: '#{spec_name}'"
file = "file: 'spec.rb'".ljust(160)
"#{name}\n#{file}"
"**name**: #{spec_name}<br>**file**: spec.rb"
end
before do
......@@ -151,10 +144,14 @@ describe QA::Tools::ReliableReport do
# Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
Total amount: **1**
#{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
# Reliable specs with failures (#{Date.today - range} - #{Date.today})
Total amount: **1**
#{markdown_section([['create', 1]], [[name_column('unstable spec'), 3, 2, '66.67%']], 'create', 'unstable')}
TXT
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7