Merge branch 'acunskis-report-filter-date' into 'master'

E2E: Filter out new spec executions from reliable spec report

See merge request gitlab-org/gitlab!77096

qa/qa/tools/reliable_report.rb

@@ -34,9 +34,8 @@ module QA
         reporter.print_report
         reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true"
       rescue StandardError => e
-        puts "Report creation failed! Error: '#{e}'".colorize(:red)
         reporter.notify_failure(e)
-        exit(1)
+        raise(e)
       end
 
       # Print top stable specs
@@ -96,13 +95,17 @@ module QA
       #
       # @return [String]
       def report_issue_body
+        execution_interval = "(#{Date.today - range} - #{Date.today})"
+
         issue = []
         issue << "[[_TOC_]]"
-        issue << "# Candidates for promotion to reliable\n\n```\n#{stable_summary_table}\n```"
+        issue << "# Candidates for promotion to reliable #{execution_interval}"
+        issue << "```\n#{stable_summary_table}\n```"
         issue << results_markdown(stable_results_tables)
         return issue.join("\n\n") if unstable_reliable_test_runs.empty?
 
-        issue << "# Reliable specs with failures\n\n```\n#{unstable_summary_table}\n```"
+        issue << "# Reliable specs with failures #{execution_interval}"
+        issue << "```\n#{unstable_summary_table}\n```"
         issue << results_markdown(unstable_reliable_results_tables)
         issue.join("\n\n")
       end
@@ -255,9 +258,14 @@ module QA
         all_runs = query_api.query(query: query(reliable)).values
         all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
           records = table.records
-          name = records.last.values["name"]
-          file = records.last.values["file_path"].split("/").last
-          stage = records.last.values["stage"] || "unknown"
+          # skip specs that executed less time than defined by range
+          # offset 1 day due to how schedulers are configured and first run can be 1 day later
+          next if (Date.today - Date.parse(records.first.values["_time"])).to_i < (range - 1)
+
+          last_record = records.last.values
+          name = last_record["name"]
+          file = last_record["file_path"].split("/").last
+          stage = last_record["stage"] || "unknown"
 
           runs = records.count
           failed = records.count { |r| r.values["status"] == "failed" }

qa/spec/tools/reliable_report_spec.rb

@@ -13,9 +13,16 @@ describe QA::Tools::ReliableReport do
   let(:slack_channel) { "#quality-reports" }
   let(:range) { 14 }
   let(:issue_url) { "https://gitlab.com/issue/1" }
+  let(:time) { "2021-12-07T04:05:25.000000000+00:00" }
 
   let(:runs) do
-    values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb", "stage" => "manage" }
+    values = {
+      "name" => "stable spec",
+      "status" => "passed",
+      "file_path" => "some/spec.rb",
+      "stage" => "manage",
+      "_time" => time
+    }
     {
       0 => instance_double(
         "InfluxDB2::FluxTable",
@@ -29,7 +36,13 @@ describe QA::Tools::ReliableReport do
   end
 
   let(:reliable_runs) do
-    values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb", "stage" => "create" }
+    values = {
+      "name" => "unstable spec",
+      "status" => "failed",
+      "file_path" => "some/spec.rb",
+      "stage" => "create",
+      "_time" => time
+    }
     {
       0 => instance_double(
         "InfluxDB2::FluxTable",
@@ -136,11 +149,11 @@ describe QA::Tools::ReliableReport do
       <<~TXT.strip
         [[_TOC_]]
 
-        # Candidates for promotion to reliable
+        # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
 
         #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
 
-        # Reliable specs with failures
+        # Reliable specs with failures (#{Date.today - range} - #{Date.today})
 
         #{markdown_section([['create', 1]], [[name_column('unstable spec'), 3, 2, '66.67%']], 'create', 'unstable')}
       TXT
@@ -180,7 +193,7 @@ describe QA::Tools::ReliableReport do
     end
 
     it "notifies failure", :aggregate_failures do
-      expect { expect { run }.to raise_error(SystemExit) }.to output.to_stdout
+      expect { expect { run }.to raise_error("Connection error!") }.to output.to_stdout
 
       expect(slack_notifier).to have_received(:post).with(
         icon_emoji: ":sadpanda:",