Merge branch 'acunskis-reliable-report' into 'master'

E2E: Update reliable spec report creation

See merge request gitlab-org/gitlab!76335

qa/qa/tools/reliable_report.rb

 # frozen_string_literal: true
 
+require_relative "../../qa"
+
 require "influxdb-client"
 require "terminal-table"
 require "slack-notifier"
+require "colorize"
 
 module QA
   module Tools
     class ReliableReport
-      def initialize(run_type, range = 30)
-        @results = 2
-        @slack_channel = "#quality-reports"
+      include Support::API
+
+      # Project for report creation: https://gitlab.com/gitlab-org/gitlab
+      PROJECT_ID = 278964
+
+      def initialize(range)
         @range = range
-        @run_type = run_type
-        @stable_title = "Top #{results} stable specs for past #{@range} days in '#{run_type}' runs"
-        @unstable_title = "Top #{results} unstable reliable specs for past #{@range} days in '#{run_type}' runs"
+        @influxdb_bucket = "e2e-test-stats"
+        @slack_channel = "#quality-reports"
+        @influxdb_url = ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable")
+        @influxdb_token = ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN env variable")
       end
 
-      # Print top stable specs
+      # Run reliable reporter
       #
+      # @param [Integer] range amount of days for results range
+      # @param [String] report_in_issue_and_slack
       # @return [void]
-      def show_top_stable
-        results_table(:stable).each { |table| puts "#{table}\n\n" }
+      def self.run(range: 14, report_in_issue_and_slack: "false")
+        reporter = new(range)
+
+        reporter.print_report
+        reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true"
+      rescue StandardError => e
+        puts "Report creation failed! Error: '#{e}'".colorize(:red)
+        reporter.notify_failure(e)
+        exit(1)
       end
 
-      # Post top stable spec report to slack
+      # Print top stable specs
       #
       # @return [void]
-      def notify_top_stable
-        puts "\nSending top stable spec report to #{slack_channel} slack channel"
-        slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
-        notifier.post(text: "*#{stable_title}*", **slack_args)
-        results_table(:stable).each { |table| notifier.post(text: "```#{table}```", **slack_args) }
+      def print_report
+        puts "#{stable_summary_table}\n\n"
+        stable_results_tables.each { |stage, table| puts "#{table}\n\n" }
+        return puts("No unstable reliable tests present!".colorize(:yellow)) if unstable_reliable_test_runs.empty?
+
+        puts "#{unstable_summary_table}\n\n"
+        unstable_reliable_results_tables.each { |stage, table| puts "#{table}\n\n" }
       end
 
-      # Print top unstable specs
+      # Create report issue
       #
       # @return [void]
-      def show_top_unstable
-        return puts("No unstable tests present!") if top_unstable_reliable.empty?
+      def report_in_issue_and_slack
+        puts "Creating report".colorize(:green)
+        response = post(
+          "#{gitlab_api_url}/projects/#{PROJECT_ID}/issues",
+          { title: "Reliable spec report", description: report_issue_body, labels: "Quality,test" },
+          headers: { "PRIVATE-TOKEN" => gitlab_access_token }
+        )
+        web_url = parse_body(response)[:web_url]
+        puts "Created report issue: #{web_url}"
 
-        results_table(:unstable).each { |table| puts "#{table}\n\n" }
+        puts "Sending slack notification".colorize(:green)
+        notifier.post(
+          icon_emoji: ":tanuki-protect:",
+          text: <<~TEXT
+            ```#{stable_summary_table}```
+            ```#{unstable_summary_table}```
+
+            #{web_url}
+          TEXT
+        )
+        puts "Done!"
       end
 
-      # Post top unstable reliable spec report to slack
+      # Notify failure
       #
+      # @param [StandardError] error
       # @return [void]
-      def notify_top_unstable
-        return puts("No unstable tests present!") if top_unstable_reliable.empty?
-
-        puts "\nSending top unstable reliable spec report to #{slack_channel} slack channel"
-        slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
-        notifier.post(text: "*#{unstable_title}*", **slack_args)
-        results_table(:unstable).each { |table| notifier.post(text: "```#{table}```", **slack_args) }
+      def notify_failure(error)
+        notifier.post(
+          text: "Reliable reporter failed to create report. Error: ```#{error}```",
+          icon_emoji: ":sadpanda:"
+        )
       end
 
       private
 
-      attr_reader :results,
-                  :slack_channel,
-                  :range,
-                  :run_type,
-                  :stable_title,
-                  :unstable_title
+      attr_reader :range, :influxdb_bucket, :slack_channel, :influxdb_url, :influxdb_token
+
+      # Markdown formatted report issue body
+      #
+      # @return [String]
+      def report_issue_body
+        issue = []
+        issue << "[[_TOC_]]"
+        issue << "# Candidates for promotion to reliable\n\n```\n#{stable_summary_table}\n```"
+        issue << results_markdown(stable_results_tables)
+        return issue.join("\n\n") if unstable_reliable_test_runs.empty?
+
+        issue << "# Reliable specs with failures\n\n```\n#{unstable_summary_table}\n```"
+        issue << results_markdown(unstable_reliable_results_tables)
+        issue.join("\n\n")
+      end
+
+      # Stable spec summary table
+      #
+      # @return [Terminal::Table]
+      def stable_summary_table
+        @stable_summary_table ||= terminal_table(
+          rows: stable_test_runs.map { |stage, specs| [stage, specs.length] },
+          title: "Stable spec summary for past #{range} days".ljust(50),
+          headings: %w[STAGE COUNT]
+        )
+      end
+
+      # Unstable reliable summary table
+      #
+      # @return [Terminal::Table]
+      def unstable_summary_table
+        @unstable_summary_table ||= terminal_table(
+          rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] },
+          title: "Unstable spec summary for past #{range} days".ljust(50),
+          headings: %w[STAGE COUNT]
+        )
+      end
+
+      # Result tables for stable specs
+      #
+      # @return [Hash]
+      def stable_results_tables
+        @stable_results ||= results_tables(:stable)
+      end
+
+      # Result table for unstable specs
+      #
+      # @return [Hash]
+      def unstable_reliable_results_tables
+        @unstable_results ||= results_tables(:unstable)
+      end
+
+      # Markdown formatted tables
+      #
+      # @param [Hash] results
+      # @return [String]
+      def results_markdown(results)
+        results.map do |stage, table|
+          <<~STAGE.strip
+            ## #{stage}
+
+            <details>
+            <summary>Executions table</summary>
+
+            ```
+            #{table}
+            ```
+
+            </details>
+          STAGE
+        end.join("\n\n")
+      end
 
       # Results table
       #
       # @param [Symbol] type result type - :stable, :unstable
-      # @return [Hash]
-      def results_table(type)
-        (type == :stable ? top_stable : top_unstable_reliable).map do |stage, specs|
-          terminal_table(
+      # @return [Hash<Symbol, Terminal::Table>]
+      def results_tables(type)
+        (type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs|
+          headings = ["name", "runs", "failures", "failure rate"]
+
+          [stage, terminal_table(
             rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
-            title: "Top #{type} specs in '#{stage}' stage"
-          )
+            title: "Top #{type} specs in '#{stage}' stage for past #{range} days",
+            headings: headings.map(&:upcase)
+          )]
         end
       end
 
-      # Top stable specs
+      # Stable specs
       #
       # @return [Hash]
-      def top_stable
-        @top_stable ||= runs(reliable: false).transform_values do |specs|
-          specs.sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h
+      def stable_test_runs
+        @top_stable ||= begin
+          stable_specs = test_runs(reliable: false).transform_values do |specs|
+            specs
+              .reject { |k, v| v[:failure_rate] != 0 }
+              .sort_by { |k, v| -v[:runs] }
+              .to_h
+          end
+
+          stable_specs.reject { |k, v| v.empty? }
         end
       end
 
-      # Top unstable reliable specs
+      # Unstable reliable specs
       #
       # @return [Hash]
-      def top_unstable_reliable
+      def unstable_reliable_test_runs
         @top_unstable_reliable ||= begin
-          unstable = runs(reliable: true).transform_values do |specs|
+          unstable = test_runs(reliable: true).transform_values do |specs|
             specs
               .reject { |k, v| v[:failure_rate] == 0 }
-              .sort_by { |k, v| -v[:failure_rate] }[0..results - 1]
+              .sort_by { |k, v| -v[:failure_rate] }
               .to_h
           end
 
@@ -104,9 +215,9 @@ module QA
       # Terminal table for result formatting
       #
       # @return [Terminal::Table]
-      def terminal_table(rows:, title: nil)
+      def terminal_table(rows:, headings:, title: nil)
         Terminal::Table.new(
-          headings: ["name", "runs", "failed", "failure rate"],
+          headings: headings,
           style: { all_separators: true },
           title: title,
           rows: rows
@@ -127,20 +238,19 @@ module QA
       # @param [String] file
       # @return [String]
       def name_column(name, file)
-        spec_name = name.length > 100 ? "#{name} ".scan(/.{1,100} /).map(&:strip).join("\n") : name
+        spec_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name
         name_line = "name: '#{spec_name}'"
         file_line = "file: '#{file}'"
 
-        "#{name_line}\n#{file_line.ljust(110)}"
+        "#{name_line}\n#{file_line.ljust(160)}"
       end
 
       # Test executions grouped by name
       #
       # @param [Boolean] reliable
       # @return [Hash<String, Hash>]
-      def runs(reliable:)
-        puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past 30 days in '#{run_type}' runs")
-        puts
+      def test_runs(reliable:)
+        puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past #{range} days\n".colorize(:green))
 
         all_runs = query_api.query(query: query(reliable)).values
         all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
@@ -168,17 +278,24 @@ module QA
       # @return [String]
       def query(reliable)
         <<~QUERY
-        from(bucket: "e2e-test-stats")
-          |> range(start: -#{range}d)
-          |> filter(fn: (r) => r._measurement == "test-stats" and
-            r.run_type == "#{run_type}" and
-            r.status != "pending" and
-            r.merge_request == "false" and
-            r.quarantined == "false" and
-            r.reliable == "#{reliable}" and
-            r._field == "id"
-          )
-          |> group(columns: ["name"])
+          from(bucket: "#{influxdb_bucket}")
+            |> range(start: -#{range}d)
+            |> filter(fn: (r) => r._measurement == "test-stats")
+            |> filter(fn: (r) => r.run_type == "staging-full" or
+              r.run_type == "staging-sanity" or
+              r.run_type == "staging-sanity-no-admin" or
+              r.run_type == "production-full" or
+              r.run_type == "production-sanity" or
+              r.run_type == "package-and-qa" or
+              r.run_type == "nightly"
+            )
+            |> filter(fn: (r) => r.status != "pending" and
+              r.merge_request == "false" and
+              r.quarantined == "false" and
+              r.reliable == "#{reliable}" and
+              r._field == "id"
+            )
+            |> group(columns: ["name"])
         QUERY
       end
 
@@ -196,7 +313,7 @@ module QA
         @influx_client ||= InfluxDB2::Client.new(
           influxdb_url,
           influxdb_token,
-          bucket: "e2e-test-stats",
+          bucket: influxdb_bucket,
           org: "gitlab-qa",
           precision: InfluxDB2::WritePrecision::NANOSECOND
         )
@@ -209,29 +326,29 @@ module QA
         @notifier ||= Slack::Notifier.new(
           slack_webhook_url,
           channel: slack_channel,
-          username: "Reliable spec reporter"
+          username: "Reliable Spec Report"
         )
       end
 
-      # InfluxDb instance url
+      # Gitlab access token
       #
       # @return [String]
-      def influxdb_url
-        @influxdb_url ||= ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL environment variable")
+      def gitlab_access_token
+        @gitlab_access_token ||= ENV["GITLAB_ACCESS_TOKEN"] || raise("Missing GITLAB_ACCESS_TOKEN env variable")
       end
 
-      # Influxdb token
+      # Gitlab api url
       #
       # @return [String]
-      def influxdb_token
-        @influxdb_token ||= ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN environment variable")
+      def gitlab_api_url
+        @gitlab_api_url ||= ENV["CI_API_V4_URL"] || raise("Missing CI_API_V4_URL env variable")
       end
 
       # Slack webhook url
       #
       # @return [String]
       def slack_webhook_url
-        @slack_webhook_url ||= ENV["CI_SLACK_WEBHOOK_URL"] || raise("Missing CI_SLACK_WEBHOOK_URL environment variable")
+        @slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable")
       end
     end
   end

qa/spec/tools/reliable_report_spec.rb

@@ -3,63 +3,94 @@
 describe QA::Tools::ReliableReport do
   include QA::Support::Helpers::StubEnv
 
-  subject(:reporter) { described_class.new(run_type, range) }
+  subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) }
 
+  let(:gitlab_response) { instance_double("RestClient::Response", code: 200, body: { web_url: issue_url }.to_json) }
   let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) }
   let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) }
   let(:query_api) { instance_double("InfluxDB2::QueryApi") }
 
   let(:slack_channel) { "#quality-reports" }
-  let(:run_type) { "package-and-qa" }
-  let(:range) { 30 }
-  let(:results) { 2 }
-
-  let(:runs) { { 0 => stable_spec, 1 => unstable_spec } }
-
-  let(:spec_values) { { "file_path" => "some/spec.rb", "stage" => "manage" } }
-  let(:stable_spec) do
-    values = { "name" => "stable spec", "status" => "passed", **spec_values }
-    instance_double(
-      "InfluxDB2::FluxTable",
-      records: [
-        instance_double("InfluxDB2::FluxRecord", values: values),
-        instance_double("InfluxDB2::FluxRecord", values: values),
-        instance_double("InfluxDB2::FluxRecord", values: values)
-      ]
-    )
+  let(:range) { 14 }
+  let(:issue_url) { "https://gitlab.com/issue/1" }
+
+  let(:runs) do
+    values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb", "stage" => "manage" }
+    {
+      0 => instance_double(
+        "InfluxDB2::FluxTable",
+        records: [
+          instance_double("InfluxDB2::FluxRecord", values: values),
+          instance_double("InfluxDB2::FluxRecord", values: values),
+          instance_double("InfluxDB2::FluxRecord", values: values)
+        ]
+      )
+    }
   end
 
-  let(:unstable_spec) do
-    values = { "name" => "unstable spec", "status" => "failed", **spec_values }
-    instance_double(
-      "InfluxDB2::FluxTable",
-      records: [
-        instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }),
-        instance_double("InfluxDB2::FluxRecord", values: values),
-        instance_double("InfluxDB2::FluxRecord", values: values)
-      ]
-    )
+  let(:reliable_runs) do
+    values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb", "stage" => "create" }
+    {
+      0 => instance_double(
+        "InfluxDB2::FluxTable",
+        records: [
+          instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }),
+          instance_double("InfluxDB2::FluxRecord", values: values),
+          instance_double("InfluxDB2::FluxRecord", values: values)
+        ]
+      )
+    }
   end
 
-  def flux_query(reliable)
+  def flux_query(reliable:)
     <<~QUERY
-        from(bucket: "e2e-test-stats")
-          |> range(start: -#{range}d)
-          |> filter(fn: (r) => r._measurement == "test-stats" and
-            r.run_type == "#{run_type}" and
-            r.status != "pending" and
-            r.merge_request == "false" and
-            r.quarantined == "false" and
-            r.reliable == "#{reliable}" and
-            r._field == "id"
-          )
-          |> group(columns: ["name"])
+      from(bucket: "e2e-test-stats")
+        |> range(start: -#{range}d)
+        |> filter(fn: (r) => r._measurement == "test-stats")
+        |> filter(fn: (r) => r.run_type == "staging-full" or
+          r.run_type == "staging-sanity" or
+          r.run_type == "staging-sanity-no-admin" or
+          r.run_type == "production-full" or
+          r.run_type == "production-sanity" or
+          r.run_type == "package-and-qa" or
+          r.run_type == "nightly"
+        )
+        |> filter(fn: (r) => r.status != "pending" and
+          r.merge_request == "false" and
+          r.quarantined == "false" and
+          r.reliable == "#{reliable}" and
+          r._field == "id"
+        )
+        |> group(columns: ["name"])
     QUERY
   end
 
-  def table(rows, title = nil)
+  def markdown_section(summary, result, stage, type)
+    <<~SECTION.strip
+      ```
+      #{summary_table(summary, type)}
+      ```
+
+      ## #{stage}
+
+      <details>
+      <summary>Executions table</summary>
+
+      ```
+      #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days")}
+      ```
+
+      </details>
+    SECTION
+  end
+
+  def summary_table(summary, type)
+    table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50))
+  end
+
+  def table(rows, headings, title)
     Terminal::Table.new(
-      headings: ["name", "runs", "failed", "failure rate"],
+      headings: headings,
       style: { all_separators: true },
       title: title,
       rows: rows
@@ -68,7 +99,7 @@ describe QA::Tools::ReliableReport do
 
   def name_column(spec_name)
     name = "name: '#{spec_name}'"
-    file = "file: 'spec.rb'".ljust(110)
+    file = "file: 'spec.rb'".ljust(160)
 
     "#{name}\n#{file}"
   end
@@ -76,73 +107,85 @@ describe QA::Tools::ReliableReport do
   before do
     stub_env("QA_INFLUXDB_URL", "url")
     stub_env("QA_INFLUXDB_TOKEN", "token")
-    stub_env("CI_SLACK_WEBHOOK_URL", "slack_url")
+    stub_env("SLACK_WEBHOOK", "slack_url")
+    stub_env("CI_API_V4_URL", "gitlab_api_url")
+    stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token")
 
+    allow(RestClient::Request).to receive(:execute).and_return(gitlab_response)
     allow(Slack::Notifier).to receive(:new).and_return(slack_notifier)
     allow(InfluxDB2::Client).to receive(:new).and_return(influx_client)
-    allow(query_api).to receive(:query).with(query: query).and_return(runs)
-  end
 
-  context "with stable spec report" do
-    let(:query) { flux_query(false) }
-    let(:fetch_message) { "Fetching data on test execution for past #{range} days in '#{run_type}' runs" }
-    let(:slack_send_message) { "Sending top stable spec report to #{slack_channel} slack channel" }
-    let(:message_title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" }
-    let(:table_title) { "Top stable specs in 'manage' stage" }
-    let(:rows) do
-      [
-        [name_column("stable spec"), 3, 0, "0%"],
-        [name_column("unstable spec"), 3, 2, "66.67%"]
-      ]
-    end
+    allow(query_api).to receive(:query).with(query: flux_query(reliable: false)).and_return(runs)
+    allow(query_api).to receive(:query).with(query: flux_query(reliable: true)).and_return(reliable_runs)
+  end
 
-    it "prints top stable spec report to console" do
-      expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, table_title)}\n\n").to_stdout
-    end
+  context "without report creation" do
+    let(:create_issue) { "false" }
 
-    it "sends top stable spec report to slack" do
-      slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
+    it "does not create report issue", :aggregate_failures do
+      expect { run }.to output.to_stdout
 
-      expect { reporter.notify_top_stable }.to output("\n#{slack_send_message}\n#{fetch_message}\n\n").to_stdout
-      expect(slack_notifier).to have_received(:post).with(text: "*#{message_title}*", **slack_args)
-      expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows, table_title)}```", **slack_args)
+      expect(RestClient::Request).not_to have_received(:execute)
+      expect(slack_notifier).not_to have_received(:post)
     end
   end
 
-  context "with unstable spec report" do
-    let(:query) { flux_query(true) }
-    let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" }
-    let(:slack_send_message) { "Sending top unstable reliable spec report to #{slack_channel} slack channel" }
-    let(:message_title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" }
-    let(:table_title) { "Top unstable specs in 'manage' stage" }
-    let(:rows) { [[name_column("unstable spec"), 3, 2, "66.67%"]] }
+  context "with report creation" do
+    let(:create_issue) { "true" }
+    let(:issue_body) do
+      <<~TXT.strip
+        [[_TOC_]]
 
-    it "prints top unstable spec report to console" do
-      expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, table_title)}\n\n").to_stdout
-    end
+        # Candidates for promotion to reliable
 
-    it "sends top unstable reliable spec report to slack" do
-      slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
+        #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
 
-      expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout
-      expect(slack_notifier).to have_received(:post).with(text: "*#{message_title}*", **slack_args)
-      expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows, table_title)}```", **slack_args)
+        # Reliable specs with failures
+
+        #{markdown_section([['create', 1]], [[name_column('unstable spec'), 3, 2, '66.67%']], 'create', 'unstable')}
+      TXT
+    end
+
+    it "creates report issue", :aggregate_failures do
+      expect { run }.to output.to_stdout
+
+      expect(RestClient::Request).to have_received(:execute).with(
+        method: :post,
+        url: "gitlab_api_url/projects/278964/issues",
+        verify_ssl: false,
+        headers: { "PRIVATE-TOKEN" => "gitlab_token" },
+        payload: {
+          title: "Reliable spec report",
+          description: issue_body,
+          labels: "Quality,test"
+        }
+      )
+      expect(slack_notifier).to have_received(:post).with(
+        icon_emoji: ":tanuki-protect:",
+        text: <<~TEXT
+          ```#{summary_table([['manage', 1]], 'stable')}```
+          ```#{summary_table([['create', 1]], 'unstable')}```
+
+          #{issue_url}
+        TEXT
+      )
     end
   end
 
-  context "without unstable reliable specs" do
-    let(:query) { flux_query(true) }
-    let(:runs) { { 0 => stable_spec } }
-    let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" }
-    let(:no_result_message) { "No unstable tests present!" }
+  context "with failure" do
+    let(:create_issue) { "true" }
 
-    it "prints no result message to console" do
-      expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout
+    before do
+      allow(query_api).to receive(:query).and_raise("Connection error!")
     end
 
-    it "skips slack notification" do
-      expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout
-      expect(slack_notifier).not_to have_received(:post)
+    it "notifies failure", :aggregate_failures do
+      expect { expect { run }.to raise_error(SystemExit) }.to output.to_stdout
+
+      expect(slack_notifier).to have_received(:post).with(
+        icon_emoji: ":sadpanda:",
+        text: "Reliable reporter failed to create report. Error: ```Connection error!```"
+      )
     end
   end
 end

qa/tasks/reliable_report.rake

@@ -3,19 +3,8 @@
 
 require_relative "../qa/tools/reliable_report"
 
-desc "Fetch top most reliable specs"
-task :reliable_spec_report, [:run_type, :range, :create_slack_report] do |_task, args|
-  report = QA::Tools::ReliableReport.new(args[:run_type] || "package-and-qa", args[:range])
-
-  report.show_top_stable
-  report.notify_top_stable if args[:create_slack_report] == 'true'
-end
-
-desc "Fetch top most unstable reliable specs"
-task :unreliable_spec_report, [:run_type, :range, :create_slack_report] do |_task, args|
-  report = QA::Tools::ReliableReport.new(args[:run_type] || "package-and-qa", args[:range])
-
-  report.show_top_unstable
-  report.notify_top_unstable if args[:create_slack_report] == 'true'
+desc "Fetch reliable and unreliable spec data and create report"
+task :reliable_spec_report, [:range, :report_in_issue_and_slack] do |_task, args|
+  QA::Tools::ReliableReport.run(**args)
 end
 # rubocop:enable Rails/RakeEnvironment