Commit ebfb5a50 authored by Rémy Coutable's avatar Rémy Coutable

Ensure RSpecFlaky doesn't automatically update flaky examples

Previously, instantiating a RspecFlaky::FlakyExample object would
automatically update its first_flaky_at, last_flaky_at and
last_flaky_job.

That was wrong because we would overwrite every time the suite report
with this false data.

We now:

- Get the suite report and only read from it
- Write only the currently detected flaky examples in the report, so
  that the final report is only updated with flaky examples that were
  actually detected in each job. Before, job1 could overwrite the legit
  report from job2!
- Write the newly detected flaky examples by rejecting the already
  tracked flaky specs instead of using another hash.
Signed-off-by: default avatarRémy Coutable <remy@rymai.me>
parent 4a0f720a
...@@ -9,24 +9,21 @@ module RspecFlaky ...@@ -9,24 +9,21 @@ module RspecFlaky
line: example.line, line: example.line,
description: example.description, description: example.description,
last_attempts_count: example.attempts, last_attempts_count: example.attempts,
flaky_reports: 1) flaky_reports: 0)
else else
super super
end end
end end
def first_flaky_at def update_flakiness!(last_attempts_count: nil)
self[:first_flaky_at] || Time.now self.first_flaky_at ||= Time.now
end self.last_flaky_at = Time.now
self.flaky_reports += 1
self.last_attempts_count = last_attempts_count if last_attempts_count
def last_flaky_at if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
Time.now self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
end end
def last_flaky_job
return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
"#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
end end
def to_h def to_h
......
...@@ -2,11 +2,15 @@ require 'json' ...@@ -2,11 +2,15 @@ require 'json'
module RspecFlaky module RspecFlaky
class Listener class Listener
attr_reader :all_flaky_examples, :new_flaky_examples # - suite_flaky_examples: contains all the currently tracked flacky example
# for the whole RSpec suite
def initialize # - flaky_examples: contains the examples detected as flaky during the
@new_flaky_examples = {} # current RSpec run
@all_flaky_examples = init_all_flaky_examples attr_reader :suite_flaky_examples, :flaky_examples
def initialize(suite_flaky_examples_json = nil)
@flaky_examples = {}
@suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json)
end end
def example_passed(notification) def example_passed(notification)
...@@ -14,24 +18,16 @@ module RspecFlaky ...@@ -14,24 +18,16 @@ module RspecFlaky
return unless current_example.attempts > 1 return unless current_example.attempts > 1
flaky_example_hash = all_flaky_examples[current_example.uid] flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) }
flaky_example.update_flakiness!(last_attempts_count: current_example.attempts)
all_flaky_examples[current_example.uid] = flaky_examples[current_example.uid] = flaky_example
if flaky_example_hash
FlakyExample.new(flaky_example_hash).tap do |ex|
ex.last_attempts_count = current_example.attempts
ex.flaky_reports += 1
end
else
FlakyExample.new(current_example).tap do |ex|
new_flaky_examples[current_example.uid] = ex
end
end
end end
def dump_summary(_) def dump_summary(_)
write_report_file(all_flaky_examples, all_flaky_examples_report_path) write_report_file(flaky_examples, flaky_examples_report_path)
new_flaky_examples = _new_flaky_examples
if new_flaky_examples.any? if new_flaky_examples.any?
Rails.logger.warn "\nNew flaky examples detected:\n" Rails.logger.warn "\nNew flaky examples detected:\n"
Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples)) Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples))
...@@ -46,12 +42,24 @@ module RspecFlaky ...@@ -46,12 +42,24 @@ module RspecFlaky
private private
def init_all_flaky_examples def init_suite_flaky_examples(suite_flaky_examples_json = nil)
return {} unless File.exist?(all_flaky_examples_report_path) unless suite_flaky_examples_json
return {} unless File.exist?(suite_flaky_examples_report_path)
suite_flaky_examples_json = File.read(suite_flaky_examples_report_path)
end
suite_flaky_examples = JSON.parse(suite_flaky_examples_json)
Hash[(suite_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }].freeze
end
all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path)) def _new_flaky_examples
flaky_examples.reject { |uid, _| already_flaky?(uid) }
end
Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }] def already_flaky?(example_uid)
suite_flaky_examples.key?(example_uid)
end end
def write_report_file(examples, file_path) def write_report_file(examples, file_path)
...@@ -62,9 +70,14 @@ module RspecFlaky ...@@ -62,9 +70,14 @@ module RspecFlaky
File.write(file_path, JSON.pretty_generate(to_report(examples))) File.write(file_path, JSON.pretty_generate(to_report(examples)))
end end
def all_flaky_examples_report_path def suite_flaky_examples_report_path
@all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] || @suite_flaky_examples_report_path ||= ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] ||
Rails.root.join("rspec_flaky/all-report.json") Rails.root.join("rspec_flaky/suite-report.json")
end
def flaky_examples_report_path
@flaky_examples_report_path ||= ENV['FLAKY_RSPEC_REPORT_PATH'] ||
Rails.root.join("rspec_flaky/report.json")
end end
def new_flaky_examples_report_path def new_flaky_examples_report_path
......
require 'spec_helper' require 'spec_helper'
describe RspecFlaky::FlakyExample do describe RspecFlaky::FlakyExample, :aggregate_failures do
let(:flaky_example_attrs) do let(:flaky_example_attrs) do
{ {
example_id: 'spec/foo/bar_spec.rb:2', example_id: 'spec/foo/bar_spec.rb:2',
...@@ -9,6 +9,7 @@ describe RspecFlaky::FlakyExample do ...@@ -9,6 +9,7 @@ describe RspecFlaky::FlakyExample do
description: 'hello world', description: 'hello world',
first_flaky_at: 1234, first_flaky_at: 1234,
last_flaky_at: 2345, last_flaky_at: 2345,
last_flaky_job: 'https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/12',
last_attempts_count: 2, last_attempts_count: 2,
flaky_reports: 1 flaky_reports: 1
} }
...@@ -27,57 +28,78 @@ describe RspecFlaky::FlakyExample do ...@@ -27,57 +28,78 @@ describe RspecFlaky::FlakyExample do
end end
let(:example) { double(example_attrs) } let(:example) { double(example_attrs) }
before do
# Stub these env variables otherwise specs don't behave the same on the CI
stub_env('CI_PROJECT_URL', nil)
stub_env('CI_JOB_ID', nil)
end
describe '#initialize' do describe '#initialize' do
shared_examples 'a valid FlakyExample instance' do shared_examples 'a valid FlakyExample instance' do
it 'returns valid attributes' do let(:flaky_example) { described_class.new(args) }
flaky_example = described_class.new(args)
it 'returns valid attributes' do
expect(flaky_example.uid).to eq(flaky_example_attrs[:uid]) expect(flaky_example.uid).to eq(flaky_example_attrs[:uid])
expect(flaky_example.example_id).to eq(flaky_example_attrs[:example_id]) expect(flaky_example.file).to eq(flaky_example_attrs[:file])
expect(flaky_example.line).to eq(flaky_example_attrs[:line])
expect(flaky_example.description).to eq(flaky_example_attrs[:description])
expect(flaky_example.first_flaky_at).to eq(expected_first_flaky_at)
expect(flaky_example.last_flaky_at).to eq(expected_last_flaky_at)
expect(flaky_example.last_attempts_count).to eq(flaky_example_attrs[:last_attempts_count])
expect(flaky_example.flaky_reports).to eq(expected_flaky_reports)
end end
end end
context 'when given an Rspec::Example' do context 'when given an Rspec::Example' do
it_behaves_like 'a valid FlakyExample instance' do
let(:args) { example } let(:args) { example }
let(:expected_first_flaky_at) { nil }
it_behaves_like 'a valid FlakyExample instance' let(:expected_last_flaky_at) { nil }
let(:expected_flaky_reports) { 0 }
end
end end
context 'when given a hash' do context 'when given a hash' do
it_behaves_like 'a valid FlakyExample instance' do
let(:args) { flaky_example_attrs } let(:args) { flaky_example_attrs }
let(:expected_flaky_reports) { flaky_example_attrs[:flaky_reports] }
it_behaves_like 'a valid FlakyExample instance' let(:expected_first_flaky_at) { flaky_example_attrs[:first_flaky_at] }
let(:expected_last_flaky_at) { flaky_example_attrs[:last_flaky_at] }
end end
end end
describe '#to_h' do
before do
# Stub these env variables otherwise specs don't behave the same on the CI
stub_env('CI_PROJECT_URL', nil)
stub_env('CI_JOB_ID', nil)
end end
shared_examples 'a valid FlakyExample hash' do describe '#update_flakiness!' do
let(:additional_attrs) { {} } shared_examples 'an up-to-date FlakyExample instance' do
let(:flaky_example) { described_class.new(args) }
it 'returns a valid hash' do it 'updates the first_flaky_at' do
flaky_example = described_class.new(args) now = Time.now
final_hash = flaky_example_attrs expected_first_flaky_at = flaky_example.first_flaky_at ? flaky_example.first_flaky_at : now
.merge(last_flaky_at: instance_of(Time), last_flaky_job: nil) Timecop.freeze(now) { flaky_example.update_flakiness! }
.merge(additional_attrs)
expect(flaky_example.to_h).to match(hash_including(final_hash)) expect(flaky_example.first_flaky_at).to eq(expected_first_flaky_at)
end end
it 'updates the last_flaky_at' do
now = Time.now
Timecop.freeze(now) { flaky_example.update_flakiness! }
expect(flaky_example.last_flaky_at).to eq(now)
end end
context 'when given an Rspec::Example' do it 'updates the flaky_reports' do
let(:args) { example } expected_flaky_reports = flaky_example.first_flaky_at ? flaky_example.flaky_reports + 1 : 1
context 'when run locally' do expect { flaky_example.update_flakiness! }.to change { flaky_example.flaky_reports }.by(1)
it_behaves_like 'a valid FlakyExample hash' do expect(flaky_example.flaky_reports).to eq(expected_flaky_reports)
let(:additional_attrs) do
{ first_flaky_at: instance_of(Time) }
end end
context 'when passed a :last_attempts_count' do
it 'updates the last_attempts_count' do
flaky_example.update_flakiness!(last_attempts_count: 42)
expect(flaky_example.last_attempts_count).to eq(42)
end end
end end
...@@ -87,10 +109,45 @@ describe RspecFlaky::FlakyExample do ...@@ -87,10 +109,45 @@ describe RspecFlaky::FlakyExample do
stub_env('CI_JOB_ID', 42) stub_env('CI_JOB_ID', 42)
end end
it 'updates the last_flaky_job' do
flaky_example.update_flakiness!
expect(flaky_example.last_flaky_job).to eq('https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/42')
end
end
end
context 'when given an Rspec::Example' do
it_behaves_like 'an up-to-date FlakyExample instance' do
let(:args) { example }
end
end
context 'when given a hash' do
it_behaves_like 'an up-to-date FlakyExample instance' do
let(:args) { flaky_example_attrs }
end
end
end
describe '#to_h' do
shared_examples 'a valid FlakyExample hash' do
let(:additional_attrs) { {} }
it 'returns a valid hash' do
flaky_example = described_class.new(args)
final_hash = flaky_example_attrs.merge(additional_attrs)
expect(flaky_example.to_h).to eq(final_hash)
end
end
context 'when given an Rspec::Example' do
let(:args) { example }
it_behaves_like 'a valid FlakyExample hash' do it_behaves_like 'a valid FlakyExample hash' do
let(:additional_attrs) do let(:additional_attrs) do
{ first_flaky_at: instance_of(Time), last_flaky_job: "https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/42" } { first_flaky_at: nil, last_flaky_at: nil, last_flaky_job: nil, flaky_reports: 0 }
end
end end
end end
end end
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment