Merge branch 'move-rspec-flaky-code-outside-to-tooling' into 'master'

Move RSpecFlaky from lib/ to tooling/

See merge request gitlab-org/gitlab!53283

.rubocop.yml

@@ -243,7 +243,7 @@ Gitlab/Json:
     - 'db/**/*'
     - 'qa/**/*'
     - 'scripts/**/*'
-    - 'lib/rspec_flaky/**/*'
+    - 'tooling/rspec_flaky/**/*'
     - 'lib/quality/**/*'
     - 'tooling/danger/**/*'
 

.rubocop_manual_todo.yml

@@ -485,8 +485,8 @@ Rails/TimeZone:
     - 'lib/json_web_token/token.rb'
     - 'lib/object_storage/direct_upload.rb'
     - 'lib/quality/seeders/issues.rb'
-    - 'lib/rspec_flaky/flaky_example.rb'
-    - 'lib/rspec_flaky/report.rb'
+    - 'tooling/rspec_flaky/flaky_example.rb'
+    - 'tooling/rspec_flaky/report.rb'
     - 'lib/tasks/gitlab/assets.rake'
     - 'lib/tasks/gitlab/backup.rake'
     - 'lib/tasks/gitlab/cleanup.rake'
@@ -552,9 +552,9 @@ Rails/TimeZone:
     - 'spec/lib/gitlab/x509/signature_spec.rb'
     - 'spec/lib/grafana/time_window_spec.rb'
     - 'spec/lib/json_web_token/hmac_token_spec.rb'
-    - 'spec/lib/rspec_flaky/flaky_example_spec.rb'
-    - 'spec/lib/rspec_flaky/listener_spec.rb'
-    - 'spec/lib/rspec_flaky/report_spec.rb'
+    - 'spec/tooling/rspec_flaky/flaky_example_spec.rb'
+    - 'spec/tooling/rspec_flaky/listener_spec.rb'
+    - 'spec/tooling/rspec_flaky/report_spec.rb'
 
 RSpec/TimecopFreeze:
   Exclude:
@@ -627,8 +627,8 @@ RSpec/TimecopFreeze:
     - 'spec/lib/gitlab/puma_logging/json_formatter_spec.rb'
     - 'spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb'
     - 'spec/lib/json_web_token/hmac_token_spec.rb'
-    - 'spec/lib/rspec_flaky/flaky_example_spec.rb'
-    - 'spec/lib/rspec_flaky/listener_spec.rb'
+    - 'spec/tooling/rspec_flaky/flaky_example_spec.rb'
+    - 'spec/tooling/rspec_flaky/listener_spec.rb'
     - 'spec/models/active_session_spec.rb'
     - 'spec/serializers/entity_date_helper_spec.rb'
     - 'spec/support/cycle_analytics_helpers/test_generation.rb'

scripts/flaky_examples/prune-old-flaky-examples

 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# lib/rspec_flaky/flaky_examples_collection.rb is requiring
+# tooling/rspec_flaky/flaky_examples_collection.rb is requiring
 # `active_support/hash_with_indifferent_access`, and we install the `activesupport`
 # gem manually on the CI
 require 'rubygems'
-
-# In newer Ruby, alias_method is not private then we don't need __send__
-singleton_class.__send__(:alias_method, :require_dependency, :require) # rubocop:disable GitlabSecurity/PublicSend
-$:.unshift(File.expand_path('../../lib', __dir__))
-
-require 'rspec_flaky/report'
+require_relative '../../tooling/rspec_flaky/report'
 
 report_file = ARGV.shift
 unless report_file

spec/spec_helper.rb

@@ -180,6 +180,8 @@ RSpec.configure do |config|
   end
 
   if ENV['FLAKY_RSPEC_GENERATE_REPORT']
+    require_relative '../tooling/rspec_flaky/listener'
+
     config.reporter.register_listener(
       RspecFlaky::Listener.new,
       :example_passed,

spec/lib/rspec_flaky/config_spec.rb → spec/tooling/rspec_flaky/config_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
+require 'rspec-parameterized'
+require_relative '../../support/helpers/stub_env'
+
+require_relative '../../../tooling/rspec_flaky/config'
 
 RSpec.describe RspecFlaky::Config, :aggregate_failures do
+  include StubENV
+
   before do
     # Stub these env variables otherwise specs don't behave the same on the CI
     stub_env('FLAKY_RSPEC_GENERATE_REPORT', nil)
     stub_env('SUITE_FLAKY_RSPEC_REPORT_PATH', nil)
     stub_env('FLAKY_RSPEC_REPORT_PATH', nil)
     stub_env('NEW_FLAKY_RSPEC_REPORT_PATH', nil)
+    # Ensure the behavior is the same locally and on CI (where Rails is defined since we run this test as part of the whole suite), i.e. Rails isn't defined
+    allow(described_class).to receive(:rails_path).and_wrap_original do |method, path|
+      path
+    end
   end
 
   describe '.generate_report?' do
@@ -44,10 +53,7 @@ RSpec.describe RspecFlaky::Config, :aggregate_failures do
   describe '.suite_flaky_examples_report_path' do
     context "when ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] is not set" do
       it 'returns the default path' do
-        expect(Rails.root).to receive(:join).with('rspec_flaky/suite-report.json')
-          .and_return('root/rspec_flaky/suite-report.json')
-
-        expect(described_class.suite_flaky_examples_report_path).to eq('root/rspec_flaky/suite-report.json')
+        expect(described_class.suite_flaky_examples_report_path).to eq('rspec_flaky/suite-report.json')
       end
     end
 
@@ -65,10 +71,7 @@ RSpec.describe RspecFlaky::Config, :aggregate_failures do
   describe '.flaky_examples_report_path' do
     context "when ENV['FLAKY_RSPEC_REPORT_PATH'] is not set" do
       it 'returns the default path' do
-        expect(Rails.root).to receive(:join).with('rspec_flaky/report.json')
-          .and_return('root/rspec_flaky/report.json')
-
-        expect(described_class.flaky_examples_report_path).to eq('root/rspec_flaky/report.json')
+        expect(described_class.flaky_examples_report_path).to eq('rspec_flaky/report.json')
       end
     end
 
@@ -86,10 +89,7 @@ RSpec.describe RspecFlaky::Config, :aggregate_failures do
   describe '.new_flaky_examples_report_path' do
     context "when ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] is not set" do
       it 'returns the default path' do
-        expect(Rails.root).to receive(:join).with('rspec_flaky/new-report.json')
-          .and_return('root/rspec_flaky/new-report.json')
-
-        expect(described_class.new_flaky_examples_report_path).to eq('root/rspec_flaky/new-report.json')
+        expect(described_class.new_flaky_examples_report_path).to eq('rspec_flaky/new-report.json')
       end
     end
 

spec/lib/rspec_flaky/example_spec.rb → spec/tooling/rspec_flaky/example_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
+require_relative '../../../tooling/rspec_flaky/example'
 
 RSpec.describe RspecFlaky::Example do
   let(:example_attrs) do

spec/lib/rspec_flaky/flaky_example_spec.rb → spec/tooling/rspec_flaky/flaky_example_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
+require 'active_support/testing/time_helpers'
+require_relative '../../support/helpers/stub_env'
+
+require_relative '../../../tooling/rspec_flaky/flaky_example'
 
 RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do
+  include ActiveSupport::Testing::TimeHelpers
+  include StubENV
+
   let(:flaky_example_attrs) do
     {
       example_id: 'spec/foo/bar_spec.rb:2',
@@ -30,7 +36,7 @@ RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do
     }
   end
 
-  let(:example) { double(example_attrs) }
+  let(:example) { OpenStruct.new(example_attrs) }
 
   before do
     # Stub these env variables otherwise specs don't behave the same on the CI
@@ -77,19 +83,33 @@ RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do
     shared_examples 'an up-to-date FlakyExample instance' do
       let(:flaky_example) { described_class.new(args) }
 
-      it 'updates the first_flaky_at' do
-        now = Time.now
-        expected_first_flaky_at = flaky_example.first_flaky_at || now
-        Timecop.freeze(now) { flaky_example.update_flakiness! }
+      it 'sets the first_flaky_at if none exists' do
+        args[:first_flaky_at] = nil
 
-        expect(flaky_example.first_flaky_at).to eq(expected_first_flaky_at)
+        freeze_time do
+          flaky_example.update_flakiness!
+
+          expect(flaky_example.first_flaky_at).to eq(Time.now)
+        end
+      end
+
+      it 'maintains the first_flaky_at if exists' do
+        flaky_example.update_flakiness!
+        expected_first_flaky_at = flaky_example.first_flaky_at
+
+        travel_to(Time.now + 42) do
+          flaky_example.update_flakiness!
+          expect(flaky_example.first_flaky_at).to eq(expected_first_flaky_at)
+        end
       end
 
       it 'updates the last_flaky_at' do
-        now = Time.now
-        Timecop.freeze(now) { flaky_example.update_flakiness! }
+        travel_to(Time.now + 42) do
+          the_future = Time.now
+          flaky_example.update_flakiness!
 
-        expect(flaky_example.last_flaky_at).to eq(now)
+          expect(flaky_example.last_flaky_at).to eq(the_future)
+        end
       end
 
       it 'updates the flaky_reports' do

spec/lib/rspec_flaky/flaky_examples_collection_spec.rb → spec/tooling/rspec_flaky/flaky_examples_collection_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
+require_relative '../../../tooling/rspec_flaky/flaky_examples_collection'
 
 RSpec.describe RspecFlaky::FlakyExamplesCollection, :aggregate_failures do
   let(:collection_hash) do

spec/lib/rspec_flaky/listener_spec.rb → spec/tooling/rspec_flaky/listener_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
+require 'active_support/testing/time_helpers'
+require_relative '../../support/helpers/stub_env'
+
+require_relative '../../../tooling/rspec_flaky/listener'
 
 RSpec.describe RspecFlaky::Listener, :aggregate_failures do
+  include ActiveSupport::Testing::TimeHelpers
+  include StubENV
+
   let(:already_flaky_example_uid) { '6e869794f4cfd2badd93eb68719371d1' }
   let(:suite_flaky_example_report) do
     {
@@ -130,14 +136,13 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
       it 'changes the flaky examples hash' do
         new_example = RspecFlaky::Example.new(rspec_example)
 
-        now = Time.now
-        Timecop.freeze(now) do
+        travel_to(Time.now + 42) do
+          the_future = Time.now
           expect { listener.example_passed(notification) }
             .to change { listener.flaky_examples[new_example.uid].to_h }
+          expect(listener.flaky_examples[new_example.uid].to_h)
+            .to eq(expected_flaky_example.merge(last_flaky_at: the_future))
         end
-
-        expect(listener.flaky_examples[new_example.uid].to_h)
-          .to eq(expected_flaky_example.merge(last_flaky_at: now))
       end
     end
 
@@ -157,14 +162,13 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
       it 'changes the all flaky examples hash' do
         new_example = RspecFlaky::Example.new(rspec_example)
 
-        now = Time.now
-        Timecop.freeze(now) do
+        travel_to(Time.now + 42) do
+          the_future = Time.now
           expect { listener.example_passed(notification) }
             .to change { listener.flaky_examples[new_example.uid].to_h }
+          expect(listener.flaky_examples[new_example.uid].to_h)
+            .to eq(expected_flaky_example.merge(first_flaky_at: the_future, last_flaky_at: the_future))
         end
-
-        expect(listener.flaky_examples[new_example.uid].to_h)
-          .to eq(expected_flaky_example.merge(first_flaky_at: now, last_flaky_at: now))
       end
     end
 
@@ -198,6 +202,10 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
     let(:notification_new_flaky_rspec_example) { double(example: new_flaky_rspec_example) }
     let(:notification_already_flaky_rspec_example) { double(example: already_flaky_rspec_example) }
 
+    before do
+      allow(Kernel).to receive(:warn)
+    end
+
     context 'when a report file path is set by FLAKY_RSPEC_REPORT_PATH' do
       it 'delegates the writes to RspecFlaky::Report' do
         listener.example_passed(notification_new_flaky_rspec_example)

spec/lib/rspec_flaky/report_spec.rb → spec/tooling/rspec_flaky/report_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
+require 'tempfile'
+
+require_relative '../../../tooling/rspec_flaky/report'
 
 RSpec.describe RspecFlaky::Report, :aggregate_failures do
   let(:thirty_one_days) { 3600 * 24 * 31 }
@@ -30,10 +32,14 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures do
   let(:flaky_examples) { RspecFlaky::FlakyExamplesCollection.new(collection_hash) }
   let(:report) { described_class.new(flaky_examples) }
 
+  before do
+    allow(Kernel).to receive(:warn)
+  end
+
   describe '.load' do
     let!(:report_file) do
       Tempfile.new(%w[rspec_flaky_report .json]).tap do |f|
-        f.write(Gitlab::Json.pretty_generate(suite_flaky_example_report))
+        f.write(JSON.pretty_generate(suite_flaky_example_report)) # rubocop:disable Gitlab/Json
         f.rewind
       end
     end
@@ -50,7 +56,7 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures do
 
   describe '.load_json' do
     let(:report_json) do
-      Gitlab::Json.pretty_generate(suite_flaky_example_report)
+      JSON.pretty_generate(suite_flaky_example_report) # rubocop:disable Gitlab/Json
     end
 
     it 'loads the report file' do
@@ -73,7 +79,7 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures do
   end
 
   describe '#write' do
-    let(:report_file_path) { Rails.root.join('tmp', 'rspec_flaky_report.json') }
+    let(:report_file_path) { File.join('tmp', 'rspec_flaky_report.json') }
 
     before do
       FileUtils.rm(report_file_path) if File.exist?(report_file_path)
@@ -105,7 +111,7 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures do
 
         expect(File.exist?(report_file_path)).to be(true)
         expect(File.read(report_file_path))
-          .to eq(Gitlab::Json.pretty_generate(report.flaky_examples.to_h))
+          .to eq(JSON.pretty_generate(report.flaky_examples.to_h)) # rubocop:disable Gitlab/Json
       end
     end
   end

lib/rspec_flaky/config.rb → tooling/rspec_flaky/config.rb

@@ -7,15 +7,21 @@ module RspecFlaky
     end
 
     def self.suite_flaky_examples_report_path
-      ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/suite-report.json")
+      ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/suite-report.json")
     end
 
     def self.flaky_examples_report_path
-      ENV['FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/report.json")
+      ENV['FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/report.json")
     end
 
     def self.new_flaky_examples_report_path
-      ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/new-report.json")
+      ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/new-report.json")
+    end
+
+    def self.rails_path(path)
+      return path unless defined?(Rails)
+
+      Rails.root.join(path)
     end
   end
 end

lib/rspec_flaky/example.rb → tooling/rspec_flaky/example.rb

 # frozen_string_literal: true
 
+require 'forwardable'
+require 'digest'
+
 module RspecFlaky
   # This is a wrapper class for RSpec::Core::Example
   class Example
-    delegate :status, :exception, to: :execution_result
+    extend Forwardable
+
+    def_delegators :execution_result, :status, :exception
 
     def initialize(rspec_example)
-      @rspec_example = rspec_example.try(:example) || rspec_example
+      @rspec_example = rspec_example.respond_to?(:example) ? rspec_example.example : rspec_example
     end
 
     def uid
@@ -30,7 +35,7 @@ module RspecFlaky
     end
 
     def attempts
-      rspec_example.try(:attempts) || 1
+      rspec_example.respond_to?(:attempts) ? rspec_example.attempts : 1
     end
 
     private

lib/rspec_flaky/flaky_example.rb → tooling/rspec_flaky/flaky_example.rb

 # frozen_string_literal: true
 
+require 'ostruct'
+
 module RspecFlaky
   # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
   class FlakyExample < OpenStruct

lib/rspec_flaky/listener.rb → tooling/rspec_flaky/listener.rb

@@ -2,11 +2,11 @@
 
 require 'json'
 
-require_dependency 'rspec_flaky/config'
-require_dependency 'rspec_flaky/example'
-require_dependency 'rspec_flaky/flaky_example'
-require_dependency 'rspec_flaky/flaky_examples_collection'
-require_dependency 'rspec_flaky/report'
+require_relative 'config'
+require_relative 'example'
+require_relative 'flaky_example'
+require_relative 'flaky_examples_collection'
+require_relative 'report'
 
 module RspecFlaky
   class Listener
@@ -32,21 +32,19 @@ module RspecFlaky
       flaky_examples[current_example.uid] = flaky_example
     end
 
-    # rubocop:disable Gitlab/RailsLogger
     def dump_summary(_)
       RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path)
       # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path)
 
       new_flaky_examples = flaky_examples - suite_flaky_examples
       if new_flaky_examples.any?
-        Rails.logger.warn "\nNew flaky examples detected:\n"
-        Rails.logger.warn Gitlab::Json.pretty_generate(new_flaky_examples.to_h)
+        rails_logger_warn("\nNew flaky examples detected:\n")
+        rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h)) # rubocop:disable Gitlab/Json
 
         RspecFlaky::Report.new(new_flaky_examples).write(RspecFlaky::Config.new_flaky_examples_report_path)
         # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path)
       end
     end
-    # rubocop:enable Gitlab/RailsLogger
 
     private
 
@@ -59,5 +57,11 @@ module RspecFlaky
         RspecFlaky::Report.load(RspecFlaky::Config.suite_flaky_examples_report_path).flaky_examples
       end
     end
+
+    def rails_logger_warn(text)
+      target = defined?(Rails) ? Rails.logger : Kernel
+
+      target.warn(text)
+    end
   end
 end

lib/rspec_flaky/report.rb → tooling/rspec_flaky/report.rb

@@ -3,8 +3,8 @@
 require 'json'
 require 'time'
 
-require_dependency 'rspec_flaky/config'
-require_dependency 'rspec_flaky/flaky_examples_collection'
+require_relative 'config'
+require_relative 'flaky_examples_collection'
 
 module RspecFlaky
   # This class is responsible for loading/saving JSON reports, and pruning
@@ -33,7 +33,7 @@ module RspecFlaky
 
     def write(file_path)
       unless RspecFlaky::Config.generate_report?
-        puts "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !" # rubocop:disable Rails/Output
+        Kernel.warn "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !"
         return
       end