Merge branch 'acunskis-rspec-retry-patch' into 'master'

E2E: Respect quarantine context for rspec-retry

See merge request gitlab-org/gitlab!78408

qa/qa/specs/features/api/1_manage/migration/gitlab_migration_group_spec.rb

@@ -174,7 +174,6 @@ module QA
           expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration)
 
           imported_member = imported_group.reload!.members.find { |usr| usr.username == member.username }
-
           aggregate_failures do
             expect(imported_member).not_to be_nil
             expect(imported_member.access_level).to eq(Resource::Members::AccessLevel::DEVELOPER)

qa/qa/specs/features/api/1_manage/migration/gitlab_migration_project_spec.rb

@@ -65,9 +65,9 @@ module QA
           'successfully imports repository',
           testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347570'
         ) do
-          aggregate_failures do
           expect_import_finished
 
+          aggregate_failures do
             expect(imported_commits).to match_array(source_commits)
             expect(imported_tags).to match_array(source_tags)
             expect(imported_branches).to match_array(source_branches)

qa/qa/specs/helpers/quarantine.rb

@@ -11,9 +11,9 @@ module QA
         extend self
 
         # Skip tests in quarantine unless we explicitly focus on them.
-        def skip_or_run_quarantined_tests_or_contexts(filters, example)
+        def skip_or_run_quarantined_tests_or_contexts(example)
           if filters.key?(:quarantine)
-            included_filters = filters_other_than_quarantine(filters)
+            included_filters = filters_other_than_quarantine
 
             # If :quarantine is focused, skip the test/context unless its metadata
             # includes quarantine and any other filters
@@ -29,18 +29,17 @@ module QA
           elsif example.metadata.key?(:quarantine)
             quarantine_tag = example.metadata[:quarantine]
 
-            if quarantine_tag.is_a?(Hash) && quarantine_tag&.key?(:only) && !ContextSelector.context_matches?(quarantine_tag[:only])
             # If the :quarantine hash contains :only, we respect that.
-              # For instance `quarantine: { only: { subdomain: :staging } }` will only quarantine the test when it runs against staging.
-              return
-            end
+            # For instance `quarantine: { only: { subdomain: :staging } }`
+            # will only quarantine the test when it runs against staging.
+            return if quarantined_different_context?(quarantine_tag)
 
             example.metadata[:skip] = quarantine_message(quarantine_tag)
           end
         end
 
-        def filters_other_than_quarantine(filter)
-          filter.reject { |key, _| key == :quarantine }
+        def filters_other_than_quarantine
+          filters.reject { |key, _| key == :quarantine }
         end
 
         def quarantine_message(quarantine_tag)
@@ -70,6 +69,14 @@ module QA
 
           (metadata.keys & included_filters.keys).empty?
         end
+
+        def quarantined_different_context?(quarantine)
+          quarantine.is_a?(Hash) && quarantine.key?(:only) && !ContextSelector.context_matches?(quarantine[:only])
+        end
+
+        def filters
+          @filters ||= ::RSpec.configuration.inclusion_filter.rules
+        end
       end
     end
   end

qa/qa/support/formatters/quarantine_formatter.rb

@@ -18,7 +18,7 @@ module QA
         def example_group_started(example_group_notification)
           group = example_group_notification.group
 
-          skip_or_run_quarantined_tests_or_contexts(filters, group)
+          skip_or_run_quarantined_tests_or_contexts(group)
         end
 
         # Starts example
@@ -28,13 +28,7 @@ module QA
           example = example_notification.example
 
           # if skip propagated from example_group, do not reset skip metadata
-          skip_or_run_quarantined_tests_or_contexts(filters, example) unless example.metadata[:skip]
-        end
-
-        private
-
-        def filters
-          @filters ||= ::RSpec.configuration.inclusion_filter.rules
+          skip_or_run_quarantined_tests_or_contexts(example) unless example.metadata[:skip]
         end
       end
     end

qa/spec/spec_helper.rb

@@ -100,8 +100,14 @@ RSpec.configure do |config|
   if ENV['CI'] && !QA::Runtime::Env.disable_rspec_retry?
     non_quarantine_retries = QA::Runtime::Env.ci_project_name =~ /staging|canary|production/ ? 3 : 2
     config.around do |example|
-      retry_times = example.metadata.key?(:quarantine) ? 1 : non_quarantine_retries
-      example.run_with_retry retry: retry_times
+      quarantine = example.metadata[:quarantine]
+      different_quarantine_context = QA::Specs::Helpers::Quarantine.quarantined_different_context?(quarantine)
+      focused_quarantine = QA::Specs::Helpers::Quarantine.filters.key?(:quarantine)
+
+      # Do not disable retry when spec is quarantined but on different environment
+      next example.run_with_retry(retry: non_quarantine_retries) if different_quarantine_context && !focused_quarantine
+
+      example.run_with_retry(retry: quarantine ? 1 : non_quarantine_retries)
     end
   end
 end