Wait and retry if `Retry later` error appears

The error appears when accessing protected paths, most frequently
when signing in or signing out. This detects the error when visiting
those pages, and waits for 60 seconds before refreshing. This
process repeats, adding 60 seconds each time.

To prevent running indefinitely, it allows the test to fail after
5 attempts.

qa/qa/page/base.rb

@@ -16,6 +16,10 @@ module QA
 
       def_delegators :evaluator, :view, :views
 
+      def initialize
+        @retry_later_backoff = QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME
+      end
+
       def assert_no_element(name)
         assert_no_selector(element_selector_css(name))
       end
@@ -259,6 +263,19 @@ module QA
         visit find_element(name)['href']
       end
 
+      def wait_if_retry_later
+        return if @retry_later_backoff > QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME * 5
+
+        if has_css?('body', text: 'Retry later', wait: 0)
+          QA::Runtime::Logger.warn("`Retry later` error occurred. Sleeping for #{@retry_later_backoff} seconds...")
+          sleep @retry_later_backoff
+          refresh
+          @retry_later_backoff += QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME
+
+          wait_if_retry_later
+        end
+      end
+
       def self.path
         raise NotImplementedError
       end

qa/qa/page/main/login.rb

@@ -152,6 +152,8 @@ module QA
         private
 
         def sign_in_using_gitlab_credentials(user:, skip_page_validation: false)
+          wait_if_retry_later
+
           switch_to_sign_in_tab if has_sign_in_tab?
           switch_to_standard_tab if has_standard_tab?
 

qa/qa/page/main/menu.rb

@@ -77,6 +77,8 @@ module QA
 
         def sign_out
           retry_until do
+            wait_if_retry_later
+
             break true unless signed_in?
 
             within_user_menu do

qa/qa/page/validatable.rb

@@ -8,6 +8,8 @@ module QA
       def validate_elements_present!
         base_page = self.new
 
+        base_page.wait_if_retry_later
+
         elements.each do |element|
           next unless element.required?
 

qa/qa/runtime/env.rb

@@ -43,6 +43,10 @@ module QA
         ENV['GITLAB_QA_ADMIN_ACCESS_TOKEN']
       end
 
+      def ci_project_name
+        ENV['CI_PROJECT_NAME']
+      end
+
       def debug?
         enabled?(ENV['QA_DEBUG'], default: false)
       end

qa/spec/spec_helper.rb

@@ -63,8 +63,9 @@ RSpec.configure do |config|
   config.display_try_failure_messages = true
 
   if ENV['CI'] && !QA::Runtime::Env.disable_rspec_retry?
+    non_quarantine_retries = QA::Runtime::Env.ci_project_name =~ /staging|canary|production/ ? 3 : 2
     config.around do |example|
-      retry_times = example.metadata.key?(:quarantine) ? 1 : 2
+      retry_times = example.metadata.key?(:quarantine) ? 1 : non_quarantine_retries
       example.run_with_retry retry: retry_times
     end
   end