Check QA end to end tests if an HTTP 404 / 500 returned

This adds a check if the current page is an HTTP
404, 500 or GDK backtrace page. If so it reports
the error

qa/qa/resource/base.rb

@@ -166,11 +166,11 @@ module QA
         raise NotImplementedError
       end
 
-      def visit!
+      def visit!(skip_resp_code_check: false)
         Runtime::Logger.debug(%(Visiting #{self.class.name} at "#{web_url}"))
 
         # Just in case an async action is not yet complete
-        Support::WaitForRequests.wait_for_requests
+        Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check)
 
         Support::Retrier.retry_until do
           visit(web_url)
@@ -178,7 +178,7 @@ module QA
         end
 
         # Wait until the new page is ready for us to interact with it
-        Support::WaitForRequests.wait_for_requests
+        Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check)
       end
 
       def populate(*attribute_names)

qa/qa/specs/features/ee/browser_ui/1_manage/group/restrict_by_ip_address_spec.rb

@@ -52,11 +52,11 @@ module QA
           it 'denies access', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347923' do
             Flow::Login.sign_in(as: @user)
 
-            @group.sandbox.visit!
+            @group.sandbox.visit!(skip_resp_code_check: true)
             expect(page).to have_text('Page Not Found')
             page.go_back
 
-            @group.visit!
+            @group.visit!(skip_resp_code_check: true)
             expect(page).to have_text('Page Not Found')
             page.go_back
           end

qa/qa/support/page_error_checker.rb

+# frozen_string_literal: true
+
+module QA
+  module Support
+    class PageErrorChecker
+      class << self
+        def report!(page, error_code)
+          report = if QA::Runtime::Env.browser == :chrome
+                     return_chrome_errors(page, error_code)
+                   else
+                     status_code_report(error_code)
+                   end
+
+          raise "#{report}\n\n"\
+            "Path: #{page.current_path}"
+        end
+
+        def return_chrome_errors(page, error_code)
+          severe_errors = logs(page).select { |log| log.level == 'SEVERE' }
+          if severe_errors.none?
+            status_code_report(error_code)
+          else
+            "There #{severe_errors.count == 1 ? 'was' : 'were'} #{severe_errors.count} "\
+              "SEVERE level error#{severe_errors.count == 1 ? '' : 's'}:\n\n#{error_report_for(severe_errors)}"
+          end
+        end
+
+        def status_code_report(error_code)
+          "Status code #{error_code} found"
+        end
+
+        def check_page_for_error_code(page)
+          error_code = 0
+          # Test for 404 img alt
+          error_code = 404 if Nokogiri::HTML.parse(page.html).xpath("//img").map { |t| t[:alt] }.first.eql?('404')
+
+          # 500 error page in header surrounded by newlines, try to match
+          five_hundred_test = Nokogiri::HTML.parse(page.html).xpath("//h1").map.first
+          unless five_hundred_test.nil?
+            error_code = 500 if five_hundred_test.text.include?('500')
+          end
+          # GDK shows backtrace rather than error page
+          error_code = 500 if Nokogiri::HTML.parse(page.html).xpath("//body//section").map { |t| t[:class] }.first.eql?('backtrace')
+
+          unless error_code == 0
+            report!(page, error_code)
+          end
+        end
+
+        def error_report_for(logs)
+          logs
+              .map(&:message)
+              .map { |message| message.gsub('\\n', "\n") }
+        end
+
+        def logs(page)
+          page.driver.browser.manage.logs.get(:browser)
+        end
+      end
+    end
+  end
+end

qa/qa/support/wait_for_requests.rb

@@ -7,7 +7,12 @@ module QA
 
       DEFAULT_MAX_WAIT_TIME = 60
 
-      def wait_for_requests(skip_finished_loading_check: false)
+      def wait_for_requests(skip_finished_loading_check: false, skip_resp_code_check: false)
+        # We have tests that use 404 pages, allow them to skip this check
+        unless skip_resp_code_check
+          QA::Support::PageErrorChecker.check_page_for_error_code(Capybara.page)
+        end
+
         Waiter.wait_until(log: false) do
           finished_all_ajax_requests? && (!skip_finished_loading_check ? finished_loading?(wait: 1) : true)
         end

qa/spec/page/logging_spec.rb

@@ -14,6 +14,7 @@ RSpec.describe QA::Support::Page::Logging do
     allow(page).to receive(:find).and_return(page)
     allow(page).to receive(:current_url).and_return('http://current-url')
     allow(page).to receive(:has_css?).with(any_args).and_return(true)
+    allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).and_return(0)
   end
 
   subject do

qa/spec/resource/base_spec.rb

@@ -277,17 +277,30 @@ RSpec.describe QA::Resource::Base do
   describe '#visit!' do
     include_context 'with simple resource'
 
+    let(:wait_for_requests_class) { QA::Support::WaitForRequests }
+
     before do
       allow(resource).to receive(:visit)
     end
 
     it 'calls #visit with the underlying #web_url' do
       allow(resource).to receive(:current_url).and_return(subject.current_url)
+      expect(wait_for_requests_class).to receive(:wait_for_requests).with({ skip_resp_code_check: false }).twice
 
       resource.web_url = subject.current_url
       resource.visit!
 
       expect(resource).to have_received(:visit).with(subject.current_url)
     end
+
+    it 'calls #visit with the underlying #web_url with skip_resp_code_check specified as true' do
+      allow(resource).to receive(:current_url).and_return(subject.current_url)
+      expect(wait_for_requests_class).to receive(:wait_for_requests).with({ skip_resp_code_check: true }).twice
+
+      resource.web_url = subject.current_url
+      resource.visit!(skip_resp_code_check: true)
+
+      expect(resource).to have_received(:visit).with(subject.current_url)
+    end
   end
 end

qa/spec/support/page_error_checker_spec.rb

+# frozen_string_literal: true
+
+RSpec.describe QA::Support::PageErrorChecker do
+  let(:test_path) { '/test/path' }
+
+  let(:page) { double(Capybara.page) }
+
+  describe '.report!' do
+    context 'reports errors' do
+      let(:expected_chrome_error) do
+        "chrome errors\n\n"\
+        "Path: #{test_path}"
+      end
+
+      let(:expected_basic_error) do
+        "foo status\n\n"\
+        "Path: #{test_path}"
+      end
+
+      it 'reports error message on chrome browser' do
+        allow(QA::Support::PageErrorChecker).to receive(:return_chrome_errors).and_return('chrome errors')
+        allow(page).to receive(:current_path).and_return(test_path)
+        allow(QA::Runtime::Env).to receive(:browser).and_return(:chrome)
+
+        expect { QA::Support::PageErrorChecker.report!(page, 500) }.to raise_error(RuntimeError, expected_chrome_error)
+      end
+
+      it 'reports basic message on non-chrome browser' do
+        allow(QA::Support::PageErrorChecker).to receive(:status_code_report).and_return('foo status')
+        allow(page).to receive(:current_path).and_return(test_path)
+        allow(QA::Runtime::Env).to receive(:browser).and_return(:firefox)
+
+        expect { QA::Support::PageErrorChecker.report!(page, 500) }.to raise_error(RuntimeError, expected_basic_error)
+      end
+    end
+  end
+
+  describe '.return_chrome_errors' do
+    context 'returns error message' do
+      before do
+        single_log = Class.new do
+          def level
+            'SEVERE'
+          end
+        end
+        stub_const('SingleLog', single_log)
+        one_error_mocked_logs = Class.new do
+          def self.select
+            [SingleLog]
+          end
+        end
+        stub_const('OneErrorMockedLogs', one_error_mocked_logs)
+        three_errors_mocked_logs = Class.new do
+          def self.select
+            [SingleLog, SingleLog, SingleLog]
+          end
+        end
+        stub_const('ThreeErrorsMockedLogs', three_errors_mocked_logs)
+        no_error_mocked_logs = Class.new do
+          def self.select
+            []
+          end
+        end
+        stub_const('NoErrorMockedLogs', no_error_mocked_logs)
+      end
+
+      let(:expected_single_error) do
+        "There was 1 SEVERE level error:\n\n"\
+        "bar foo"
+      end
+
+      let(:expected_multiple_error) do
+        "There were 3 SEVERE level errors:\n\n"\
+        "bar foo\n"\
+        "foo\n"\
+        "bar"
+      end
+
+      it 'returns status code report on no severe errors found' do
+        allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(NoErrorMockedLogs)
+        allow(QA::Support::PageErrorChecker).to receive(:status_code_report).with('123').and_return('Test Status Code return 123')
+
+        expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq('Test Status Code return 123')
+      end
+
+      it 'returns report on 1 severe error found' do
+        allow(QA::Support::PageErrorChecker).to receive(:error_report_for).with([SingleLog]).and_return('bar foo')
+        allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(OneErrorMockedLogs)
+        allow(page).to receive(:current_path).and_return(test_path)
+
+        expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq(expected_single_error)
+      end
+
+      it 'returns report on multiple severe errors found' do
+        allow(QA::Support::PageErrorChecker).to receive(:error_report_for)
+                                                    .with([SingleLog, SingleLog, SingleLog]).and_return("bar foo\nfoo\nbar")
+        allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(ThreeErrorsMockedLogs)
+        allow(page).to receive(:current_path).and_return(test_path)
+
+        expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq(expected_multiple_error)
+      end
+    end
+  end
+
+  describe '.check_page_for_error_code' do
+    require 'nokogiri'
+    before do
+      nokogiri_parse = Class.new do
+        def self.parse(str)
+          Nokogiri::HTML.parse(str)
+        end
+      end
+      stub_const('NokogiriParse', nokogiri_parse)
+    end
+    let(:error_404_str) do
+      "<div class=\"error\">"\
+        "<img src=\"404.png\" alt=\"404\" />"\
+      "</div>"
+    end
+
+    let(:error_500_str) { "<h1>   500   </h1>"}
+    let(:backtrace_str) {"<body><section class=\"backtrace\">foo</section></body>"}
+    let(:no_error_str) {"<body>no 404 or 500 or backtrace</body>"}
+
+    it 'calls report with 404 if 404 found' do
+      allow(page).to receive(:html).and_return(error_404_str)
+      allow(Nokogiri::HTML).to receive(:parse).with(error_404_str).and_return(NokogiriParse.parse(error_404_str))
+
+      expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 404)
+      QA::Support::PageErrorChecker.check_page_for_error_code(page)
+    end
+    it 'calls report with 500 if 500 found' do
+      allow(page).to receive(:html).and_return(error_500_str)
+      allow(Nokogiri::HTML).to receive(:parse).with(error_500_str).and_return(NokogiriParse.parse(error_500_str))
+
+      expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 500)
+      QA::Support::PageErrorChecker.check_page_for_error_code(page)
+    end
+    it 'calls report with 500 if GDK backtrace found' do
+      allow(page).to receive(:html).and_return(backtrace_str)
+      allow(Nokogiri::HTML).to receive(:parse).with(backtrace_str).and_return(NokogiriParse.parse(backtrace_str))
+
+      expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 500)
+      QA::Support::PageErrorChecker.check_page_for_error_code(page)
+    end
+    it 'does not call report if no 404, 500 or backtrace found' do
+      allow(page).to receive(:html).and_return(no_error_str)
+      allow(Nokogiri::HTML).to receive(:parse).with(no_error_str).and_return(NokogiriParse.parse(no_error_str))
+
+      expect(QA::Support::PageErrorChecker).not_to receive(:report!)
+      QA::Support::PageErrorChecker.check_page_for_error_code(page)
+    end
+  end
+
+  describe '.error_report_for' do
+    before do
+      logs_class_one = Class.new do
+        def self.message
+          'foo\\n'
+        end
+      end
+      stub_const('LogOne', logs_class_one)
+      logs_class_two = Class.new do
+        def self.message
+          'bar'
+        end
+      end
+      stub_const('LogTwo', logs_class_two)
+    end
+
+    it 'returns error report array of log messages' do
+      expect(QA::Support::PageErrorChecker.error_report_for([LogOne, LogTwo]))
+          .to eq(%W(foo\n bar))
+    end
+  end
+
+  describe '.logs' do
+    before do
+      logs_class = Class.new do
+        def self.get(level)
+          "logs at #{level} level"
+        end
+      end
+      stub_const('Logs', logs_class)
+      manage_class = Class.new do
+        def self.logs
+          Logs
+        end
+      end
+      stub_const('Manage', manage_class)
+      browser_class = Class.new do
+        def self.manage
+          Manage
+        end
+      end
+      stub_const('Browser', browser_class)
+      driver_class = Class.new do
+        def self.browser
+          Browser
+        end
+      end
+      stub_const('Driver', driver_class)
+    end
+
+    it 'gets driver browser logs' do
+      allow(page).to receive(:driver).and_return(Driver)
+
+      expect(QA::Support::PageErrorChecker.logs(page)).to eq('logs at browser level')
+    end
+  end
+
+  describe '.status_code_report' do
+    it 'returns a string message containing the status code' do
+      expect(QA::Support::PageErrorChecker.status_code_report(1234)).to eq('Status code 1234 found')
+    end
+  end
+end

qa/spec/support/wait_for_requests_spec.rb

@@ -22,5 +22,21 @@ RSpec.describe QA::Support::WaitForRequests do
         subject.wait_for_requests(skip_finished_loading_check: true)
       end
     end
+
+    context 'when skip_resp_code_check is defaulted to false' do
+      it 'call report' do
+        allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).with(Capybara.page)
+
+        subject.wait_for_requests
+      end
+    end
+
+    context 'when skip_resp_code_check is true' do
+      it 'does not parse for an error code' do
+        expect(QA::Support::PageErrorChecker).not_to receive(:check_page_for_error_code)
+
+        subject.wait_for_requests(skip_resp_code_check: true)
+      end
+    end
   end
 end