Merge branch '32046-differentiate-between-errors-failures-in-xunit-result' into 'master'

Resolve "Differentiate between errors/failures in xUnit result"

See merge request gitlab-org/gitlab!23476

app/assets/javascripts/reports/store/utils.js

@@ -11,9 +11,10 @@ const textBuilder = results => {
   const { failed, errored, resolved, total } = results;
 
   const failedOrErrored = (failed || 0) + (errored || 0);
-  const failedString = failedOrErrored
-    ? n__('%d failed/error test result', '%d failed/error test results', failedOrErrored)
-    : null;
+  const failedString = failed ? n__('%d failed', '%d failed', failed) : null;
+  const erroredString = errored ? n__('%d error', '%d errors', errored) : null;
+  const combinedString =
+    failed && errored ? `${failedString}, ${erroredString}` : failedString || erroredString;
   const resolvedString = resolved
     ? n__('%d fixed test result', '%d fixed test results', resolved)
     : null;
@@ -23,12 +24,12 @@ const textBuilder = results => {
 
   if (failedOrErrored) {
     if (resolved) {
-      resultsString = sprintf(s__('Reports|%{failedString} and %{resolvedString}'), {
-        failedString,
+      resultsString = sprintf(s__('Reports|%{combinedString} and %{resolvedString}'), {
+        combinedString,
         resolvedString,
       });
     } else {
-      resultsString = failedString;
+      resultsString = combinedString;
     }
   } else if (resolved) {
     resultsString = resolvedString;

changelogs/unreleased/32046-differentiate-between-errors-failures-in-xunit-result.yml

+---
+title: Differentiate between errors and failures in xUnit result
+merge_request: 23476
+author:
+type: changed

doc/ci/junit_test_reports.md

@@ -42,13 +42,15 @@ JUnit test reports, where:
 - The base branch is the target branch (usually `master`).
 - The head branch is the source branch (the latest pipeline in each merge request).
 
-The reports panel has a summary showing how many tests failed and how many were fixed.
-If no comparison can be done because data for the base branch is not available,
-the panel will just show the list of failed tests for head.
+The reports panel has a summary showing how many tests failed, how many had errors
+and how many were fixed. If no comparison can be done because data for the base branch
+is not available, the panel will just show the list of failed tests for head.
 
-There are three types of results:
+There are four types of results:
 
 1. **Newly failed tests:** Test cases which passed on base branch and failed on head branch
+1. **Newly encountered errors:** Test cases which passed on base branch and failed due to a
+   test error on head branch
 1. **Existing failures:**  Test cases which failed on base branch and failed on head branch
 1. **Resolved failures:**  Test cases which failed on base branch and passed on head branch
 

locale/gitlab.pot

@@ -106,13 +106,18 @@ msgid_plural "%d contributions"
 msgstr[0] ""
 msgstr[1] ""
 
+msgid "%d error"
+msgid_plural "%d errors"
+msgstr[0] ""
+msgstr[1] ""
+
 msgid "%d exporter"
 msgid_plural "%d exporters"
 msgstr[0] ""
 msgstr[1] ""
 
-msgid "%d failed/error test result"
-msgid_plural "%d failed/error test results"
+msgid "%d failed"
+msgid_plural "%d failed"
 msgstr[0] ""
 msgstr[1] ""
 
@@ -16578,7 +16583,7 @@ msgstr ""
 msgid "Reporting"
 msgstr ""
 
-msgid "Reports|%{failedString} and %{resolvedString}"
+msgid "Reports|%{combinedString} and %{resolvedString}"
 msgstr ""
 
 msgid "Reports|Actions"

spec/features/merge_request/user_sees_merge_widget_spec.rb

@@ -585,10 +585,10 @@ describe 'Merge request > User sees merge widget', :js do
           within(".js-reports-container") do
             click_button 'Expand'
 
-            expect(page).to have_content('Test summary contained 1 failed/error test result out of 2 total tests')
+            expect(page).to have_content('Test summary contained 1 failed out of 2 total tests')
             within(".js-report-section-container") do
               expect(page).to have_content('rspec found no changed test results out of 1 total test')
-              expect(page).to have_content('junit found 1 failed/error test result out of 1 total test')
+              expect(page).to have_content('junit found 1 failed out of 1 total test')
               expect(page).to have_content('New')
               expect(page).to have_content('addTest')
             end
@@ -630,9 +630,9 @@ describe 'Merge request > User sees merge widget', :js do
           within(".js-reports-container") do
             click_button 'Expand'
 
-            expect(page).to have_content('Test summary contained 1 failed/error test result out of 2 total tests')
+            expect(page).to have_content('Test summary contained 1 failed out of 2 total tests')
             within(".js-report-section-container") do
-              expect(page).to have_content('rspec found 1 failed/error test result out of 1 total test')
+              expect(page).to have_content('rspec found 1 failed out of 1 total test')
               expect(page).to have_content('junit found no changed test results out of 1 total test')
               expect(page).not_to have_content('New')
               expect(page).to have_content('Test#sum when a is 1 and b is 3 returns summary')
@@ -718,10 +718,10 @@ describe 'Merge request > User sees merge widget', :js do
           within(".js-reports-container") do
             click_button 'Expand'
 
-            expect(page).to have_content('Test summary contained 1 failed/error test result out of 2 total tests')
+            expect(page).to have_content('Test summary contained 1 error out of 2 total tests')
             within(".js-report-section-container") do
               expect(page).to have_content('rspec found no changed test results out of 1 total test')
-              expect(page).to have_content('junit found 1 failed/error test result out of 1 total test')
+              expect(page).to have_content('junit found 1 error out of 1 total test')
               expect(page).to have_content('New')
               expect(page).to have_content('addTest')
             end
@@ -762,9 +762,9 @@ describe 'Merge request > User sees merge widget', :js do
           within(".js-reports-container") do
             click_button 'Expand'
 
-            expect(page).to have_content('Test summary contained 1 failed/error test result out of 2 total tests')
+            expect(page).to have_content('Test summary contained 1 error out of 2 total tests')
             within(".js-report-section-container") do
-              expect(page).to have_content('rspec found 1 failed/error test result out of 1 total test')
+              expect(page).to have_content('rspec found 1 error out of 1 total test')
               expect(page).to have_content('junit found no changed test results out of 1 total test')
               expect(page).not_to have_content('New')
               expect(page).to have_content('Test#sum when a is 4 and b is 4 returns summary')
@@ -857,10 +857,10 @@ describe 'Merge request > User sees merge widget', :js do
           within(".js-reports-container") do
             click_button 'Expand'
 
-            expect(page).to have_content('Test summary contained 20 failed/error test results out of 20 total tests')
+            expect(page).to have_content('Test summary contained 20 failed out of 20 total tests')
             within(".js-report-section-container") do
-              expect(page).to have_content('rspec found 10 failed/error test results out of 10 total tests')
-              expect(page).to have_content('junit found 10 failed/error test results out of 10 total tests')
+              expect(page).to have_content('rspec found 10 failed out of 10 total tests')
+              expect(page).to have_content('junit found 10 failed out of 10 total tests')
 
               expect(page).to have_content('Test#sum when a is 1 and b is 3 returns summary', count: 2)
             end

spec/frontend/reports/store/utils_spec.js

@@ -30,9 +30,7 @@ describe('Reports store utils', () => {
       const data = { failed: 3, total: 10 };
       const result = utils.summaryTextBuilder(name, data);
 
-      expect(result).toBe(
-        'Test summary contained 3 failed/error test results out of 10 total tests',
-      );
+      expect(result).toBe('Test summary contained 3 failed out of 10 total tests');
     });
 
     it('should render text for multiple errored results', () => {
@@ -40,9 +38,7 @@ describe('Reports store utils', () => {
       const data = { errored: 7, total: 10 };
       const result = utils.summaryTextBuilder(name, data);
 
-      expect(result).toBe(
-        'Test summary contained 7 failed/error test results out of 10 total tests',
-      );
+      expect(result).toBe('Test summary contained 7 errors out of 10 total tests');
     });
 
     it('should render text for multiple fixed results', () => {
@@ -59,7 +55,7 @@ describe('Reports store utils', () => {
       const result = utils.summaryTextBuilder(name, data);
 
       expect(result).toBe(
-        'Test summary contained 3 failed/error test results and 4 fixed test results out of 10 total tests',
+        'Test summary contained 3 failed and 4 fixed test results out of 10 total tests',
       );
     });
 
@@ -69,18 +65,17 @@ describe('Reports store utils', () => {
       const result = utils.summaryTextBuilder(name, data);
 
       expect(result).toBe(
-        'Test summary contained 1 failed/error test result and 1 fixed test result out of 10 total tests',
+        'Test summary contained 1 failed and 1 fixed test result out of 10 total tests',
       );
     });
 
     it('should render text for singular failed, errored, and fixed results', () => {
-      // these will be singular when the copy is updated
       const name = 'Test summary';
       const data = { failed: 1, errored: 1, resolved: 1, total: 10 };
       const result = utils.summaryTextBuilder(name, data);
 
       expect(result).toBe(
-        'Test summary contained 2 failed/error test results and 1 fixed test result out of 10 total tests',
+        'Test summary contained 1 failed, 1 error and 1 fixed test result out of 10 total tests',
       );
     });
 
@@ -90,7 +85,7 @@ describe('Reports store utils', () => {
       const result = utils.summaryTextBuilder(name, data);
 
       expect(result).toBe(
-        'Test summary contained 5 failed/error test results and 4 fixed test results out of 10 total tests',
+        'Test summary contained 2 failed, 3 errors and 4 fixed test results out of 10 total tests',
       );
     });
   });
@@ -117,7 +112,7 @@ describe('Reports store utils', () => {
       const data = { failed: 3, total: 10 };
       const result = utils.reportTextBuilder(name, data);
 
-      expect(result).toBe('Rspec found 3 failed/error test results out of 10 total tests');
+      expect(result).toBe('Rspec found 3 failed out of 10 total tests');
     });
 
     it('should render text for multiple errored results', () => {
@@ -125,7 +120,7 @@ describe('Reports store utils', () => {
       const data = { errored: 7, total: 10 };
       const result = utils.reportTextBuilder(name, data);
 
-      expect(result).toBe('Rspec found 7 failed/error test results out of 10 total tests');
+      expect(result).toBe('Rspec found 7 errors out of 10 total tests');
     });
 
     it('should render text for multiple fixed results', () => {
@@ -141,9 +136,7 @@ describe('Reports store utils', () => {
       const data = { failed: 3, resolved: 4, total: 10 };
       const result = utils.reportTextBuilder(name, data);
 
-      expect(result).toBe(
-        'Rspec found 3 failed/error test results and 4 fixed test results out of 10 total tests',
-      );
+      expect(result).toBe('Rspec found 3 failed and 4 fixed test results out of 10 total tests');
     });
 
     it('should render text for a singular fixed, and a singular failed result', () => {
@@ -151,19 +144,16 @@ describe('Reports store utils', () => {
       const data = { failed: 1, resolved: 1, total: 10 };
       const result = utils.reportTextBuilder(name, data);
 
-      expect(result).toBe(
-        'Rspec found 1 failed/error test result and 1 fixed test result out of 10 total tests',
-      );
+      expect(result).toBe('Rspec found 1 failed and 1 fixed test result out of 10 total tests');
     });
 
     it('should render text for singular failed, errored, and fixed results', () => {
-      // these will be singular when the copy is updated
       const name = 'Rspec';
       const data = { failed: 1, errored: 1, resolved: 1, total: 10 };
       const result = utils.reportTextBuilder(name, data);
 
       expect(result).toBe(
-        'Rspec found 2 failed/error test results and 1 fixed test result out of 10 total tests',
+        'Rspec found 1 failed, 1 error and 1 fixed test result out of 10 total tests',
       );
     });
 
@@ -173,7 +163,7 @@ describe('Reports store utils', () => {
       const result = utils.reportTextBuilder(name, data);
 
       expect(result).toBe(
-        'Rspec found 5 failed/error test results and 4 fixed test results out of 10 total tests',
+        'Rspec found 2 failed, 3 errors and 4 fixed test results out of 10 total tests',
       );
     });
   });

spec/javascripts/reports/components/grouped_test_reports_app_spec.js

@@ -84,12 +84,10 @@ describe('Grouped Test Reports App', () => {
       setTimeout(() => {
         expect(vm.$el.querySelector('.gl-spinner')).toBeNull();
         expect(vm.$el.querySelector('.js-code-text').textContent.trim()).toEqual(
-          'Test summary contained 2 failed/error test results out of 11 total tests',
+          'Test summary contained 2 failed out of 11 total tests',
         );
 
-        expect(vm.$el.textContent).toContain(
-          'rspec:pg found 2 failed/error test results out of 8 total tests',
-        );
+        expect(vm.$el.textContent).toContain('rspec:pg found 2 failed out of 8 total tests');
 
         expect(vm.$el.textContent).toContain('New');
         expect(vm.$el.textContent).toContain(
@@ -112,12 +110,10 @@ describe('Grouped Test Reports App', () => {
       setTimeout(() => {
         expect(vm.$el.querySelector('.gl-spinner')).toBeNull();
         expect(vm.$el.querySelector('.js-code-text').textContent.trim()).toEqual(
-          'Test summary contained 2 failed/error test results out of 11 total tests',
+          'Test summary contained 2 errors out of 11 total tests',
         );
 
-        expect(vm.$el.textContent).toContain(
-          'karma found 2 failed/error test results out of 3 total tests',
-        );
+        expect(vm.$el.textContent).toContain('karma found 2 errors out of 3 total tests');
 
         expect(vm.$el.textContent).toContain('New');
         expect(vm.$el.textContent).toContain(
@@ -140,17 +136,15 @@ describe('Grouped Test Reports App', () => {
       setTimeout(() => {
         expect(vm.$el.querySelector('.gl-spinner')).toBeNull();
         expect(vm.$el.querySelector('.js-code-text').textContent.trim()).toEqual(
-          'Test summary contained 2 failed/error test results and 2 fixed test results out of 11 total tests',
+          'Test summary contained 2 failed and 2 fixed test results out of 11 total tests',
         );
 
         expect(vm.$el.textContent).toContain(
-          'rspec:pg found 1 failed/error test result and 2 fixed test results out of 8 total tests',
+          'rspec:pg found 1 failed and 2 fixed test results out of 8 total tests',
         );
 
         expect(vm.$el.textContent).toContain('New');
-        expect(vm.$el.textContent).toContain(
-          ' java ant found 1 failed/error test result out of 3 total tests',
-        );
+        expect(vm.$el.textContent).toContain(' java ant found 1 failed out of 3 total tests');
         done();
       }, 0);
     });