test_result: Retry failed test result lines

If a test result finish and its title match the test suite pattern allowed to
restart defined on test suite, it will be retried at most once.

Note about the test: The current behavior is to use PASS/FAIL for test result
and PASSED/FAILED for test result line. The test checks the actual values, if
we ever fix this inconsistency, we can safely update the test I believe.

bt5/erp5_test_result/TestTemplateItem/portal_components/test.erp5.testTaskDistribution.py

@@ -1418,6 +1418,90 @@ class TestTaskDistribution(TaskDistributionTestCase):
     self.assertEqual(None, test_suite.getAlarmDate())
 
 
+class TestRetryFailedTest(TaskDistributionTestCase):
+  """Test how failed tests can be automatically retried.
+  """
+
+  def afterSetUp(self):
+    super(TestRetryFailedTest, self).afterSetUp()
+    self.test_suite, = self.test_suite_module.objectValues()
+    self._createTestNode()
+    self.tic()
+
+  def test_failed_test_not_retried_by_default(self):
+    test_result_path, _ = self._createTestResult(test_list=['testFoo', ])
+    test_result = self.portal.unrestrictedTraverse(test_result_path)
+    line_url, _ = self.tool.startUnitTest(test_result_path)
+    test_result_line = self.portal.restrictedTraverse(line_url)
+    status_dict = {
+        'test_count': 100,
+        'error_count': 2,
+        'failure_count': 3,
+    }
+    self.tool.stopUnitTest(line_url, status_dict)
+    self.tic()
+    self.assertEqual(test_result_line.getStringIndex(), 'FAILED')
+    self.assertEqual(test_result_line.getSimulationState(), 'stopped')
+    self.assertEqual(test_result.getStringIndex(), 'FAIL')
+    self.assertEqual(test_result.getSimulationState(), 'stopped')
+
+  def test_failed_retried_once_then_fail(self):
+    self.test_suite.setRetryTestPattern('testF.*')
+    test_result_path, _ = self._createTestResult(test_list=['testFoo', ])
+    test_result = self.portal.unrestrictedTraverse(test_result_path)
+    line_url, _ = self.tool.startUnitTest(test_result_path)
+    test_result_line = self.portal.restrictedTraverse(line_url)
+    status_dict = {
+        'test_count': 100,
+        'error_count': 2,
+        'failure_count': 3,
+    }
+    self.tool.stopUnitTest(line_url, status_dict)
+    self.tic()
+    # test failed, but it will be retried
+    self.assertEqual(test_result_line.getStringIndex(), 'RETRYING')
+    self.assertEqual(test_result_line.getSimulationState(), 'draft')
+    # if it fails again ...
+    self.tool.stopUnitTest(line_url, status_dict)
+    self.tic()
+    # ... the test result will be fail.
+    self.assertEqual(test_result_line.getStringIndex(), 'FAILED')
+    self.assertEqual(test_result_line.getSimulationState(), 'stopped')
+    self.assertEqual(test_result.getStringIndex(), 'FAIL')
+    self.assertEqual(test_result.getSimulationState(), 'stopped')
+    self.assertEqual(test_result.getProperty('errors'), 2)
+    self.assertEqual(test_result.getProperty('failures'), 3)
+    self.assertEqual(test_result.getProperty('test_result_retry_count'), 1)
+
+  def test_failed_retried_once_then_pass(self):
+    self.test_suite.setRetryTestPattern('testF.*')
+    test_result_path, _ = self._createTestResult(test_list=['testFoo', ])
+    test_result = self.portal.unrestrictedTraverse(test_result_path)
+    line_url, _ = self.tool.startUnitTest(test_result_path)
+    test_result_line = self.portal.restrictedTraverse(line_url)
+    status_dict = {
+        'test_count': 100,
+        'error_count': 2,
+        'failure_count': 3,
+    }
+    self.tool.stopUnitTest(line_url, status_dict)
+    self.tic()
+    # test failed, but it will be retried
+    self.assertEqual(test_result_line.getStringIndex(), 'RETRYING')
+    self.assertEqual(test_result_line.getSimulationState(), 'draft')
+    # if it succeed next time ...
+    status_dict['error_count'] = 0
+    status_dict['failure_count'] = 0
+    self.tool.stopUnitTest(line_url, status_dict)
+    self.tic()
+    # ... the test result will be successful.
+    self.assertEqual(test_result_line.getStringIndex(), 'PASSED')
+    self.assertEqual(test_result_line.getSimulationState(), 'stopped')
+    self.assertEqual(test_result.getStringIndex(), 'PASS')
+    self.assertEqual(test_result.getSimulationState(), 'stopped')
+    self.assertEqual(test_result.getProperty('test_result_retry_count'), 1)
+
+
 class TestGitlabRESTConnectorInterface(ERP5TypeTestCase):
   """Tests for Gitlab commits annotations.
   """

bt5/erp5_test_result/WorkflowTemplateItem/portal_workflow/test_result_workflow/scripts/TestResult_complete.py

+import re
 test_result = sci['object']
 kw = sci['kwargs']
 test_result.setStopDate(kw.get('date') or DateTime())
@@ -8,6 +9,30 @@ def unexpected(test_result):
   # passed if there's no unexpected failures.
   return test_result.getSourceProjectTitle() != "NEO R&D"
 
+
+def shouldRetry(test_result_line):
+  # type: (erp5.portal_type.TestResultLine,) -> bool
+  """Should the test result line be retried ?
+
+  We retry test result line once for tests matching pattern defined on test suite.
+  Unless if there's already another failed test result line, in that case we don't retry.
+  """
+  if test_result_line.getProperty('test_result_retry_count') or 0:
+    return False
+
+  test_result = test_result_line.getParentValue()
+  for other_test_result_line in test_result.contentValues(portal_type='Test Result Line'):
+    if test_result_line != other_test_result_line and other_test_result_line.getStringIndex() in ('UNKNOWN', 'FAILED'):
+      return False
+
+  test_suite_data = test_result.TestResult_getTestSuiteData()
+  if not test_suite_data:
+    return False
+  if not test_suite_data['retry_test_pattern']:
+    return False
+  return re.search(test_suite_data['retry_test_pattern'], test_result_line.getTitle() or '')
+
+
 if test_result.getPortalType() == 'Test Result':
   has_unknown_result = False
   edit_kw = dict(duration=0,
@@ -48,12 +73,12 @@ elif test_result.getPortalType() == 'Test Result Line':
   duration = kw.get('duration')
   if duration is None:
     duration = (test_result.getStopDate() - test_result.getStartDate()) * (24*60*60)
-  cmdline = kw.get('command', getattr(test_result, 'cmdline', ''))
+  cmdline = kw.get('command', '')
   if same_type(cmdline, []):
     cmdline = ' '.join(map(repr, cmdline))
-  stdout = kw.get('stdout', getattr(test_result, 'stdout', ''))
-  stderr = kw.get('stderr', getattr(test_result, 'stderr', ''))
-  html_test_result = kw.get('html_test_result', getattr(test_result, 'html_test_result', ''))
+  stdout = kw.get('stdout', '')
+  stderr = kw.get('stderr', '')
+  html_test_result = kw.get('html_test_result', '')
   test_result.edit(cmdline=cmdline,
                    stdout=stdout,
                    stderr=stderr,
@@ -64,5 +89,11 @@ elif test_result.getPortalType() == 'Test Result Line':
                    failures=failures,
                    skips=skips,
                    html_test_result=html_test_result)
+  if status == 'FAILED' and shouldRetry(test_result):
+    test_result.edit(
+        test_result_retry_count=1 + (test_result.getProperty('test_result_retry_count') or 0),
+        string_index='RETRYING',
+    )
+    test_result.redraft(comment="Retried after a first failure")
 else:
   raise NotImplementedError("unknown type : %r" % test_result.getPortalType())

bt5/erp5_test_result/WorkflowTemplateItem/portal_workflow/test_result_workflow/states/stopped.xml

@@ -32,6 +32,7 @@
                 <string>cancel_action</string>
                 <string>fail</string>
                 <string>publish_stopped</string>
+                <string>redraft</string>
               </tuple>
             </value>
         </item>