test_result: Retry flaky tests

ERP5 functional tests are failing too often, until we improve the tests or the
code so that we don't have these intermitent failures, restart the failed tests
a bit automatically.

This is of course not good, because it's hiding problems, but the current state
of the test suite with many randomly failing test makes that we sometimes miss
when permanent failures are introduced.

See merge request nexedi/erp5!1206

bt5/erp5_test_result/PropertySheetTemplateItem/portal_property_sheets/TestSuite/retry_test_pattern_property.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Standard Property" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>categories</string> </key>
+            <value>
+              <tuple>
+                <string>elementary_type/string</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value> <string>Test results whose title match this (regular expression) pattern will be retried once if they fail.</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>retry_test_pattern_property</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Standard Property</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestResultLine_view.xml

@@ -103,6 +103,7 @@
                         <string>my_failures</string>
                         <string>my_errors</string>
                         <string>my_skips</string>
+                        <string>my_test_result_retry_count</string>
                         <string>my_string_index</string>
                         <string>my_translated_simulation_state_title</string>
                       </list>

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestResult_getTestSuiteData.py

 """Returns info about a test result, a mapping containing:
 
   test_suite_relative_url: relative url of test suite
+  retry_test_pattern: the pattern for test result lines which can be restarted
   repository_dict: for each test suite repository, keyed by buildout section id:
      revision: commit sha
      repository_url: git url of the repository
@@ -53,5 +54,6 @@ if REQUEST:
 
 return {
     'test_suite_relative_url': test_suite.getRelativeUrl(),
+    'retry_test_pattern': test_suite.getRetryTestPattern(),
     'repository_dict': repository_dict,
 }

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestResult_statTestResultLineList.py

@@ -4,4 +4,6 @@ return [Object(duration=int(context.getProperty('duration', 0)),
                all_tests=context.getProperty('all_tests'),
                errors=context.getProperty('errors'),
                failures=context.getProperty('failures'),
-               skips=context.getProperty('skips'))]
+               skips=context.getProperty('skips'),
+               test_result_retry_count=context.getProperty('test_result_retry_count', 0),
+               )]

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestResult_view.xml

@@ -107,6 +107,7 @@
                         <string>my_failures</string>
                         <string>my_errors</string>
                         <string>my_skips</string>
+                        <string>my_test_result_retry_count</string>
                         <string>my_string_index</string>
                         <string>my_translated_simulation_state_title</string>
                       </list>

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestResult_view/listbox.xml

@@ -371,6 +371,10 @@
                           <string>skips</string>
                           <string>Skips</string>
                         </tuple>
+                        <tuple>
+                          <string>test_result_retry_count</string>
+                          <string>Retries</string>
+                        </tuple>
                         <tuple>
                           <string>string_index</string>
                           <string>Result</string>

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestSuite_view.xml

@@ -104,6 +104,7 @@
                         <string>my_additional_bt5_repository_id</string>
                         <string>my_source_project_title</string>
                         <string>my_specialise_title</string>
+                        <string>my_retry_test_pattern</string>
                       </list>
                     </value>
                 </item>

bt5/erp5_test_result/WorkflowTemplateItem/portal_workflow/test_result_workflow/scripts/TestResult_complete.py

+import re
 test_result = sci['object']
 kw = sci['kwargs']
 test_result.setStopDate(kw.get('date') or DateTime())
@@ -8,13 +9,38 @@ def unexpected(test_result):
   # passed if there's no unexpected failures.
   return test_result.getSourceProjectTitle() != "NEO R&D"
 
+
+def shouldRetry(test_result_line):
+  # type: (erp5.portal_type.TestResultLine,) -> bool
+  """Should the test result line be retried ?
+
+  We retry test result line once for tests matching pattern defined on test suite.
+  Unless if there's already another failed test result line, in that case we don't retry.
+  """
+  if test_result_line.getProperty('test_result_retry_count') or 0:
+    return False
+
+  test_result = test_result_line.getParentValue()
+  for other_test_result_line in test_result.contentValues(portal_type='Test Result Line'):
+    if test_result_line != other_test_result_line and other_test_result_line.getStringIndex() in ('UNKNOWN', 'FAILED'):
+      return False
+
+  test_suite_data = test_result.TestResult_getTestSuiteData()
+  if not test_suite_data:
+    return False
+  if not test_suite_data['retry_test_pattern']:
+    return False
+  return re.search(test_suite_data['retry_test_pattern'], test_result_line.getTitle() or '')
+
+
 if test_result.getPortalType() == 'Test Result':
   has_unknown_result = False
   edit_kw = dict(duration=0,
                  all_tests=0,
                  errors=0,
                  failures=0,
-                 skips=0)
+                 skips=0,
+                 test_result_retry_count=0)
   for line in test_result.objectValues(portal_type='Test Result Line'):
     for prop in edit_kw:
       try:
@@ -47,12 +73,12 @@ elif test_result.getPortalType() == 'Test Result Line':
   duration = kw.get('duration')
   if duration is None:
     duration = (test_result.getStopDate() - test_result.getStartDate()) * (24*60*60)
-  cmdline = kw.get('command', getattr(test_result, 'cmdline', ''))
+  cmdline = kw.get('command', '')
   if same_type(cmdline, []):
     cmdline = ' '.join(map(repr, cmdline))
-  stdout = kw.get('stdout', getattr(test_result, 'stdout', ''))
-  stderr = kw.get('stderr', getattr(test_result, 'stderr', ''))
-  html_test_result = kw.get('html_test_result', getattr(test_result, 'html_test_result', ''))
+  stdout = kw.get('stdout', '')
+  stderr = kw.get('stderr', '')
+  html_test_result = kw.get('html_test_result', '')
   test_result.edit(cmdline=cmdline,
                    stdout=stdout,
                    stderr=stderr,
@@ -63,5 +89,11 @@ elif test_result.getPortalType() == 'Test Result Line':
                    failures=failures,
                    skips=skips,
                    html_test_result=html_test_result)
+  if status == 'FAILED' and shouldRetry(test_result):
+    test_result.edit(
+        test_result_retry_count=1 + (test_result.getProperty('test_result_retry_count') or 0),
+        string_index='RETRYING',
+    )
+    test_result.redraft(comment="Retried after a first failure")
 else:
   raise NotImplementedError("unknown type : %r" % test_result.getPortalType())

bt5/erp5_test_result/WorkflowTemplateItem/portal_workflow/test_result_workflow/states/stopped.xml

@@ -32,6 +32,7 @@
                 <string>cancel_action</string>
                 <string>fail</string>
                 <string>publish_stopped</string>
+                <string>redraft</string>
               </tuple>
             </value>
         </item>