Commit b559f4c5 authored by Jérome Perrin's avatar Jérome Perrin

test_result: Retry flaky tests

ERP5 functional tests are failing too often, until we improve the tests or the
code so that we don't have these intermitent failures, restart the failed tests
a bit automatically.

This is of course not good, because it's hiding problems, but the current state
of the test suite with many randomly failing test makes that we sometimes miss
when permanent failures are introduced.

See merge request nexedi/erp5!1206
parents 06a7737f 435dee38
Pipeline #11446 failed with stage
in 0 seconds
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/string</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Test results whose title match this (regular expression) pattern will be retried once if they fail.</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>retry_test_pattern_property</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -103,6 +103,7 @@ ...@@ -103,6 +103,7 @@
<string>my_failures</string> <string>my_failures</string>
<string>my_errors</string> <string>my_errors</string>
<string>my_skips</string> <string>my_skips</string>
<string>my_test_result_retry_count</string>
<string>my_string_index</string> <string>my_string_index</string>
<string>my_translated_simulation_state_title</string> <string>my_translated_simulation_state_title</string>
</list> </list>
......
"""Returns info about a test result, a mapping containing: """Returns info about a test result, a mapping containing:
test_suite_relative_url: relative url of test suite test_suite_relative_url: relative url of test suite
retry_test_pattern: the pattern for test result lines which can be restarted
repository_dict: for each test suite repository, keyed by buildout section id: repository_dict: for each test suite repository, keyed by buildout section id:
revision: commit sha revision: commit sha
repository_url: git url of the repository repository_url: git url of the repository
...@@ -53,5 +54,6 @@ if REQUEST: ...@@ -53,5 +54,6 @@ if REQUEST:
return { return {
'test_suite_relative_url': test_suite.getRelativeUrl(), 'test_suite_relative_url': test_suite.getRelativeUrl(),
'retry_test_pattern': test_suite.getRetryTestPattern(),
'repository_dict': repository_dict, 'repository_dict': repository_dict,
} }
...@@ -4,4 +4,6 @@ return [Object(duration=int(context.getProperty('duration', 0)), ...@@ -4,4 +4,6 @@ return [Object(duration=int(context.getProperty('duration', 0)),
all_tests=context.getProperty('all_tests'), all_tests=context.getProperty('all_tests'),
errors=context.getProperty('errors'), errors=context.getProperty('errors'),
failures=context.getProperty('failures'), failures=context.getProperty('failures'),
skips=context.getProperty('skips'))] skips=context.getProperty('skips'),
test_result_retry_count=context.getProperty('test_result_retry_count', 0),
)]
...@@ -107,6 +107,7 @@ ...@@ -107,6 +107,7 @@
<string>my_failures</string> <string>my_failures</string>
<string>my_errors</string> <string>my_errors</string>
<string>my_skips</string> <string>my_skips</string>
<string>my_test_result_retry_count</string>
<string>my_string_index</string> <string>my_string_index</string>
<string>my_translated_simulation_state_title</string> <string>my_translated_simulation_state_title</string>
</list> </list>
......
...@@ -371,6 +371,10 @@ ...@@ -371,6 +371,10 @@
<string>skips</string> <string>skips</string>
<string>Skips</string> <string>Skips</string>
</tuple> </tuple>
<tuple>
<string>test_result_retry_count</string>
<string>Retries</string>
</tuple>
<tuple> <tuple>
<string>string_index</string> <string>string_index</string>
<string>Result</string> <string>Result</string>
......
...@@ -104,6 +104,7 @@ ...@@ -104,6 +104,7 @@
<string>my_additional_bt5_repository_id</string> <string>my_additional_bt5_repository_id</string>
<string>my_source_project_title</string> <string>my_source_project_title</string>
<string>my_specialise_title</string> <string>my_specialise_title</string>
<string>my_retry_test_pattern</string>
</list> </list>
</value> </value>
</item> </item>
......
import re
test_result = sci['object'] test_result = sci['object']
kw = sci['kwargs'] kw = sci['kwargs']
test_result.setStopDate(kw.get('date') or DateTime()) test_result.setStopDate(kw.get('date') or DateTime())
...@@ -8,13 +9,38 @@ def unexpected(test_result): ...@@ -8,13 +9,38 @@ def unexpected(test_result):
# passed if there's no unexpected failures. # passed if there's no unexpected failures.
return test_result.getSourceProjectTitle() != "NEO R&D" return test_result.getSourceProjectTitle() != "NEO R&D"
def shouldRetry(test_result_line):
# type: (erp5.portal_type.TestResultLine,) -> bool
"""Should the test result line be retried ?
We retry test result line once for tests matching pattern defined on test suite.
Unless if there's already another failed test result line, in that case we don't retry.
"""
if test_result_line.getProperty('test_result_retry_count') or 0:
return False
test_result = test_result_line.getParentValue()
for other_test_result_line in test_result.contentValues(portal_type='Test Result Line'):
if test_result_line != other_test_result_line and other_test_result_line.getStringIndex() in ('UNKNOWN', 'FAILED'):
return False
test_suite_data = test_result.TestResult_getTestSuiteData()
if not test_suite_data:
return False
if not test_suite_data['retry_test_pattern']:
return False
return re.search(test_suite_data['retry_test_pattern'], test_result_line.getTitle() or '')
if test_result.getPortalType() == 'Test Result': if test_result.getPortalType() == 'Test Result':
has_unknown_result = False has_unknown_result = False
edit_kw = dict(duration=0, edit_kw = dict(duration=0,
all_tests=0, all_tests=0,
errors=0, errors=0,
failures=0, failures=0,
skips=0) skips=0,
test_result_retry_count=0)
for line in test_result.objectValues(portal_type='Test Result Line'): for line in test_result.objectValues(portal_type='Test Result Line'):
for prop in edit_kw: for prop in edit_kw:
try: try:
...@@ -47,12 +73,12 @@ elif test_result.getPortalType() == 'Test Result Line': ...@@ -47,12 +73,12 @@ elif test_result.getPortalType() == 'Test Result Line':
duration = kw.get('duration') duration = kw.get('duration')
if duration is None: if duration is None:
duration = (test_result.getStopDate() - test_result.getStartDate()) * (24*60*60) duration = (test_result.getStopDate() - test_result.getStartDate()) * (24*60*60)
cmdline = kw.get('command', getattr(test_result, 'cmdline', '')) cmdline = kw.get('command', '')
if same_type(cmdline, []): if same_type(cmdline, []):
cmdline = ' '.join(map(repr, cmdline)) cmdline = ' '.join(map(repr, cmdline))
stdout = kw.get('stdout', getattr(test_result, 'stdout', '')) stdout = kw.get('stdout', '')
stderr = kw.get('stderr', getattr(test_result, 'stderr', '')) stderr = kw.get('stderr', '')
html_test_result = kw.get('html_test_result', getattr(test_result, 'html_test_result', '')) html_test_result = kw.get('html_test_result', '')
test_result.edit(cmdline=cmdline, test_result.edit(cmdline=cmdline,
stdout=stdout, stdout=stdout,
stderr=stderr, stderr=stderr,
...@@ -63,5 +89,11 @@ elif test_result.getPortalType() == 'Test Result Line': ...@@ -63,5 +89,11 @@ elif test_result.getPortalType() == 'Test Result Line':
failures=failures, failures=failures,
skips=skips, skips=skips,
html_test_result=html_test_result) html_test_result=html_test_result)
if status == 'FAILED' and shouldRetry(test_result):
test_result.edit(
test_result_retry_count=1 + (test_result.getProperty('test_result_retry_count') or 0),
string_index='RETRYING',
)
test_result.redraft(comment="Retried after a first failure")
else: else:
raise NotImplementedError("unknown type : %r" % test_result.getPortalType()) raise NotImplementedError("unknown type : %r" % test_result.getPortalType())
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
<string>cancel_action</string> <string>cancel_action</string>
<string>fail</string> <string>fail</string>
<string>publish_stopped</string> <string>publish_stopped</string>
<string>redraft</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment