Commit cddfab91 authored by Jérome Perrin's avatar Jérome Perrin

test_result: Support evaluating test result status per line

This is to support SLAPOS-EGG-TEST which runs tests for multiple
repositories from the same test suite. Evaluating each commit from
each repository with the result of the test result is problematic because
every time one test fail, all repositories are marked as failed.

Introduce a new property on test result repository to optionally define
the pattern of test result line to consider for this repository. This way,
with our SLAPOS-EGG-TEST running tests for slapos, slapos.core,
... etc and producing test result line with the same title as the repository
name, we can use something like `slapos$` (the pattern is a regular expression)
to match only slapos,  `slapos\.core` to match only slapos.core etc.

This also allow to define more complex relation if necessary, for example kedifa
and caucase seems related, maybe we want to configure so that a failure on kedifa
test suite marks the commits from both kedifa and caucase as failed (or maybe
not, but at least this way should make it possible)

/reviewed-on nexedi/erp5!964
parent 427469aa
"""Annotate a commit with state of the test (running, canceled, success, failed) """Annotate a commit with state of the test (running, canceled, success, failed)
""" """
import re
portal = context.getPortalObject() portal = context.getPortalObject()
portal_url = portal.ERP5Site_getAbsoluteUrl() portal_url = portal.ERP5Site_getAbsoluteUrl()
...@@ -7,6 +8,27 @@ test_result_relative_url = context.getRelativeUrl() ...@@ -7,6 +8,27 @@ test_result_relative_url = context.getRelativeUrl()
test_result_relative_id = context.getId() test_result_relative_id = context.getId()
test_result_reference = context.getReference() test_result_reference = context.getReference()
def getTestResultStateForTestSuiteRepository(test_result_line_pattern):
"""Compute the state for this test repository.
This is used in test results like SLAPOS-EGG-TEST where we run in the
same test suite tests for multiple more-or-less independent repositories.
We don't want to mark commits from all repositories failed when only one
test as a problem.
On test suite repository, we define the pattern of test lines to consider,
this is used here to evaluate this repository only based on the result
of the individual test result lines.
"""
if test_result_line_pattern:
# state of test result lines matching the pattern.
return 'success' if {'PASSED'} == {
test_result_line.getStringIndex() for test_result_line
in context.contentValues(portal_type='Test Result Line')
if re.search(test_result_line_pattern, test_result_line.getTitle())
} else 'failed'
return state # global state of the test result.
test_suite_data = context.TestResult_getTestSuiteData() test_suite_data = context.TestResult_getTestSuiteData()
if test_suite_data: if test_suite_data:
for repository_info in test_suite_data['repository_dict'].values(): for repository_info in test_suite_data['repository_dict'].values():
...@@ -16,7 +38,8 @@ if test_suite_data: ...@@ -16,7 +38,8 @@ if test_suite_data:
connector.postCommitStatus( connector.postCommitStatus(
repository_info['repository_url'], repository_info['repository_url'],
repository_info['revision'], repository_info['revision'],
state, getTestResultStateForTestSuiteRepository(
repository_info['test_result_line_pattern']) if state != 'running' else state,
connector.getTestResultUrlTemplate().format( connector.getTestResultUrlTemplate().format(
portal_url=portal_url, portal_url=portal_url,
test_result_relative_url=test_result_relative_url, test_result_relative_url=test_result_relative_url,
......
...@@ -5,6 +5,9 @@ ...@@ -5,6 +5,9 @@
revision: commit sha revision: commit sha
repository_url: git url of the repository repository_url: git url of the repository
commits_count: number of commits commits_count: number of commits
connector_relative_url: URL of the gitlab connector
test_result_line_pattern: pattern of test result lines titles that needs
to be considered to assess the result of this test.
returns None if test suite cannot be found. returns None if test suite cannot be found.
""" """
...@@ -42,6 +45,8 @@ for test_result_repository in test_suite.contentValues(portal_type='Test Suite R ...@@ -42,6 +45,8 @@ for test_result_repository in test_suite.contentValues(portal_type='Test Suite R
repository_data = repository_dict.setdefault(buildout_section_id, {}) repository_data = repository_dict.setdefault(buildout_section_id, {})
repository_data['repository_url'] = test_result_repository.getGitUrl() repository_data['repository_url'] = test_result_repository.getGitUrl()
repository_data['connector_relative_url'] = test_result_repository.getDestination() repository_data['connector_relative_url'] = test_result_repository.getDestination()
repository_data['test_result_line_pattern'] = test_result_repository.getSourceReference()
if REQUEST: if REQUEST:
REQUEST.RESPONSE.setHeader('content-type', 'application/json; charset=utf-8') REQUEST.RESPONSE.setHeader('content-type', 'application/json; charset=utf-8')
......
...@@ -105,6 +105,7 @@ ...@@ -105,6 +105,7 @@
<value> <value>
<list> <list>
<string>my_destination_title</string> <string>my_destination_title</string>
<string>my_source_reference</string>
</list> </list>
</value> </value>
</item> </item>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>description</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_source_reference</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>description</string> </key>
<value> <string>Only annotate commits based on the state of test result lines matching this regular expression.\n
\n
Leave this blank so that the status of test result is considered for all repositories.</string> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_string_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Test Result Line Pattern</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -1557,12 +1557,14 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase): ...@@ -1557,12 +1557,14 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase):
"connector_relative_url": None, "connector_relative_url": None,
"repository_url": "https://lab.nexedi.com/nexedi/erp5.git", "repository_url": "https://lab.nexedi.com/nexedi/erp5.git",
"revision": "dc7b6e2e85e9434a97694a698884b057b7d30286", "revision": "dc7b6e2e85e9434a97694a698884b057b7d30286",
"test_result_line_pattern": None,
}, },
"test": { "test": {
"commits_count": 10, "commits_count": 10,
"connector_relative_url": self.connector.getRelativeUrl(), "connector_relative_url": self.connector.getRelativeUrl(),
"repository_url": "https://lab.example.com/nexedi/test.git", "repository_url": "https://lab.example.com/nexedi/test.git",
"revision": "cc4c79c003f7cfe0bfcbc7b302eac988110c96ae", "revision": "cc4c79c003f7cfe0bfcbc7b302eac988110c96ae",
"test_result_line_pattern": None,
}, },
}, },
"test_suite_relative_url": self.test_suite.getRelativeUrl() "test_suite_relative_url": self.test_suite.getRelativeUrl()
...@@ -1588,6 +1590,7 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase): ...@@ -1588,6 +1590,7 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase):
"connector_relative_url": None, "connector_relative_url": None,
"repository_url": "https://lab.nexedi.com/nexedi/erp5.git", "repository_url": "https://lab.nexedi.com/nexedi/erp5.git",
"revision": "dc7b6e2e85e9434a97694a698884b057b7d30286", "revision": "dc7b6e2e85e9434a97694a698884b057b7d30286",
"test_result_line_pattern": None,
}, },
test_suite_data['repository_dict']['foo-bar']) test_suite_data['repository_dict']['foo-bar'])
...@@ -1640,6 +1643,62 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase): ...@@ -1640,6 +1643,62 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase):
self.test_result.start() self.test_result.start()
self.tic() self.tic()
def test_status_per_test_result_line(self):
self.test_suite.test_repo.setSourceReference(
'test_result_line')
self.test_result.newContent(
portal_type='Test Result Line',
title='this test_result_line is OK',
all_tests=1,
string_index='PASSED'
)
self.test_result.newContent(
portal_type='Test Result Line',
title='another unrelated line',
all_tests=1,
string_index='FAILED'
)
self._start_test_result()
self.test_result.setStringIndex('FAILED')
# this test failed, but we are only interested in lines matching `test_result_line`
# because that line was successful, the test is OK.
with responses.RequestsMock() as rsps:
rsps.add_callback(
responses.POST,
self.post_commit_status_url,
self._response_callback('success'))
self.test_result.stop()
self.tic()
def test_status_per_test_result_line_multiple_matches(self):
self.test_suite.test_repo.setSourceReference(
'test_result_line')
self.test_result.newContent(
portal_type='Test Result Line',
title='this test_result_line is OK',
all_tests=1,
string_index='PASSED'
)
self.test_result.newContent(
portal_type='Test Result Line',
title='also test_result_line but failed',
all_tests=1,
string_index='FAILED'
)
self._start_test_result()
self.test_result.setStringIndex('PASSED')
# this test is marked as passed, but we are interested in all
# lines matching `test_result_line` and one of them is failed.
with responses.RequestsMock() as rsps:
rsps.add_callback(
responses.POST,
self.post_commit_status_url,
self._response_callback('failed'))
self.test_result.stop()
self.tic()
def test_commit_not_found(self): def test_commit_not_found(self):
with responses.RequestsMock() as rsps: with responses.RequestsMock() as rsps:
rsps.add( rsps.add(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment