Commit 53689cf4 authored by Jérome Perrin's avatar Jérome Perrin

ERP5TypeFunctionalTestCase improvements and fixes

 * expose firefox console and error logs in `geckodriver.log`
 * expose `geckodriver.log` in the folder testnodes expose on http
 * fix some cases where test were detected as "did not run" and cases where "did not run" tests were not reported as failures

See merge request nexedi/erp5!1444
parents 6f3cb199 1b1dbf60
...@@ -39,6 +39,7 @@ import logging ...@@ -39,6 +39,7 @@ import logging
from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.HTTPResponse import HTTPResponse
from zExceptions.ExceptionFormatter import format_exception from zExceptions.ExceptionFormatter import format_exception
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.runUnitTest import log_directory
from Products.ERP5Type.Utils import stopProcess, PR_SET_PDEATHSIG from Products.ERP5Type.Utils import stopProcess, PR_SET_PDEATHSIG
from lxml import etree from lxml import etree
from lxml.html import builder as E from lxml.html import builder as E
...@@ -207,11 +208,20 @@ class FunctionalTestRunner: ...@@ -207,11 +208,20 @@ class FunctionalTestRunner:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1338144 # https://bugzilla.mozilla.org/show_bug.cgi?id=1338144
options = webdriver.FirefoxOptions() options = webdriver.FirefoxOptions()
options.set_preference('dom.serviceWorkers.enabled', True) options.set_preference('dom.serviceWorkers.enabled', True)
# output javascript console and errors on stdout to help diagnosing failures
options.set_preference('devtools.console.stdout.content', True)
kw = dict(capabilities=capabilities, options=options) kw = dict(capabilities=capabilities, options=options)
firefox_bin = os.environ.get('firefox_bin') firefox_bin = os.environ.get('firefox_bin')
if firefox_bin: if firefox_bin:
geckodriver = os.path.join(os.path.dirname(firefox_bin), 'geckodriver') geckodriver = os.path.join(os.path.dirname(firefox_bin), 'geckodriver')
kw.update(firefox_binary=firefox_bin, executable_path=geckodriver) kw.update(
firefox_binary=firefox_bin,
executable_path=geckodriver,
# BBB in selenium 3.8.0 this option was named log_path
log_path=os.path.join(log_directory, 'geckodriver.log'),
# service_log_path=os.path.join(log_directory, 'geckodriver.log'),
)
browser = webdriver.Firefox(**kw) browser = webdriver.Firefox(**kw)
start_time = time.time() start_time = time.time()
logger.info("Running with browser: %s", browser) logger.info("Running with browser: %s", browser)
...@@ -248,13 +258,25 @@ class FunctionalTestRunner: ...@@ -248,13 +258,25 @@ class FunctionalTestRunner:
))) )))
# XXX No idea how to wait for the iframe content to be loaded # XXX No idea how to wait for the iframe content to be loaded
time.sleep(5) time.sleep(5)
# Count number of test to be executed # Count number of tests to be executed
test_count = browser.execute_script( test_count = browser.execute_script(
"return document.getElementById('testSuiteFrame').contentDocument.querySelector('tbody').children.length" "return document.getElementById('testSuiteFrame').contentDocument.querySelector('tbody').children.length"
) - 1 ) - 1
# Wait for tests to end
WebDriverWait(browser, self.timeout).until(EC.presence_of_element_located(( WebDriverWait(browser, self.timeout).until(EC.presence_of_element_located((
By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count
))) )))
# At the end of each test, updateSuiteWithResultOfPreviousTest updates
# testSuiteFrame iframe with hidden div containing the test results table.
# We will inspect these tables to know which tests have failed. First we
# need to wait a bit more, because at the end of test ( testComplete ),
# updateSuiteWithResultOfPreviousTest is called by setTimeout. We want to
# wait for the last test (which is the last td) result table to be present
browser.switch_to_frame('testSuiteFrame')
WebDriverWait(browser, 10).until(EC.presence_of_element_located((
By.XPATH, '//table/tbody/tr/td[last()]//table'
)))
browser.switch_to_default_content()
self.execution_duration = round(time.time() - start_time, 2) self.execution_duration = round(time.time() - start_time, 2)
html_parser = etree.HTMLParser(recover=True) html_parser = etree.HTMLParser(recover=True)
iframe = etree.fromstring( iframe = etree.fromstring(
......
...@@ -133,20 +133,21 @@ class ERP5(_ERP5): ...@@ -133,20 +133,21 @@ class ERP5(_ERP5):
def _updateFunctionalTestResponse(self, status_dict): def _updateFunctionalTestResponse(self, status_dict):
""" Convert the Unit Test output into more accurate information """ Convert the Unit Test output into more accurate information
related to funcional test run. related to functional test run.
""" """
# Parse relevant information to update response information # Parse relevant information to update response information
try: try:
summary, html_test_result = status_dict['stderr'].split("-"*79)[1:3] summary, html_test_result = status_dict['stderr'].split("-"*79)[1:3]
except ValueError: except ValueError:
# In case of error when parse the file, preserve the original # In case of error when parse the file, preserve the original
# informations. This prevents we have unfinished tests. # information. This prevents we have unfinished tests.
return status_dict return status_dict
status_dict['html_test_result'] = html_test_result status_dict['html_test_result'] = html_test_result
search = self.FTEST_PASS_FAIL_RE.search(summary) search = self.FTEST_PASS_FAIL_RE.search(summary)
if search: if search:
group_dict = search.groupdict() group_dict = search.groupdict()
status_dict['failure_count'] = int(group_dict['failures']) status_dict['failure_count'] = int(group_dict['failures']) \
+ int(status_dict.get('failure_count', 0))
status_dict['test_count'] = int(group_dict['total']) status_dict['test_count'] = int(group_dict['total'])
status_dict['skip_count'] = int(group_dict['expected_failure']) status_dict['skip_count'] = int(group_dict['expected_failure'])
return status_dict return status_dict
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment