Commit 40e2c4ab authored by Jérome Perrin's avatar Jérome Perrin

Unittest and Python3 support

These are the necessary changes to run `SlapOS.Eggs.UnitTest-*` and `SlapOS.SoftwareReleases.IntegrationTest-*` using nxdtest

See merge request nexedi/nxdtest!6

/reviewed-by @kirr
parents d829e9ca a129b560
...@@ -60,12 +60,14 @@ from subprocess import Popen, PIPE ...@@ -60,12 +60,14 @@ from subprocess import Popen, PIPE
from time import time, strftime, gmtime, localtime from time import time, strftime, gmtime, localtime
import os, sys, threading, argparse, logging, traceback, re, pwd, socket import os, sys, threading, argparse, logging, traceback, re, pwd, socket
import six import six
from golang import b
# loadNXDTestFile loads .nxdtest file located @path. # loadNXDTestFile loads .nxdtest file located @path.
def loadNXDTestFile(path): # -> TestEnv def loadNXDTestFile(path): # -> TestEnv
t = TestEnv() t = TestEnv()
g = {'TestCase': t.TestCase, # TODO + all other public TestEnv methods g = {'TestCase': t.TestCase, # TODO + all other public TestEnv methods
'PyTest': PyTest} 'PyTest': PyTest,
'UnitTest': UnitTest,}
with open(path, "r") as f: with open(path, "r") as f:
src = f.read() src = f.read()
six.exec_(src, g) six.exec_(src, g)
...@@ -99,6 +101,12 @@ class TestEnv: ...@@ -99,6 +101,12 @@ class TestEnv:
self.testv.append(t) self.testv.append(t)
self.byname[name] = t self.byname[name] = t
def emit(*message):
"""Emit a message on stdout and flush output.
"""
print(*message)
sys.stdout.flush()
def main(): def main():
# testnode executes us giving URL to master results collecting instance and other details # testnode executes us giving URL to master results collecting instance and other details
# https://lab.nexedi.com/nexedi/erp5/blob/744f3fde/erp5/util/testnode/UnitTestRunner.py#L137 # https://lab.nexedi.com/nexedi/erp5/blob/744f3fde/erp5/util/testnode/UnitTestRunner.py#L137
...@@ -134,7 +142,7 @@ def main(): ...@@ -134,7 +142,7 @@ def main():
# --list # --list
if args.list: if args.list:
for t in tenv.testv: for t in tenv.testv:
print(t.name) emit(t.name)
return return
# master_url provided -> run tests under master control # master_url provided -> run tests under master control
...@@ -164,6 +172,13 @@ def main(): ...@@ -164,6 +172,13 @@ def main():
# log information about local node # log information about local node
system_info() system_info()
if sys.version_info < (3,):
bstdout = sys.stdout
bstderr = sys.stderr
else:
bstdout = sys.stdout.buffer
bstderr = sys.stderr.buffer
# run the tests # run the tests
devnull = open(os.devnull) devnull = open(os.devnull)
while 1: while 1:
...@@ -176,8 +191,8 @@ def main(): ...@@ -176,8 +191,8 @@ def main():
t = tenv.byname[test_result_line.name] t = tenv.byname[test_result_line.name]
tstart = time() tstart = time()
print('\n>>> %s' % t.name) emit('\n>>> %s' % t.name)
print('$ %s' % t.command_str()) emit('$ %s' % t.command_str())
# default status dict # default status dict
status = { status = {
...@@ -193,29 +208,28 @@ def main(): ...@@ -193,29 +208,28 @@ def main():
# In addition to kw['env'], kw['envadj'] allows users to define # In addition to kw['env'], kw['envadj'] allows users to define
# only adjustments instead of providing full env dict. # only adjustments instead of providing full env dict.
# Test command is spawned with unchanged cwd. Instance wrapper cares to set cwd before running us. # Test command is spawned with unchanged cwd. Instance wrapper cares to set cwd before running us.
# bufsize=1 means 'line buffered'
kw = t.kw.copy() kw = t.kw.copy()
env = kw.pop('env', os.environ) env = kw.pop('env', os.environ)
env = env.copy() env = env.copy()
envadj = kw.pop('envadj', {}) envadj = kw.pop('envadj', {})
env.update(envadj) env.update(envadj)
p = Popen(t.argv, env=env, stdin=devnull, stdout=PIPE, stderr=PIPE, bufsize=1, **kw) p = Popen(t.argv, env=env, stdin=devnull, stdout=PIPE, stderr=PIPE, bufsize=0, **kw)
except: except:
stdout, stderr = '', traceback.format_exc() stdout, stderr = b'', b(traceback.format_exc())
sys.stderr.write(stderr) bstderr.write(stderr)
status['error_count'] += 1 status['error_count'] += 1
else: else:
# tee >stdout,stderr so we can also see in testnode logs # tee >stdout,stderr so we can also see in testnode logs
# (explicit teeing instead of p.communicate() to be able to see incremental progress) # (explicit teeing instead of p.communicate() to be able to see incremental progress)
buf_out = [] buf_out = []
buf_err = [] buf_err = []
tout = threading.Thread(target=tee, args=(p.stdout, sys.stdout, buf_out)) tout = threading.Thread(target=tee, args=(p.stdout, bstdout, buf_out))
terr = threading.Thread(target=tee, args=(p.stderr, sys.stderr, buf_err)) terr = threading.Thread(target=tee, args=(p.stderr, bstderr, buf_err))
tout.start() tout.start()
terr.start() terr.start()
tout.join(); stdout = ''.join(buf_out) tout.join(); stdout = b''.join(buf_out)
terr.join(); stderr = ''.join(buf_err) terr.join(); stderr = b''.join(buf_err)
p.wait() p.wait()
if p.returncode != 0: if p.returncode != 0:
...@@ -224,10 +238,10 @@ def main(): ...@@ -224,10 +238,10 @@ def main():
# postprocess output, if we can # postprocess output, if we can
if t.summaryf is not None: if t.summaryf is not None:
try: try:
summary = t.summaryf(stdout) summary = t.summaryf(stdout, stderr)
except: except:
bad = traceback.format_exc() bad = b(traceback.format_exc())
sys.stderr.write(bad) bstderr.write(bad)
stderr += bad stderr += bad
status['error_count'] += 1 status['error_count'] += 1
...@@ -248,7 +262,7 @@ def main(): ...@@ -248,7 +262,7 @@ def main():
} }
tres.update(status) tres.update(status)
print(_test_result_summary(t.name, tres)) emit(_test_result_summary(t.name, tres))
test_result_line.stop(**tres) test_result_line.stop(**tres)
# tee, similar to tee(1) utility, copies data from fin to fout appending them to buf. # tee, similar to tee(1) utility, copies data from fin to fout appending them to buf.
...@@ -296,11 +310,11 @@ def _test_result_summary(name, kw): ...@@ -296,11 +310,11 @@ def _test_result_summary(name, kw):
# system_info prints information about local computer. # system_info prints information about local computer.
def system_info(): def system_info():
print('date:\t%s' % (strftime("%a, %d %b %Y %H:%M:%S %Z", localtime()))) emit('date:\t%s' % (strftime("%a, %d %b %Y %H:%M:%S %Z", localtime())))
whoami = pwd.getpwuid(os.getuid()).pw_name whoami = pwd.getpwuid(os.getuid()).pw_name
print('xnode:\t%s@%s' % (whoami, socket.getfqdn())) emit('xnode:\t%s@%s' % (whoami, socket.getfqdn()))
print('uname:\t%s' % ' '.join(os.uname())) emit('uname:\t%s' % ' '.join(os.uname()))
print('cpu:\t%s' % get1('/proc/cpuinfo', 'model name')) emit('cpu:\t%s' % get1('/proc/cpuinfo', 'model name'))
# get1 returns first entry from file @path prefixed with ^<field>\s*: # get1 returns first entry from file @path prefixed with ^<field>\s*:
def get1(path, field, default=None): def get1(path, field, default=None):
...@@ -350,7 +364,7 @@ class LocalTestResultLine: ...@@ -350,7 +364,7 @@ class LocalTestResultLine:
# support for well-known summary functions # support for well-known summary functions
class PyTest: class PyTest:
@staticmethod @staticmethod
def summary(out): # -> status_dict def summary(out, err): # -> status_dict
# end of output is like # end of output is like
# ================ 1 failed, 1 passed, 12 skipped in 0.39 seconds ================ # ================ 1 failed, 1 passed, 12 skipped in 0.39 seconds ================
# ... # ...
...@@ -363,7 +377,7 @@ class PyTest: ...@@ -363,7 +377,7 @@ class PyTest:
return {} return {}
def get(name, default=None): def get(name, default=None):
m = re.search(r'\b([0-9]+) '+name+r'\b', pytail) m = re.search(br'\b([0-9]+) ' + name.encode() + br'\b', pytail)
if m is None: if m is None:
return default return default
return int(m.group(1)) return int(m.group(1))
...@@ -384,5 +398,46 @@ class PyTest: ...@@ -384,5 +398,46 @@ class PyTest:
return stat return stat
class UnitTest:
@staticmethod
def summary(out, err): # -> status_dict
run_re = re.compile(
br'.*Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
re.DOTALL)
status_re = re.compile(br"""
.*(OK|FAILED)\s+\(
(failures=(?P<failures>\d+),?\s*)?
(errors=(?P<errors>\d+),?\s*)?
(skipped=(?P<skips>\d+),?\s*)?
(expected\s+failures=(?P<expected_failures>\d+),?\s*)?
(unexpected\s+successes=(?P<unexpected_successes>\d+),?\s*)?
\)
""", re.DOTALL | re.VERBOSE)
status_dict = {
}
run = run_re.search(err)
if run:
groupdict = run.groupdict()
status_dict.update(
duration=float(groupdict['seconds']),
test_count=int(groupdict['all_tests']),
error_count=0,
failure_count=0,
skip_count=0,
)
status = status_re.search(err)
if status:
groupdict = status.groupdict()
status_dict.update(
error_count=int(groupdict.get('errors') or 0),
failure_count=int(groupdict.get('failures') or 0)
+ int(groupdict.get('unexpected_successes') or 0),
skip_count=int(groupdict.get('skips') or 0)
+ int(groupdict.get('expected_failures') or 0))
return status_dict
if __name__ == '__main__': if __name__ == '__main__':
main() main()
...@@ -21,12 +21,13 @@ ...@@ -21,12 +21,13 @@
from nxdtest import _test_result_summary, PyTest from nxdtest import _test_result_summary, PyTest
import pytest import pytest
from golang import b
# [] of (name, textout, summaryok) # [] of (name, out, err, summaryok)
testv = [] testv = []
def case1(name, textout, summaryok): testv.append((name, textout, summaryok)) def case1(name, out, err, summaryok): testv.append((name, out, err, summaryok))
case1('ok+xfail', """\ case1('ok+xfail', b("""\
============================= test session starts ============================== ============================= test session starts ==============================
platform linux2 -- Python 2.7.18, pytest-4.6.11, py-1.9.0, pluggy-0.13.1 platform linux2 -- Python 2.7.18, pytest-4.6.11, py-1.9.0, pluggy-0.13.1
rootdir: /srv/slapgrid/slappart9/srv/testnode/dfq/soft/46d349541123ed5fc6ceea58fd013a51/parts/zodbtools-dev rootdir: /srv/slapgrid/slappart9/srv/testnode/dfq/soft/46d349541123ed5fc6ceea58fd013a51/parts/zodbtools-dev
...@@ -39,10 +40,11 @@ zodbtools/test/test_tidrange.py ............................. [ 81%] ...@@ -39,10 +40,11 @@ zodbtools/test/test_tidrange.py ............................. [ 81%]
zodbtools/test/test_zodb.py ........ [100%] zodbtools/test/test_zodb.py ........ [100%]
=============== 41 passed, 2 xfailed, 1 warnings in 4.62 seconds =============== =============== 41 passed, 2 xfailed, 1 warnings in 4.62 seconds ===============
""", """),
b(''),
'?\ttestname\t1.000s\t# 43t ?e ?f ?s') '?\ttestname\t1.000s\t# 43t ?e ?f ?s')
case1('ok+fail', """\ case1('ok+fail', b("""\
============================= test session starts ============================== ============================= test session starts ==============================
platform linux2 -- Python 2.7.18, pytest-4.6.11, py-1.9.0, pluggy-0.13.1 platform linux2 -- Python 2.7.18, pytest-4.6.11, py-1.9.0, pluggy-0.13.1
rootdir: /srv/slapgrid/slappart16/srv/testnode/dfj/soft/8b9988ce0aa31334c6bd56b40e4bba65/parts/pygolang-dev rootdir: /srv/slapgrid/slappart16/srv/testnode/dfj/soft/8b9988ce0aa31334c6bd56b40e4bba65/parts/pygolang-dev
...@@ -111,10 +113,11 @@ E Use -v to get the full diff ...@@ -111,10 +113,11 @@ E Use -v to get the full diff
golang/time_test.py:106: AssertionError golang/time_test.py:106: AssertionError
=============== 1 failed, 98 passed, 13 skipped in 26.85 seconds =============== =============== 1 failed, 98 passed, 13 skipped in 26.85 seconds ===============
""", """),
b(''),
'?\ttestname\t1.000s\t# 112t ?e 1f 13s') '?\ttestname\t1.000s\t# 112t ?e 1f 13s')
case1('ok+tailtext', """\ case1('ok+tailtext', b("""\
date: Sun, 08 Nov 2020 12:26:24 MSK date: Sun, 08 Nov 2020 12:26:24 MSK
xnode: kirr@deco.navytux.spb.ru xnode: kirr@deco.navytux.spb.ru
uname: Linux deco 5.9.0-1-amd64 #1 SMP Debian 5.9.1-1 (2020-10-17) x86_64 uname: Linux deco 5.9.0-1-amd64 #1 SMP Debian 5.9.1-1 (2020-10-17) x86_64
...@@ -129,13 +132,14 @@ wcfs: 2020/11/08 12:26:38 /tmp/testdb_fs.Z9IvT0/1.fs: watcher: stat /tmp/testdb_ ...@@ -129,13 +132,14 @@ wcfs: 2020/11/08 12:26:38 /tmp/testdb_fs.Z9IvT0/1.fs: watcher: stat /tmp/testdb_
# unmount/stop wcfs pid39653 @ /tmp/wcfs/40cc7154ed758d6a867205e79e320c1d3b56458d # unmount/stop wcfs pid39653 @ /tmp/wcfs/40cc7154ed758d6a867205e79e320c1d3b56458d
wcfs: 2020/11/08 12:26:38 /tmp/testdb_fs.B3rbby/1.fs: watcher: stat /tmp/testdb_fs.B3rbby/1.fs: use of closed file wcfs: 2020/11/08 12:26:38 /tmp/testdb_fs.B3rbby/1.fs: watcher: stat /tmp/testdb_fs.B3rbby/1.fs: use of closed file
# unmount/stop wcfs pid39595 @ /tmp/wcfs/d0b5d036a2cce47fe73003cf2d9f0b22c7043817 # unmount/stop wcfs pid39595 @ /tmp/wcfs/d0b5d036a2cce47fe73003cf2d9f0b22c7043817
""", """),
b(''),
'?\ttestname\t1.000s\t# 55t ?e ?f ?s') '?\ttestname\t1.000s\t# 55t ?e ?f ?s')
@pytest.mark.parametrize("name,textout,summaryok", testv) @pytest.mark.parametrize("name,out,err,summaryok", testv)
def test_pytest_summary(name,textout, summaryok): def test_pytest_summary(name, out, err, summaryok):
kw = {'duration': 1.0} kw = {'duration': 1.0}
kw.update(PyTest.summary(textout)) kw.update(PyTest.summary(out, err))
summary = _test_result_summary('testname', kw) summary = _test_result_summary('testname', kw)
assert summary == summaryok assert summary == summaryok
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
# verify general functionality
import sys
import re
import pytest
from nxdtest import main
@pytest.fixture
def run_nxdtest(tmpdir):
"""Fixture which returns a function which invokes nxdtest in a temporary
directory, with the provided .nxdtest file content and with arguments
passed as `argv`.
"""
def _run_nxdtest(nxdtest_file_content, argv=("nxdtest",)):
with tmpdir.as_cwd():
with open(".nxdtest", "w") as f:
f.write(nxdtest_file_content)
sys_argv = sys.argv
sys.argv = argv
try:
main()
finally:
sys.argv = sys_argv
return _run_nxdtest
def test_main(run_nxdtest, capsys):
run_nxdtest(
"""\
TestCase('TESTNAME', ['echo', 'TEST OUPUT'])
"""
)
captured = capsys.readouterr()
output_lines = captured.out.splitlines()
assert ">>> TESTNAME" in output_lines
assert "$ echo TEST OUPUT" in output_lines
assert "TEST OUPUT" in output_lines
assert re.match("ok\tTESTNAME\t.*s\t# 1t 0e 0f 0s", output_lines[-1])
def test_error_invoking_command(run_nxdtest, capsys):
run_nxdtest(
"""\
TestCase('TESTNAME', ['not exist command'])
"""
)
captured = capsys.readouterr()
assert "No such file or directory" in captured.err
def test_error_invoking_summary(run_nxdtest, capsys):
run_nxdtest(
"""\
TestCase('TESTNAME', ['echo'], summaryf="error")
"""
)
captured = capsys.readouterr()
assert "TypeError" in captured.err
def test_run_argument(run_nxdtest, capsys):
run_nxdtest(
"""\
TestCase('TEST1', ['echo', 'TEST1'])
TestCase('TEST2', ['echo', 'TEST2'])
""",
argv=["nxdtest", "--run", "TEST1"],
)
captured = capsys.readouterr()
assert "TEST1" in captured.out
assert "TEST2" not in captured.out
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
# verify unittest-related functionality
from nxdtest import _test_result_summary, UnitTest
import pytest
from golang import b
# [] of (name, out, err, summaryok)
testv = []
def case1(name, out, err, summaryok): testv.append((name, out, err, summaryok))
case1('ok', b(''), b("""\
test_1 (test.Test) ... ok
test_2 (test.Test) ... ok
test_3 (test.Test) ... ok
----------------------------------------------------------------------
Ran 3 tests in 1.761s
OK
"""),
'ok\ttestname\t1.761s\t# 3t 0e 0f 0s')
case1('ok+xfail', b(''), b("""\
test_1 (test.Test) ... ok
test_2 (test.Test) ... ok
test_3 (test.Test) ... expected failure
----------------------------------------------------------------------
Ran 3 tests in 1.098s
OK (expected failures=1)
"""),
'ok\ttestname\t1.098s\t# 3t 0e 0f 1s')
case1('fail', b(''), b("""\
test_1 (test.Test) ... ok
test_2 (test.Test) ... ok
test_3 (test.Test) ... FAIL
======================================================================
FAIL: test_3 (test.Test)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/srv/slapgrid/slappart4/srv/project/nxdtest/tmp/test.py", line 14, in test_3
self.assertEqual(1, 2)
AssertionError: 1 != 2
----------------------------------------------------------------------
Ran 3 tests in 2.198s
FAILED (failures=1)
"""),
'fail\ttestname\t2.198s\t# 3t 0e 1f 0s')
case1('error', b(''), b("""\
test_1 (test.Test) ... ok
test_2 (test.Test) ... ok
test_3 (test.Test) ... ERROR
======================================================================
ERROR: test_3 (test.Test)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/srv/slapgrid/slappart4/srv/project/nxdtest/tmp/test.py", line 14, in test_3
boom
NameError: name 'boom' is not defined
----------------------------------------------------------------------
Ran 3 tests in 1.684s
FAILED (errors=1)
"""),
'error\ttestname\t1.684s\t# 3t 1e 0f 0s')
case1('error-no-test', b(''), b("""\
usage: python -m unittest discover [-h] [-v] [-q] [--locals] [-f] [-c] [-b]
[-k TESTNAMEPATTERNS] [-s START]
[-p PATTERN] [-t TOP]
python -m unittest discover: error: unrecognized arguments: --argument-error
"""),
'?\ttestname\t1.000s\t# ?t ?e ?f ?s')
case1('error-no-output', b(''), b(''), '?\ttestname\t1.000s\t# ?t ?e ?f ?s')
case1('failed+unexpected_success', b(''), b("""\
test_1 (test.Test) ... ok
test_2 (test.Test) ... ok
test_3 (test.Test) ... unexpected success
----------------------------------------------------------------------
Ran 3 tests in 1.039s
FAILED (unexpected successes=1)
"""),
'fail\ttestname\t1.039s\t# 3t 0e 1f 0s')
case1('mixed-output', b(''), b("""\
----------------------------------------------------------------------
Ran 1 tests in 1.111s
FAILED (failures=1)
----------------------------------------------------------------------
Ran 3 tests in 2.222s
FAILED (failures=3)
"""),
'fail\ttestname\t2.222s\t# 3t 0e 3f 0s')
@pytest.mark.parametrize("name,out,err,summaryok", testv)
def test_unittest_summary(name, out, err, summaryok):
kw = {'duration': 1.0}
kw.update(UnitTest.summary(out, err))
summary = _test_result_summary('testname', kw)
assert summary == summaryok
...@@ -13,7 +13,7 @@ setup( ...@@ -13,7 +13,7 @@ setup(
keywords = 'Nexedi testing infrastructure tool tox', keywords = 'Nexedi testing infrastructure tool tox',
packages = find_packages(), packages = find_packages(),
install_requires = ['erp5.util', 'six'], install_requires = ['erp5.util', 'six', 'pygolang'],
extras_require = { extras_require = {
'test': ['pytest'], 'test': ['pytest'],
}, },
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment