Commit b0cf277d authored by Kirill Smelkov's avatar Kirill Smelkov

Cancel test run on SIGINT/SIGTERM

In addition to canceling test run is master tells us to do so, also cancel the
run if interrupted or terminated.

With the following sample .nxdtest

    TestCase('sleep', ['sleep', '10'])

before the patch it does not react to CTRL+C:

    $ nxdtest
    ...
    >>> sleep
    $ sleep 10
    ^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C^C# ran 0 test cases.   <-- no reaction to CTRL+C, finishes after 10 seconds
    Traceback (most recent call last):
      File "/home/kirr/src/tools/go/py3.venv2/bin/nxdtest", line 33, in <module>
        sys.exit(load_entry_point('nxdtest', 'console_scripts', 'nxdtest')())
      File "/home/kirr/src/tools/go/py3.venv2/lib/python3.9/site-packages/decorator.py", line 232, in fun
        return caller(func, *(extras + args), **kw)
      File "/home/kirr/src/tools/go/pygolang-master/golang/__init__.py", line 103, in _
        return f(*argv, **kw)
      File "/home/kirr/src/wendelin/nxdtest/nxdtest/__init__.py", line 339, in main
        wg.wait()
    KeyboardInterrupt

after the patch:

    $ nxdtest
    ...
    >>> sleep
    $ sleep 10
    ^C# Interrupt					<-- prompt reaction to CTRL+C
    # stopping due to cancel
    # leaked pid=188877 'sleep' ['sleep', '10']
    error   sleep   1.030s  # 1t 1e 0f 0s
    # test run canceled
    # ran 1 test case:  1·error

Needs pygolang!17 to work.

/reviewed-by @jerome
/reviewed-on !16
parent 6f75fa90
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors. # Copyright (C) 2018-2022 Nexedi SA and Contributors.
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -62,8 +62,9 @@ import os, sys, argparse, logging, traceback, re, pwd, socket ...@@ -62,8 +62,9 @@ import os, sys, argparse, logging, traceback, re, pwd, socket
from errno import ESRCH, EPERM from errno import ESRCH, EPERM
from os.path import dirname from os.path import dirname
import six import six
from golang import b, defer, func, select, default from golang import b, chan, defer, func, go, select, default
from golang import errors, context, sync, time from golang import errors, context, os as gos, sync, syscall, time
from golang.os import signal
import psutil import psutil
# trun.py is a helper via which we run tests. # trun.py is a helper via which we run tests.
...@@ -219,12 +220,30 @@ def main(): ...@@ -219,12 +220,30 @@ def main():
bstdout = sys.stdout.buffer bstdout = sys.stdout.buffer
bstderr = sys.stderr.buffer bstderr = sys.stderr.buffer
# setup context that is canceled when/if test_result is canceled on master
# setup main context that is canceled on SIGINT/SIGTERM
# we will use this context as the base for all spawned jobs # we will use this context as the base for all spawned jobs
ctx, cancel = context.with_cancel(context.background()) ctx, cancel = context.with_cancel(context.background())
sigq = chan(1, dtype=gos.Signal)
signal.Notify(sigq, syscall.SIGINT, syscall.SIGTERM)
def _():
signal.Stop(sigq)
sigq.close()
defer(_)
def _(cancel):
sig, ok = sigq.recv_()
if not ok:
return
emit("# %s" % sig)
cancel()
go(_, cancel)
defer(cancel)
# adjust ctx to be also canceled when/if test_result is canceled on master
ctx, cancel = context.with_cancel(ctx)
cancelWG = sync.WorkGroup(ctx) cancelWG = sync.WorkGroup(ctx)
@func @func
def _(ctx): def _(ctx, cancel):
defer(cancel) defer(cancel)
while 1: while 1:
_, _rx = select( _, _rx = select(
...@@ -237,7 +256,7 @@ def main(): ...@@ -237,7 +256,7 @@ def main():
if not test_result.isAlive(): if not test_result.isAlive():
emit("# master asks to cancel test run") emit("# master asks to cancel test run")
break break
cancelWG.go(_) cancelWG.go(_, cancel)
defer(cancelWG.wait) defer(cancelWG.wait)
defer(cancel) defer(cancel)
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2020-2021 Nexedi SA and Contributors. # Copyright (C) 2020-2022 Nexedi SA and Contributors.
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -24,18 +24,18 @@ import os ...@@ -24,18 +24,18 @@ import os
import pwd import pwd
import sys import sys
import re import re
import time
import tempfile import tempfile
import signal
import shutil import shutil
import subprocess from subprocess import Popen, PIPE
from os.path import dirname, exists, devnull from os.path import dirname, exists
from golang import chan, select, default, func, defer from golang import chan, select, default, func, defer, b
from golang import context, sync, time from golang import context, sync, time
import psutil
import pytest import pytest
from nxdtest import main, trun import nxdtest
from nxdtest import trun
@pytest.fixture @pytest.fixture
...@@ -64,7 +64,7 @@ def run_nxdtest(tmpdir): ...@@ -64,7 +64,7 @@ def run_nxdtest(tmpdir):
@func @func
def _(ctx): def _(ctx):
defer(done.close) defer(done.close)
main() nxdtest.main()
wg.go(_) wg.go(_)
while 1: while 1:
...@@ -338,7 +338,7 @@ def distributor_with_cancelled_test(mocker): ...@@ -338,7 +338,7 @@ def distributor_with_cancelled_test(mocker):
# verify that nxdtest cancels test run when master reports that test_result is no longer alive. # verify that nxdtest cancels test run when master reports that test_result is no longer alive.
@pytest.mark.timeout(timeout=10) @pytest.mark.timeout(timeout=10)
def test_cancel_from_master(run_nxdtest, capsys, tmp_path, distributor_with_cancelled_test, mocker): def test_cancel_from_master(run_nxdtest, capsys, distributor_with_cancelled_test, mocker):
# nxdtest polls every 5 minutes, but in test we don't want to wait so long. # nxdtest polls every 5 minutes, but in test we don't want to wait so long.
# set master poll interval to small, but enough time for spawned hang to # set master poll interval to small, but enough time for spawned hang to
# setup its signal handler. # setup its signal handler.
...@@ -360,3 +360,57 @@ TestCase('TEST1', ['%s']) ...@@ -360,3 +360,57 @@ TestCase('TEST1', ['%s'])
assert "# test run canceled" in captured.out assert "# test run canceled" in captured.out
assert "hang: terminating" in captured.out assert "hang: terminating" in captured.out
assert captured.err == '' assert captured.err == ''
# verify that nxdtest cancels test run on SIGINT/SIGTERM.
#@pytest.mark.timeout(timeout=10)
@pytest.mark.timeout(timeout=3)
@pytest.mark.parametrize('sig', [(signal.SIGINT, "Interrupt"), (signal.SIGTERM, "Terminate")])
@func
def test_cancel_from_signal(tmpdir, sig):
hang = "%s/testprog/hang" % (dirname(__file__),)
with tmpdir.as_cwd():
with open(".nxdtest", "w") as f:
f.write("""\
TestCase('TEST1', ['%s'])
""" % hang)
proc = Popen([sys.executable, "%s/__init__.py" % dirname(nxdtest.__file__)], stdout=PIPE)
def _():
proc.terminate()
if proc.poll() is None:
time.sleep(1)
proc.kill()
proc.wait()
defer(_)
# procreadline reads next line from proc stdout.
outv = []
def procreadline():
l = proc.stdout.readline()
if len(l) != 0: # EOF
outv.append(l)
return l
# wait for hang to start and setup its signal handler
while 1:
l = procreadline()
if not l:
raise AssertionError("did not got 'hanging'")
if b"hanging" in l:
break
# send SIGINT/SIGTERM to proc and wait for it to complete
signo, sigmsg = sig
proc.send_signal(signo)
while 1:
if not procreadline():
break
out = b''.join(outv)
assert b"TEST1" in out
assert b("# %s" % sigmsg) in out
assert b"# test run canceled" in out
assert b"hang: terminating" in out
...@@ -13,7 +13,7 @@ setup( ...@@ -13,7 +13,7 @@ setup(
keywords = 'Nexedi testing infrastructure tool tox', keywords = 'Nexedi testing infrastructure tool tox',
packages = find_packages(), packages = find_packages(),
install_requires = ['erp5.util', 'six', 'pygolang', 'psutil', 'python-prctl'], install_requires = ['erp5.util', 'six', 'pygolang >= 0.1', 'psutil', 'python-prctl'],
extras_require = { extras_require = {
'test': ['pytest', 'pytest-mock', 'pytest-timeout', 'setproctitle'], 'test': ['pytest', 'pytest-mock', 'pytest-timeout', 'setproctitle'],
}, },
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment