Commit 9da63815 authored by Pere Cortes's avatar Pere Cortes

Security Thread added to kill blocked process

parent 6acf7f5f
...@@ -23,7 +23,8 @@ sys.path.extend([ ...@@ -23,7 +23,8 @@ sys.path.extend([
from erp5.util.testnode.testnode import TestNode from erp5.util.testnode.testnode import TestNode
from erp5.util.testnode.testnode import SlapOSInstance from erp5.util.testnode.testnode import SlapOSInstance
from erp5.util.testnode.ProcessManager import ProcessManager from erp5.util.testnode.ProcessManager import ProcessManager, SubprocessError
from erp5.util.testnode.SlapOSControler import SlapOSControler from erp5.util.testnode.SlapOSControler import SlapOSControler
from erp5.util.taskdistribution import TaskDistributor from erp5.util.taskdistribution import TaskDistributor
from erp5.util.taskdistribution import TaskDistributionTool from erp5.util.taskdistribution import TaskDistributionTool
...@@ -78,12 +79,12 @@ class ERP5TestNode(TestCase): ...@@ -78,12 +79,12 @@ class ERP5TestNode(TestCase):
config["test_node_title"] = "Foo-Test-Node" config["test_node_title"] = "Foo-Test-Node"
return TestNode(log, config) return TestNode(log, config)
def getTestSuiteData(self, add_third_repository=False): def getTestSuiteData(self, add_third_repository=False, reference="foo"):
data = [{ data = [{
"test_suite": "Foo", "test_suite": "Foo",
"project_title": "Foo", "project_title": reference,
"test_suite_title": "Foo-Test", "test_suite_title": "Foo-Test",
"test_suite_reference": "foo", "test_suite_reference": reference,
"vcs_repository_list": [ "vcs_repository_list": [
{'url': self.remote_repository0, {'url': self.remote_repository0,
'profile_path': 'software.cfg', 'profile_path': 'software.cfg',
...@@ -275,7 +276,7 @@ branch = foo ...@@ -275,7 +276,7 @@ branch = foo
self.assertEquals([commit_dict['rep0'][0][0],commit_dict['rep1'][1][0]], self.assertEquals([commit_dict['rep0'][0][0],commit_dict['rep1'][1][0]],
getRepInfo(hash=1)) getRepInfo(hash=1))
def test_07_checkOldTestSuite(self): def test_07_checkExistingTestSuite(self):
test_node = self.getTestNode() test_node = self.getTestNode()
test_suite_data = self.getTestSuiteData(add_third_repository=True) test_suite_data = self.getTestSuiteData(add_third_repository=True)
self.assertEquals([], os.listdir(self.working_directory)) self.assertEquals([], os.listdir(self.working_directory))
...@@ -313,68 +314,90 @@ branch = foo ...@@ -313,68 +314,90 @@ branch = foo
Check parameters passed to runTestSuite Check parameters passed to runTestSuite
Also make sure that --firefox_bin and --xvfb_bin are passed when needed Also make sure that --firefox_bin and --xvfb_bin are passed when needed
""" """
def _createPath(path_to_create, end_path): original_getSupportedParameter = ProcessManager.getSupportedParameterSet
os.makedirs(path_to_create) original_spawn = ProcessManager.spawn
return os.close(os.open(os.path.join(path_to_create, try:
end_path),os.O_CREAT)) def _createPath(path_to_create, end_path):
def get_parameters(self, *args, **kw): os.makedirs(path_to_create)
call_parameter_list.append({'args': [x for x in args], 'kw':kw}) return os.close(os.open(os.path.join(path_to_create,
def patch_getSupportedParameterSet(self, run_test_suite_path, parameter_list,): end_path),os.O_CREAT))
if '--firefox_bin' and '--xvfb_bin' in parameter_list: def get_parameters(self, *args, **kw):
return set(['--firefox_bin','--xvfb_bin']) call_parameter_list.append({'args': [x for x in args], 'kw':kw})
else: def patch_getSupportedParameterSet(self, run_test_suite_path, parameter_list,):
return [] if '--firefox_bin' and '--xvfb_bin' in parameter_list:
test_node = self.getTestNode() return set(['--firefox_bin','--xvfb_bin'])
test_node.slapos_controler = SlapOSControler(self.working_directory, else:
test_node.config,test_node.log) return []
node_test_suite = test_node.getNodeTestSuite('foo') test_node = self.getTestNode()
self.updateNodeTestSuiteData(node_test_suite) test_node.slapos_controler = SlapOSControler(self.working_directory,
node_test_suite.revision = 'dummy' test_node.config)
run_test_suite_path = _createPath( node_test_suite = test_node.getNodeTestSuite('foo')
os.path.join(test_node.slapos_controler.instance_root,'a/bin'),'runTestSuite') self.updateNodeTestSuiteData(node_test_suite)
def checkRunTestSuiteParameters(additional_parameter_list=None): node_test_suite.revision = 'dummy'
ProcessManager.getSupportedParameterSet = patch_getSupportedParameterSet run_test_suite_path = _createPath(
ProcessManager.spawn = get_parameters os.path.join(test_node.slapos_controler.instance_root,'a/bin'),'runTestSuite')
test_node.runTestSuite(node_test_suite,"http://foo.bar") def checkRunTestSuiteParameters(additional_parameter_list=None):
expected_parameter_list = ['%s/a/bin/runTestSuite' ProcessManager.getSupportedParameterSet = patch_getSupportedParameterSet
%(test_node.slapos_controler.instance_root), '--test_suite', 'Foo', '--revision', ProcessManager.spawn = get_parameters
'dummy', '--test_suite_title', 'Foo-Test', '--node_quantity', 3, '--master_url', test_node.runTestSuite(node_test_suite,"http://foo.bar")
'http://foo.bar'] expected_parameter_list = ['%s/a/bin/runTestSuite'
if additional_parameter_list: %(test_node.slapos_controler.instance_root), '--test_suite', 'Foo', '--revision',
expected_parameter_list.extend(additional_parameter_list) 'dummy', '--test_suite_title', 'Foo-Test', '--node_quantity', 3, '--master_url',
self.assertEqual(call_parameter_list[0]['args'], expected_parameter_list) 'http://foo.bar']
call_parameter_list = [] if additional_parameter_list:
checkRunTestSuiteParameters() expected_parameter_list.extend(additional_parameter_list)
_createPath(os.path.join(test_node.config['slapos_directory'], 'soft/a/parts/firefox'),'firefox-slapos') self.assertEqual(call_parameter_list[0]['args'], expected_parameter_list)
_createPath(os.path.join(test_node.config['slapos_directory'], 'soft/a/parts/xserver/bin'),'Xvfb') call_parameter_list = []
call_parameter_list = [] checkRunTestSuiteParameters()
checkRunTestSuiteParameters(additional_parameter_list=['--firefox_bin', _createPath(os.path.join(test_node.config['slapos_directory'], 'soft/a/parts/firefox'),'firefox-slapos')
'%s/soft/a/parts/firefox/firefox-slapos' _createPath(os.path.join(test_node.config['slapos_directory'], 'soft/a/parts/xserver/bin'),'Xvfb')
%(test_node.config['slapos_directory']), call_parameter_list = []
'--xvfb_bin', checkRunTestSuiteParameters(additional_parameter_list=['--firefox_bin',
'%s/soft/a/parts/xserver/bin/Xvfb' '%s/soft/a/parts/firefox/firefox-slapos'
%(test_node.config['slapos_directory'])]) %(test_node.config['slapos_directory']),
'--xvfb_bin',
'%s/soft/a/parts/xserver/bin/Xvfb'
%(test_node.config['slapos_directory'])])
finally:
ProcessManager.getSupportedParameterSet = original_getSupportedParameter
ProcessManager.spawn = original_spawn
def test_10_prepareSlapOS(self): def test_10_prepareSlapOS(self):
def patch_initializeSlapOSControler(self, *args, **kw):
pass
test_node = self.getTestNode() test_node = self.getTestNode()
test_node_slapos = SlapOSInstance() test_node_slapos = SlapOSInstance()
node_test_suite = test_node.getNodeTestSuite('foo') node_test_suite = test_node.getNodeTestSuite('foo')
node_test_suite.edit(working_directory=self.working_directory) node_test_suite.edit(working_directory=self.working_directory)
status_dict = {"status_code" : 0} status_dict = {"status_code" : 0}
def patch_run(self, *args,**kw): global call_list
return status_dict call_list = []
SlapOSControler.initializeSlapOSControler = patch_initializeSlapOSControler class Patch:
SlapOSControler.runSoftwareRelease = patch_run def __init__(self, method_name, status_code=0):
SlapOSControler.runComputerPartition = patch_run self.method_name = method_name
self.status_code = status_code
def __call__(self, *args, **kw):
global call_list
call_list.append({"method_name": self.method_name,
"args": [x for x in args],
"kw": kw})
return {"status_code": self.status_code}
SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler")
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease")
SlapOSControler.runComputerPartition = Patch("runComputerPartition")
test_node.prepareSlapOSForTestNode(test_node_slapos) test_node.prepareSlapOSForTestNode(test_node_slapos)
self.assertEqual(status_dict,{'status_code':0}) self.assertEquals(["initializeSlapOSControler", "runSoftwareRelease"],
[x["method_name"] for x in call_list])
call_list = []
test_node.prepareSlapOSForTestSuite(node_test_suite) test_node.prepareSlapOSForTestSuite(node_test_suite)
self.assertEqual(status_dict,{'status_code':0}) self.assertEquals(["initializeSlapOSControler", "runSoftwareRelease",
"runComputerPartition"],
[x["method_name"] for x in call_list])
call_list = []
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1)
self.assertRaises(SubprocessError, test_node.prepareSlapOSForTestSuite,
node_test_suite)
def test_11_run(self): def test_11_run(self):
def patch_function(self, *args, **kw): def doNothing(self, *args, **kw):
pass pass
def patch_killPrevious(): def patch_killPrevious():
pass pass
...@@ -385,29 +408,74 @@ branch = foo ...@@ -385,29 +408,74 @@ branch = foo
counter = 0 counter = 0
def patch_startTestSuite(self,test_node_title): def patch_startTestSuite(self,test_node_title):
global counter global counter
if counter == 2:
raise StopIteration
counter += 1
config_list = [] config_list = []
config_list.append(test_self.getTestSuiteData()[0]) def _checkExistingTestSuite(reference_set):
config_list.append(test_self.getTestSuiteData(add_third_repository=True)[0]) test_self.assertEquals(set(reference_set),
set(os.listdir(test_node.config["working_directory"])))
for x in reference_set:
test_self.assertTrue(os.path.exists(os.path.join(
test_node.config["working_directory"],x)),True)
if counter == 0:
config_list.append(test_self.getTestSuiteData()[0])
config_list.append(test_self.getTestSuiteData(reference='bar')[0])
elif counter == 1:
_checkExistingTestSuite(set(['foo']))
config_list.append(test_self.getTestSuiteData(reference='bar')[0])
config_list.append(test_self.getTestSuiteData()[0])
elif counter == 2:
_checkExistingTestSuite(set(['foo','bar']))
config_list.append(test_self.getTestSuiteData()[0])
config_list.append(test_self.getTestSuiteData(reference='qux')[0])
elif counter == 3:
_checkExistingTestSuite(set(['foo','qux']))
test_node.process_manager.under_cancellation = True
config_list.append(test_self.getTestSuiteData(reference='foox')[0])
elif counter == 4:
test_node.process_manager.under_cancellation = False
config_list.append(test_self.getTestSuiteData(reference='bax')[0])
elif counter == 5:
_checkExistingTestSuite(set(['bax']))
raise StopIteration
counter += 1
return json.dumps(config_list) return json.dumps(config_list)
def patch_createTestResult(self, revision, test_name_list, node_title, def patch_createTestResult(self, revision, test_name_list, node_title,
allow_restart=False, test_title=None, project_title=None): allow_restart=False, test_title=None, project_title=None):
test_result_path = os.path.join(test_result_path_root, test_title) global counter
result = TestResultProxy(self._proxy, self._retry_time, if counter == 3 and project_title != 'qux':
result = None
else:
test_result_path = os.path.join(test_result_path_root, test_title)
result = TestResultProxy(self._proxy, self._retry_time,
self._logger, test_result_path, node_title, revision) self._logger, test_result_path, node_title, revision)
return result return result
time.sleep = patch_function original_sleep = time.sleep
time.sleep = doNothing
self.generateTestRepositoryList() self.generateTestRepositoryList()
TaskDistributor.startTestSuite = patch_startTestSuite TaskDistributor.startTestSuite = patch_startTestSuite
TaskDistributionTool.createTestResult = patch_createTestResult TaskDistributionTool.createTestResult = patch_createTestResult
test_node = self.getTestNode() test_node = self.getTestNode()
test_node._prepareSlapOS = patch_function test_node._prepareSlapOS = doNothing
test_node.runTestSuite = patch_function test_node.runTestSuite = doNothing
SlapOSControler.initializeSlapOSControler = patch_function SlapOSControler.initializeSlapOSControler = doNothing
test_node.process_manager.killPreviousRun = patch_killPrevious test_node.process_manager.killPreviousRun = patch_killPrevious
try: try:
test_node.run() test_node.run()
except Exception as e: except Exception as e:
self.assertEqual(type(e),StopIteration) self.assertEqual(type(e),StopIteration)
finally:
time.sleep = original_sleep
def test_12_spawn(self):
def _log(*args,**kw):
for arg in args:
print "TESTNODE LOG : %r" % (arg,)
def _checkCorrectStatus(expected_status,*args):
result = process_manager.spawn(*args)
self.assertEqual(result['status_code'], expected_status)
def patch_sleep(n):
subprocess.check_call(["sleep", n/10000])
original_sleep = time.sleep
process_manager = ProcessManager(log=_log)
_checkCorrectStatus(0, *['sleep','3'])
_checkCorrectStatus(-15, *['sleep','8'])
time.sleep = original_sleep
...@@ -30,6 +30,9 @@ import subprocess ...@@ -30,6 +30,9 @@ import subprocess
import threading import threading
import signal import signal
import sys import sys
import time
MAX_TIMEOUT = 5
class SubprocessError(EnvironmentError): class SubprocessError(EnvironmentError):
def __init__(self, status_dict): def __init__(self, status_dict):
...@@ -39,6 +42,12 @@ class SubprocessError(EnvironmentError): ...@@ -39,6 +42,12 @@ class SubprocessError(EnvironmentError):
def __str__(self): def __str__(self):
return 'Error %i' % self.status_code return 'Error %i' % self.status_code
class TimeoutError(EnvironmentError):
def __init__(self):
pass
def __str__(self):
return 'Timeout expired. Process killed'
class CancellationError(EnvironmentError): class CancellationError(EnvironmentError):
pass pass
...@@ -98,8 +107,16 @@ class ProcessManager(object): ...@@ -98,8 +107,16 @@ class ProcessManager(object):
self.process_pid_set = set() self.process_pid_set = set()
signal.signal(signal.SIGTERM, self.sigterm_handler) signal.signal(signal.SIGTERM, self.sigterm_handler)
self.under_cancellation = False self.under_cancellation = False
self.p = None
self.result = None
def spawn(self, *args, **kw): def spawn(self, *args, **kw):
def timeoutExpired(p):
time.sleep(MAX_TIMEOUT)
if p.poll() is None:
p.terminate()
raise TimeoutError
if self.under_cancellation: if self.under_cancellation:
raise CancellationError("Test Result was cancelled") raise CancellationError("Test Result was cancelled")
get_output = kw.pop('get_output', True) get_output = kw.pop('get_output', True)
...@@ -120,6 +137,8 @@ class ProcessManager(object): ...@@ -120,6 +137,8 @@ class ProcessManager(object):
p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE, p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env, **subprocess_kw) stderr=subprocess.PIPE, env=env, **subprocess_kw)
self.process_pid_set.add(p.pid) self.process_pid_set.add(p.pid)
thread = threading.Thread(target=timeoutExpired, args=(p,))
thread.start()
stdout, stderr = subprocess_capture(p, self.log, log_prefix, stdout, stderr = subprocess_capture(p, self.log, log_prefix,
get_output=get_output) get_output=get_output)
result = dict(status_code=p.returncode, command=command, result = dict(status_code=p.returncode, command=command,
...@@ -127,7 +146,7 @@ class ProcessManager(object): ...@@ -127,7 +146,7 @@ class ProcessManager(object):
self.process_pid_set.discard(p.pid) self.process_pid_set.discard(p.pid)
if self.under_cancellation: if self.under_cancellation:
raise CancellationError("Test Result was cancelled") raise CancellationError("Test Result was cancelled")
if raise_error_if_fail and p.returncode: if raise_error_if_fail and p.returncode != -15 and p.returncode:
raise SubprocessError(result) raise SubprocessError(result)
return result return result
......
...@@ -257,7 +257,7 @@ branch = %(branch)s ...@@ -257,7 +257,7 @@ branch = %(branch)s
We will build slapos software needed by the testnode itself, We will build slapos software needed by the testnode itself,
like the building of selenium-runner by default like the building of selenium-runner by default
""" """
self._prepareSlapOS(self.config['slapos_directory'], return self._prepareSlapOS(self.config['slapos_directory'],
test_node_slapos, create_partition=0, test_node_slapos, create_partition=0,
software_path_list=self.config.get("software_list")) software_path_list=self.config.get("software_list"))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment