Commit 518df332 authored by Julien Muchembled's avatar Julien Muchembled Committed by Klaus Wölfel

testnode: some code clean up

A lot was found with pylint.
parent bb63c34b
...@@ -235,7 +235,7 @@ class ERP5TestNode(TestCase): ...@@ -235,7 +235,7 @@ class ERP5TestNode(TestCase):
node_test_suite.custom_profile_path) node_test_suite.custom_profile_path)
profile = open(node_test_suite.custom_profile_path, 'r') profile = open(node_test_suite.custom_profile_path, 'r')
if my_test_type=='UnitTest': if my_test_type=='UnitTest':
expected_profile = """ expected_profile = """\
[buildout] [buildout]
extends = %(temp_dir)s/testnode/foo/rep0/software.cfg extends = %(temp_dir)s/testnode/foo/rep0/software.cfg
...@@ -254,7 +254,7 @@ develop = false ...@@ -254,7 +254,7 @@ develop = false
else: else:
revision1 = "azerty" revision1 = "azerty"
revision2 = "qwerty" revision2 = "qwerty"
expected_profile = """ expected_profile = """\
[buildout] [buildout]
extends = %(temp_dir)s/testnode/foo/rep0/software.cfg extends = %(temp_dir)s/testnode/foo/rep0/software.cfg
...@@ -818,17 +818,17 @@ develop = false ...@@ -818,17 +818,17 @@ develop = false
test_node.node_test_suite_dict test_node.node_test_suite_dict
rand_part_set = set() rand_part_set = set()
self.assertEquals(2, len(test_node.node_test_suite_dict)) self.assertEquals(2, len(test_node.node_test_suite_dict))
assert(test_node.suite_log is not None) self.assertIsNot(test_node.suite_log, None)
assert(isinstance(test_node.suite_log, types.MethodType)) self.assertTrue(isinstance(test_node.suite_log, types.MethodType))
for ref, suite in test_node.node_test_suite_dict.items(): for ref, suite in test_node.node_test_suite_dict.items():
self.assertTrue('var/log/testnode/%s' % suite.reference in \ self.assertTrue('var/log/testnode/%s' % suite.reference in \
suite.suite_log_path, suite.suite_log_path,
"Incorrect suite log path : %r" % suite.suite_log_path) "Incorrect suite log path : %r" % suite.suite_log_path)
assert(suite.suite_log_path.endswith('suite.log')) self.assertTrue(suite.suite_log_path.endswith('suite.log'))
m = re.match('.*\-(.*)\/suite.log', suite.suite_log_path) m = re.match('.*\-(.*)\/suite.log', suite.suite_log_path)
rand_part = m.groups()[0] rand_part = m.groups()[0]
assert(len(rand_part) == 32) self.assertEqual(len(rand_part), 32)
assert(rand_part not in rand_part_set) self.assertNotIn(rand_part, rand_part_set)
rand_part_set.add(rand_part) rand_part_set.add(rand_part)
suite_log = open(suite.suite_log_path, 'r') suite_log = open(suite.suite_log_path, 'r')
self.assertEquals(1, len([x for x in suite_log.readlines() \ self.assertEquals(1, len([x for x in suite_log.readlines() \
......
...@@ -24,23 +24,13 @@ ...@@ -24,23 +24,13 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from datetime import datetime,timedelta import errno
import os import os
import subprocess
import sys
import time
import glob import glob
import SlapOSControler
import json
import time
import shutil import shutil
import logging
import string import string
import random import random
from ProcessManager import SubprocessError, ProcessManager, CancellationError from .Utils import createFolder
from subprocess import CalledProcessError
from Updater import Updater
from erp5.util import taskdistribution
class SlapOSInstance(object): class SlapOSInstance(object):
""" """
...@@ -57,7 +47,7 @@ class SlapOSInstance(object): ...@@ -57,7 +47,7 @@ class SlapOSInstance(object):
def _checkData(self): def _checkData(self):
pass pass
class NodeTestSuite(SlapOSInstance): class NodeTestSuite(SlapOSInstance):
""" """
...@@ -67,22 +57,19 @@ class NodeTestSuite(SlapOSInstance): ...@@ -67,22 +57,19 @@ class NodeTestSuite(SlapOSInstance):
self.reference = reference self.reference = reference
self.cluster_configuration = {} self.cluster_configuration = {}
def edit(self, **kw):
super(NodeTestSuite, self).edit(**kw)
def _checkData(self): def _checkData(self):
if getattr(self, "working_directory", None) is not None: if getattr(self, "working_directory", None) is not None:
if not(self.working_directory.endswith(os.path.sep + self.reference)): if not(self.working_directory.endswith(os.path.sep + self.reference)):
self.working_directory = os.path.join(self.working_directory, self.working_directory = os.path.join(self.working_directory,
self.reference) self.reference)
SlapOSControler.createFolder(self.working_directory) createFolder(self.working_directory)
self.test_suite_directory = os.path.join( self.test_suite_directory = os.path.join(
self.working_directory, "test_suite") self.working_directory, "test_suite")
self.custom_profile_path = os.path.join(self.working_directory, self.custom_profile_path = os.path.join(self.working_directory,
'software.cfg') 'software.cfg')
if getattr(self, "vcs_repository_list", None) is not None: if getattr(self, "vcs_repository_list", None) is not None:
for vcs_repository in self.vcs_repository_list: for vcs_repository in self.vcs_repository_list:
buildout_section_id = vcs_repository.get('buildout_section_id', None) buildout_section_id = vcs_repository.get('buildout_section_id')
repository_id = buildout_section_id or \ repository_id = buildout_section_id or \
vcs_repository.get('url').split('/')[-1].split('.')[0] vcs_repository.get('url').split('/')[-1].split('.')[0]
repository_path = os.path.join(self.working_directory,repository_id) repository_path = os.path.join(self.working_directory,repository_id)
...@@ -92,18 +79,20 @@ class NodeTestSuite(SlapOSInstance): ...@@ -92,18 +79,20 @@ class NodeTestSuite(SlapOSInstance):
def createSuiteLog(self): def createSuiteLog(self):
# /srv/slapgrid/slappartXX/srv/var/log/testnode/az-mlksjfmlk234Sljssdflkj23KSdfslj/suite.log # /srv/slapgrid/slappartXX/srv/var/log/testnode/az-mlksjfmlk234Sljssdflkj23KSdfslj/suite.log
alphabets = string.digits + string.letters alphabets = string.digits + string.letters
rand_part = ''.join(random.choice(alphabets) for i in xrange(32)) while 1:
random_suite_folder_id = '%s-%s' % (self.reference, rand_part) log_folder_name = '%s-%s' % (self.reference,
suite_log_directory = os.path.join(self.log_directory, ''.join(random.choice(alphabets) for i in xrange(32)))
random_suite_folder_id) log_folder_path = os.path.join(self.log_directory, log_folder_name)
SlapOSControler.createFolders(suite_log_directory) try:
os.makedirs(log_folder_path)
except OSError, e:
if e.errno != errno.EEXIST:
raise
else:
break
# XXX copy the whole content of the log viewer app # XXX copy the whole content of the log viewer app
for fname in glob.glob(os.path.join(os.path.dirname(__file__), 'js-logtail', '*')): for fname in glob.glob(os.path.join(os.path.dirname(__file__), 'js-logtail', '*')):
shutil.copy(fname, suite_log_directory) shutil.copy(fname, log_folder_path)
self.suite_log_path = os.path.join(suite_log_directory, self.suite_log_path = os.path.join(log_folder_path, 'suite.log')
'suite.log') return self.suite_log_path, log_folder_name
return self.getSuiteLogPath(), random_suite_folder_id
def getSuiteLogPath(self):
return getattr(self,"suite_log_path", None)
...@@ -131,14 +131,14 @@ class ProcessManager(object): ...@@ -131,14 +131,14 @@ class ProcessManager(object):
stdin = file(os.devnull) stdin = file(os.devnull)
def __init__(self, log, *args, **kw): def __init__(self, log, max_timeout=MAX_TIMEOUT):
self.log = log self.log = log
self.process_pid_set = set() self.process_pid_set = set()
signal.signal(signal.SIGTERM, self.sigterm_handler) signal.signal(signal.SIGTERM, self.sigterm_handler)
self.under_cancellation = False self.under_cancellation = False
self.p = None self.p = None
self.result = None self.result = None
self.max_timeout = kw.get("max_timeout") or MAX_TIMEOUT self.max_timeout = max_timeout
self.timer_set = set() self.timer_set = set()
def spawn(self, *args, **kw): def spawn(self, *args, **kw):
...@@ -186,10 +186,9 @@ class ProcessManager(object): ...@@ -186,10 +186,9 @@ class ProcessManager(object):
raise SubprocessError(result) raise SubprocessError(result)
return result return result
def getSupportedParameterSet(self, program_path ,parameter_list): def getSupportedParameterList(self, program_path):
help_string = self.spawn(*[program_path,'--help'])['stdout'] return re.findall(r'^ (--\w+)',
help_words = set(help_string.split()) self.spawn(program_path, '--help')['stdout'], re.M)
return help_words.intersection(set(parameter_list))
def killall(self, name): def killall(self, name):
""" """
...@@ -212,13 +211,15 @@ class ProcessManager(object): ...@@ -212,13 +211,15 @@ class ProcessManager(object):
continue continue
except (psutil.AccessDenied, psutil.NoSuchProcess): except (psutil.AccessDenied, psutil.NoSuchProcess):
continue continue
self.log('ProcesssManager, killall on %s having pid %s' % (name, process.pid)) self.log('ProcesssManager, killall on %s having pid %s',
name, process.pid)
to_kill_list.append(process.pid) to_kill_list.append(process.pid)
for pid in to_kill_list: for pid in to_kill_list:
killCommand(pid, self.log) killCommand(pid, self.log)
def killPreviousRun(self, cancellation=False): def killPreviousRun(self, cancellation=False):
self.log('ProcessManager killPreviousRun, going to kill %r' % (self.process_pid_set,)) self.log('ProcessManager killPreviousRun, going to kill %r',
self.process_pid_set)
if cancellation: if cancellation:
self.under_cancellation = True self.under_cancellation = True
for timer in self.timer_set: for timer in self.timer_set:
...@@ -227,12 +228,13 @@ class ProcessManager(object): ...@@ -227,12 +228,13 @@ class ProcessManager(object):
killCommand(pgpid, self.log) killCommand(pgpid, self.log)
try: try:
if os.path.exists(self.supervisord_pid_file): if os.path.exists(self.supervisord_pid_file):
supervisor_pid = int(open(self.supervisord_pid_file).read().strip()) with open(self.supervisord_pid_file) as f:
self.log('ProcessManager killPreviousRun, going to kill supervisor with pid %r' % supervisor_pid) supervisor_pid = int(f.read().strip())
self.log('ProcessManager killPreviousRun, going to kill supervisor with pid %r',
supervisor_pid)
os.kill(supervisor_pid, signal.SIGTERM) os.kill(supervisor_pid, signal.SIGTERM)
except: except Exception:
self.log('ProcessManager killPreviousRun, exception when killing supervisor') self.log('ProcessManager killPreviousRun, exception when killing supervisor')
pass
self.process_pid_set.clear() self.process_pid_set.clear()
def sigterm_handler(self, signal, frame): def sigterm_handler(self, signal, frame):
......
This diff is collapsed.
import datetime import datetime
import json import json
import sys
import traceback import traceback
import time import time
#import feedparser #import feedparser
from functools import wraps
from uritemplate import expand from uritemplate import expand
import slapos.slap import slapos.slap
from slapos.slap import SoftwareProductCollection from slapos.slap import SoftwareProductCollection
from slapos.slap.slap import ConnectionError
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from erp5.util.taskdistribution import SAFE_RPC_EXCEPTION_LIST from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST
# max time to instance changing state: 2 hour # max time to instance changing state: 2 hour
MAX_INSTANCE_TIME = 60*60*2 MAX_INSTANCE_TIME = 60*60*2
...@@ -41,36 +39,29 @@ TESTER_STATE_INSTANCE_UNINSTALLED = "TESTER_STATE_INSTANCE_UNINSTALLED" ...@@ -41,36 +39,29 @@ TESTER_STATE_INSTANCE_UNINSTALLED = "TESTER_STATE_INSTANCE_UNINSTALLED"
# Simple decorator to prevent raise due small # Simple decorator to prevent raise due small
# network failures. # network failures.
def retryOnNetworkFailure(func): def retryOnNetworkFailure(func,
def wrapper(*args, **kwargs): _except_list = SAFE_RPC_EXCEPTION_LIST + (
HTTPError, slapos.slap.ConnectionError),
):
def wrapper(*args, **kw):
retry_time = 64 retry_time = 64
while True: while True:
try: try:
return func(*args, **kwargs) return func(*args, **kw)
except SAFE_RPC_EXCEPTION_LIST, e: except _except_list:
print 'Network failure: %s , %s' % (sys.exc_info(), e) traceback.print_exc()
except HTTPError, e:
print 'Network failure: %s , %s' % (sys.exc_info(), e) print 'Network failure. Retry method %s in %i seconds' % (func, retry_time)
except ConnectionError, e:
print 'Network failure: %s , %s' % (sys.exc_info(), e)
except slapos.slap.ConnectionError, e:
print 'Network failure: %s , %s' % (sys.exc_info(), e)
print 'Retry method %s in %i seconds' % (func, retry_time)
time.sleep(retry_time) time.sleep(retry_time)
retry_time = min(retry_time*1.5, 640) retry_time = min(retry_time*1.5, 640)
return wraps(func)(wrapper)
wrapper.__name__ = func.__name__
wrapper.__doc__ = func.__doc__
return wrapper
class SlapOSMasterCommunicator(object): class SlapOSMasterCommunicator(object):
latest_state = None latest_state = None
def __init__(self, slap, slap_supply, slap_order, url, logger): def __init__(self, slap, slap_supply, slap_order, url, logger):
self._logger = logger self._logger = logger
self.slap = slap self.slap = slap
self.slap_order = slap_order self.slap_order = slap_order
...@@ -102,8 +93,7 @@ class SlapOSMasterCommunicator(object): ...@@ -102,8 +93,7 @@ class SlapOSMasterCommunicator(object):
if instance_title is not None: if instance_title is not None:
self.name = instance_title self.name = instance_title
if request_kw is not None: if request_kw is not None:
if isinstance(request_kw, str) or \ if isinstance(request_kw, basestring):
isinstance(request_kw, unicode):
self.request_kw = json.loads(request_kw) self.request_kw = json.loads(request_kw)
else: else:
self.request_kw = request_kw self.request_kw = request_kw
...@@ -116,12 +106,11 @@ class SlapOSMasterCommunicator(object): ...@@ -116,12 +106,11 @@ class SlapOSMasterCommunicator(object):
**self.request_kw) **self.request_kw)
def isInstanceRequested(self, instance_title): def isInstanceRequested(self, instance_title):
hateoas = getattr(self.slap, '_hateoas_navigator', None) hateoas = self._hateoas_navigator
return instance_title in hateoas.getHostingSubscriptionDict() return instance_title in hateoas.getHostingSubscriptionDict()
@retryOnNetworkFailure @retryOnNetworkFailure
def _hateoas_getComputer(self, reference): def _hateoas_getComputer(self, reference):
root_document = self.hateoas_navigator.getRootDocument() root_document = self.hateoas_navigator.getRootDocument()
search_url = root_document["_links"]['raw_search']['href'] search_url = root_document["_links"]['raw_search']['href']
...@@ -147,7 +136,6 @@ class SlapOSMasterCommunicator(object): ...@@ -147,7 +136,6 @@ class SlapOSMasterCommunicator(object):
@retryOnNetworkFailure @retryOnNetworkFailure
def getSoftwareInstallationList(self): def getSoftwareInstallationList(self):
# XXX Move me to slap.py API # XXX Move me to slap.py API
computer = self._hateoas_getComputer(self.computer_guid) computer = self._hateoas_getComputer(self.computer_guid)
# Not a list ? # Not a list ?
...@@ -191,7 +179,6 @@ class SlapOSMasterCommunicator(object): ...@@ -191,7 +179,6 @@ class SlapOSMasterCommunicator(object):
@retryOnNetworkFailure @retryOnNetworkFailure
def getInstanceUrlList(self): def getInstanceUrlList(self):
if self.hosting_subscription_url is None: if self.hosting_subscription_url is None:
hosting_subscription_dict = self.hateoas_navigator._hateoas_getHostingSubscriptionDict() hosting_subscription_dict = self.hateoas_navigator._hateoas_getHostingSubscriptionDict()
for hs in hosting_subscription_dict: for hs in hosting_subscription_dict:
...@@ -207,7 +194,6 @@ class SlapOSMasterCommunicator(object): ...@@ -207,7 +194,6 @@ class SlapOSMasterCommunicator(object):
@retryOnNetworkFailure @retryOnNetworkFailure
def getNewsFromInstance(self, url): def getNewsFromInstance(self, url):
result = self.hateoas_navigator.GET(url) result = self.hateoas_navigator.GET(url)
result = json.loads(result) result = json.loads(result)
if result['_links'].get('action_object_slap', None) is None: if result['_links'].get('action_object_slap', None) is None:
...@@ -221,7 +207,6 @@ class SlapOSMasterCommunicator(object): ...@@ -221,7 +207,6 @@ class SlapOSMasterCommunicator(object):
@retryOnNetworkFailure @retryOnNetworkFailure
def getInformationFromInstance(self, url): def getInformationFromInstance(self, url):
result = self.hateoas_navigator.GET(url) result = self.hateoas_navigator.GET(url)
result = json.loads(result) result = json.loads(result)
if result['_links'].get('action_object_slap', None) is None: if result['_links'].get('action_object_slap', None) is None:
...@@ -329,7 +314,7 @@ class SlapOSMasterCommunicator(object): ...@@ -329,7 +314,7 @@ class SlapOSMasterCommunicator(object):
self._logger('Got an error requesting partition for ' self._logger('Got an error requesting partition for '
'its state') 'its state')
return INSTANCE_STATE_UNKNOWN return INSTANCE_STATE_UNKNOWN
except: except Exception:
self._logger("ERROR getting instance state") self._logger("ERROR getting instance state")
return INSTANCE_STATE_UNKNOWN return INSTANCE_STATE_UNKNOWN
...@@ -377,6 +362,7 @@ class SlapOSMasterCommunicator(object): ...@@ -377,6 +362,7 @@ class SlapOSMasterCommunicator(object):
return {'error_message' : None} return {'error_message' : None}
class SlapOSTester(SlapOSMasterCommunicator): class SlapOSTester(SlapOSMasterCommunicator):
def __init__(self, def __init__(self,
name, name,
logger, logger,
...@@ -387,7 +373,6 @@ class SlapOSTester(SlapOSMasterCommunicator): ...@@ -387,7 +373,6 @@ class SlapOSTester(SlapOSMasterCommunicator):
computer_guid=None, # computer for supply if desired computer_guid=None, # computer for supply if desired
request_kw=None request_kw=None
): ):
super(SlapOSTester, self).__init__( super(SlapOSTester, self).__init__(
slap, slap_supply, slap_order, url, logger) slap, slap_supply, slap_order, url, logger)
...@@ -464,7 +449,6 @@ class SoftwareReleaseTester(SlapOSTester): ...@@ -464,7 +449,6 @@ class SoftwareReleaseTester(SlapOSTester):
software_timeout=3600, software_timeout=3600,
instance_timeout=3600, instance_timeout=3600,
): ):
super(SoftwareReleaseTester, self).__init__( super(SoftwareReleaseTester, self).__init__(
name, logger, slap, slap_order, slap_supply, url, computer_guid, request_kw) name, logger, slap, slap_order, slap_supply, url, computer_guid, request_kw)
......
...@@ -24,28 +24,22 @@ ...@@ -24,28 +24,22 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from datetime import datetime,timedelta
import os import os
import subprocess
import sys
import time
import glob import glob
import SlapOSControler
import json import json
import time from .ProcessManager import SubprocessError
import shutil from .SlapOSControler import SlapOSControler
import logging from .Utils import createFolder
import string
import random
from ProcessManager import SubprocessError, ProcessManager, CancellationError
from subprocess import CalledProcessError
from NodeTestSuite import SlapOSInstance
from Updater import Updater
from Utils import dealShebang
from erp5.util import taskdistribution
from slapos.grid.utils import md5digest from slapos.grid.utils import md5digest
class UnitTestRunner(): def dealShebang(run_test_suite_path):
with open(run_test_suite_path) as f:
if f.read(2) == '#!':
return f.readline().split(None, 1)
return []
class UnitTestRunner(object):
def __init__(self, testnode): def __init__(self, testnode):
self.testnode = testnode self.testnode = testnode
...@@ -53,12 +47,11 @@ class UnitTestRunner(): ...@@ -53,12 +47,11 @@ class UnitTestRunner():
""" """
Create a SlapOSControler Create a SlapOSControler
""" """
return SlapOSControler.SlapOSControler( return SlapOSControler(
working_directory, working_directory,
self.testnode.config, self.testnode.config,
self.testnode.log) self.testnode.log)
def _prepareSlapOS(self, working_directory, slapos_instance, log, def _prepareSlapOS(self, working_directory, slapos_instance, log,
create_partition=1, software_path_list=None, **kw): create_partition=1, software_path_list=None, **kw):
""" """
...@@ -66,11 +59,11 @@ class UnitTestRunner(): ...@@ -66,11 +59,11 @@ class UnitTestRunner():
""" """
slapproxy_log = os.path.join(self.testnode.config['log_directory'], slapproxy_log = os.path.join(self.testnode.config['log_directory'],
'slapproxy.log') 'slapproxy.log')
log('Configured slapproxy log to %r' % slapproxy_log) log('Configured slapproxy log to %r', slapproxy_log)
reset_software = slapos_instance.retry_software_count > 10 reset_software = slapos_instance.retry_software_count > 10
if reset_software: if reset_software:
slapos_instance.retry_software_count = 0 slapos_instance.retry_software_count = 0
log('testnode, retry_software_count : %r' % \ log('testnode, retry_software_count : %r',
slapos_instance.retry_software_count) slapos_instance.retry_software_count)
# XXX Create a new controler because working_directory can be # XXX Create a new controler because working_directory can be
...@@ -108,26 +101,22 @@ class UnitTestRunner(): ...@@ -108,26 +101,22 @@ class UnitTestRunner():
""" """
# report-url, report-project and suite-url are required to seleniumrunner # report-url, report-project and suite-url are required to seleniumrunner
# instance. This is a hack which must be removed. # instance. This is a hack which must be removed.
cluster_configuration = {}
config = self.testnode.config config = self.testnode.config
cluster_configuration['report-url'] = config.get("report-url", "") return self._prepareSlapOS(config['slapos_directory'],
cluster_configuration['report-project'] = config.get("report-project", "")
cluster_configuration['suite-url'] = config.get("suite-url", "")
return self._prepareSlapOS(self.testnode.config['slapos_directory'],
test_node_slapos, self.testnode.log, create_partition=0, test_node_slapos, self.testnode.log, create_partition=0,
software_path_list=self.testnode.config.get("software_list"), software_path_list=config.get("software_list"),
cluster_configuration=cluster_configuration cluster_configuration={
) 'report-url': config.get("report-url", ""),
'report-project': config.get("report-project", ""),
'suite-url': config.get("suite-url", ""),
})
def prepareSlapOSForTestSuite(self, node_test_suite): def prepareSlapOSForTestSuite(self, node_test_suite):
""" """
Build softwares needed by testsuites Build softwares needed by testsuites
""" """
log = self.testnode.log
if log is None:
log = self.testnode.log
return self._prepareSlapOS(node_test_suite.working_directory, return self._prepareSlapOS(node_test_suite.working_directory,
node_test_suite, log, node_test_suite, self.testnode.log,
software_path_list=[node_test_suite.custom_profile_path], software_path_list=[node_test_suite.custom_profile_path],
cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)}) cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)})
...@@ -173,8 +162,7 @@ class UnitTestRunner(): ...@@ -173,8 +162,7 @@ class UnitTestRunner():
# From this point, test runner becomes responsible for updating test # From this point, test runner becomes responsible for updating test
# result. We only do cleanup if the test runner itself is not able # result. We only do cleanup if the test runner itself is not able
# to run. # to run.
SlapOSControler.createFolder(node_test_suite.test_suite_directory, createFolder(node_test_suite.test_suite_directory, clean=True)
clean=True)
self.testnode.process_manager.spawn(*invocation_list, self.testnode.process_manager.spawn(*invocation_list,
cwd=node_test_suite.test_suite_directory, cwd=node_test_suite.test_suite_directory,
log_prefix='runTestSuite', get_output=False) log_prefix='runTestSuite', get_output=False)
......
...@@ -28,11 +28,8 @@ import errno ...@@ -28,11 +28,8 @@ import errno
import os import os
import re import re
import shutil import shutil
import subprocess
import sys import sys
import threading from .ProcessManager import SubprocessError
from ProcessManager import SubprocessError
SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$') SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE) SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
...@@ -44,9 +41,8 @@ SVN_TYPE = 'svn' ...@@ -44,9 +41,8 @@ SVN_TYPE = 'svn'
class Updater(object): class Updater(object):
_git_cache = {} _git_cache = {}
stdin = file(os.devnull)
def __init__(self, repository_path, log, revision=None, git_binary=None, def __init__(self, repository_path, log, revision=None, git_binary='git',
branch=None, realtime_output=True, process_manager=None, url=None, branch=None, realtime_output=True, process_manager=None, url=None,
working_directory=None): working_directory=None):
self.log = log self.log = log
...@@ -81,7 +77,7 @@ class Updater(object): ...@@ -81,7 +77,7 @@ class Updater(object):
def deletePycFiles(self, path): def deletePycFiles(self, path):
"""Delete *.pyc files so that deleted/moved files can not be imported""" """Delete *.pyc files so that deleted/moved files can not be imported"""
for path, dir_list, file_list in os.walk(path): for path, _, file_list in os.walk(path):
for file in file_list: for file in file_list:
if file[-4:] in ('.pyc', '.pyo'): if file[-4:] in ('.pyc', '.pyo'):
# allow several processes clean the same folder at the same time # allow several processes clean the same folder at the same time
...@@ -115,17 +111,17 @@ class Updater(object): ...@@ -115,17 +111,17 @@ class Updater(object):
git_repository_path = os.path.join(self.getRepositoryPath(), '.git') git_repository_path = os.path.join(self.getRepositoryPath(), '.git')
name = os.path.basename(os.path.normpath(self.getRepositoryPath())) name = os.path.basename(os.path.normpath(self.getRepositoryPath()))
git_repository_link_path = os.path.join(self.getRepositoryPath(), '%s.git' %name) git_repository_link_path = os.path.join(self.getRepositoryPath(), '%s.git' %name)
self.log("checking link %s -> %s.." self.log("checking link %s -> %s..",
%(git_repository_link_path,git_repository_path)) git_repository_link_path, git_repository_path)
if ( not os.path.lexists(git_repository_link_path) and \ if ( not os.path.lexists(git_repository_link_path) and \
not os.path.exists(git_repository_link_path) ): not os.path.exists(git_repository_link_path) ):
try: try:
os.symlink(git_repository_path, git_repository_link_path) os.symlink(git_repository_path, git_repository_link_path)
self.log("link: %s -> %s created" self.log("link: %s -> %s created",
%(git_repository_link_path,git_repository_path)) git_repository_link_path, git_repository_path)
except: except OSError:
self.log("Cannot create link from %s -> %s" self.log("Cannot create link from %s -> %s",
%(git_repository_link_path,git_repository_path)) git_repository_link_path, git_repository_path)
def _git_find_rev(self, ref): def _git_find_rev(self, ref):
try: try:
...@@ -152,7 +148,7 @@ class Updater(object): ...@@ -152,7 +148,7 @@ class Updater(object):
raise NotImplementedError raise NotImplementedError
def deleteRepository(self): def deleteRepository(self):
self.log("Wrong repository or wrong url, deleting repos %s" % \ self.log("Wrong repository or wrong url, deleting repos %s",
self.repository_path) self.repository_path)
shutil.rmtree(self.repository_path) shutil.rmtree(self.repository_path)
...@@ -165,7 +161,7 @@ class Updater(object): ...@@ -165,7 +161,7 @@ class Updater(object):
remote_url = self._git("config", "--get", "remote.origin.url") remote_url = self._git("config", "--get", "remote.origin.url")
if remote_url == self.url: if remote_url == self.url:
correct_url = True correct_url = True
except (SubprocessError,) as e: except SubprocessError:
self.log("SubprocessError", exc_info=sys.exc_info()) self.log("SubprocessError", exc_info=sys.exc_info())
if not(correct_url): if not(correct_url):
self.deleteRepository() self.deleteRepository()
......
import sys import os
import json
import shutil import shutil
import string
from random import choice def createFolder(folder, clean=False):
if os.path.exists(folder):
if not clean:
return
shutil.rmtree(folder)
os.mkdir(folder)
def deunicodeData(data): def deunicodeData(data):
if isinstance(data, list): if isinstance(data, list):
new_data = [] return map(deunicodeData, data)
for sub_data in data: if isinstance(data, unicode):
new_data.append(deunicodeData(sub_data)) return data.encode('utf8')
elif isinstance(data, unicode): if isinstance(data, dict):
new_data = data.encode('utf8') return {deunicodeData(key): deunicodeData(value)
elif isinstance(data, dict): for key, value in data.iteritems()}
new_data = {} return data
for key, value in data.iteritems():
key = deunicodeData(key)
value = deunicodeData(value)
new_data[key] = value
else:
new_data = data
return new_data
def dealShebang(run_test_suite_path):
line = open(run_test_suite_path, 'r').readline()
invocation_list = []
if line[:2] == '#!':
invocation_list = line[2:].split()
return invocation_list
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment