Skip to content

S
slapos.core
Commit 4b147eae authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

grid.promise: loadModule is now done in PromiseProcess class 

To allow promises to update thier sys.path and import needed dependencies,
module load is now moved to PromiseProcess class. This prevent promises to modify
sys.path of parent process.
parent c9c67752
Show whitespace changes
Inline Side-by-side
Showing with 354 additions and 185 deletions
slapos/grid/promise/__init__.py
View file @ 4b147eae
...@@ -44,49 +44,174 @@ from slapos.grid.utils import dropPrivileges ...@@ -44,49 +44,174 @@ from slapos.grid.utils import dropPrivileges
from slapos.grid.promise import interface from slapos.grid.promise import interface
from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult, from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult,
AnomalyResult, TestResult, AnomalyResult, TestResult,
PROMISE_RESULT_FOLDER_NAME) PROMISE_STATE_FOLDER_NAME,
PROMISE_RESULT_FOLDER_NAME,
PROMISE_PARAMETER_NAME,
PROMISE_PERIOD_FILE_NAME)
from slapos.grid.promise.wrapper import WrapPromise from slapos.grid.promise.wrapper import WrapPromise
class PromiseError(Exception): class PromiseError(Exception):
pass pass
class PromiseRunner(Process): class PromiseProcess(Process):
""" """
Run a promise in a new Process Run a promise in a new Process
""" """
def __init__(self, promise_instance, logger=None, allow_bang=True, uid=None, def __init__(self, partition_folder, promise_name, promise_path, argument_dict,
gid=None, cwd=None, check_anomaly=False): logger, allow_bang=True, uid=None, gid=None, wrap=False,
check_anomaly=False):
""" """
Initialise Promise Runner Initialise Promise Runner
@param promise_instance: Promise instance from GenericPromise class @param promise_name: The name of the promise to run
@param promise_path: path of the promise
@param argument_dict: all promise parameters in a dictionary
@param allow_bang: Bolean saying if bang should be called in case of @param allow_bang: Bolean saying if bang should be called in case of
anomaly failure. anomaly failure.
@param check_anomaly: Bolean saying if promise anomaly should be run.
@param wrap: say if the promise should be wrapped in a subprocess using
WrapPromise class
""" """
Process.__init__(self) Process.__init__(self)
self.promise = promise_instance # set deamon to True, so promise process will be terminated if parent exit
self.daemon = True
self.name = promise_name
self.promise_path = promise_path
self.logger = logger self.logger = logger
self.allow_bang = allow_bang self.allow_bang = allow_bang
self.check_anomaly = check_anomaly self.check_anomaly = check_anomaly
self.argument_dict = argument_dict
self.uid = uid self.uid = uid
self.gid = gid self.gid = gid
self.cwd = cwd self.partition_folder = partition_folder
self.wrap_promise = wrap
self._periodicity = None
self._timestamp_file = os.path.join(partition_folder,
PROMISE_STATE_FOLDER_NAME,
'%s.timestamp' % promise_name)
periodicity_file = os.path.join(partition_folder,
PROMISE_STATE_FOLDER_NAME,
PROMISE_PERIOD_FILE_NAME % promise_name)
if os.path.exists(periodicity_file) and os.stat(periodicity_file).st_size:
with open(periodicity_file) as f:
try:
self._periodicity = float(f.read())
except ValueError:
# set to None, run the promise and regenerate the file
pass
def isPeriodicityMatch(self):
"""
Return True if promise should be run now, considering the promise
periodicity in minutes
"""
if self._periodicity is not None and \
os.path.exists(self._timestamp_file) and \
os.stat(self._timestamp_file).st_size:
with open(self._timestamp_file) as f:
try:
latest_timestamp = float(f.read())
current_timediff = (time.time() - latest_timestamp) / 60.0
if current_timediff >= self._periodicity:
return True
#self.logger.debug("Skip Promise %r. periodicity=%s, time_diff=%s" % (
# self.name, self._periodicity, current_timediff))
except ValueError:
# if the file is broken, run the promise and regenerate it
return True
else:
return False
return True
def setPromiseStartTimestamp(self):
"""
Save the promise execution timestamp
"""
state_directory = os.path.dirname(self._timestamp_file)
mkdir_p(state_directory)
with open(self._timestamp_file, 'w') as f:
f.write(str(time.time()))
def getPromiseTitle(self):
return os.path.splitext(self.name)[0]
def run(self): def run(self):
"""
Run the promise
This will first load the promise module (which will update process sys.path)
"""
try:
os.chdir(self.partition_folder)
self.setPromiseStartTimestamp()
if self.uid and self.gid: if self.uid and self.gid:
dropPrivileges(self.uid, self.gid, logger=self.logger) dropPrivileges(self.uid, self.gid, logger=self.logger)
if self.cwd is not None:
os.chdir(self.cwd) if self.wrap_promise:
try: promise_instance = WrapPromise(self.argument_dict)
self.promise.run(self.check_anomaly, self.allow_bang) else:
except Exception, e: self._createInitFile()
if self.logger: promise_module = self._loadPromiseModule()
self.logger.error(str(e)) promise_instance = promise_module.RunPromise(self.argument_dict)
promise_instance.run(self.check_anomaly, self.allow_bang)
except Exception:
self.logger.error(traceback.format_exc())
raise raise
def _createInitFile(self):
promise_folder = os.path.dirname(self.promise_path)
# if there is no __init__ file, add it
init_file = os.path.join(promise_folder, '__init__.py')
if not os.path.exists(init_file):
with open(init_file, 'w') as f:
f.write("")
os.chmod(init_file, 0644)
# add promise folder to sys.path so we can import promise script
if sys.path[0] != promise_folder:
sys.path[0:0] = [promise_folder]
def _loadPromiseModule(self):
"""Load a promise from promises directory."""
if re.match(r'[a-zA-Z_]', self.name) is None:
raise ValueError("Promise plugin name %r is not valid" % self.name)
promise_module = importlib.import_module(os.path.splitext(self.name)[0])
if not hasattr(promise_module, "RunPromise"):
raise AttributeError("Class RunPromise not found in promise" \
"%s" % self.name)
if not interface.IPromise.implementedBy(promise_module.RunPromise):
raise RuntimeError("RunPromise class in %s must implements 'IPromise'" \
" interface. zope_interface.implements(interface.IPromise) is" \
" missing ?" % self.name)
from slapos.grid.promise.generic import GenericPromise
if not issubclass(promise_module.RunPromise, GenericPromise):
raise RuntimeError("RunPromise class is not a subclass of " \
"GenericPromise class.")
if promise_module.__file__ != self.promise_path:
# cached module need to be updated
promise_module = reload(promise_module)
# load extra parameters
self._loadPromiseParameterDict(promise_module)
return promise_module
def _loadPromiseParameterDict(self, promise_module):
"""Load a promise parameters."""
if hasattr(promise_module, PROMISE_PARAMETER_NAME):
extra_dict = getattr(promise_module, PROMISE_PARAMETER_NAME)
if not isinstance(extra_dict, dict):
raise ValueError("Extra parameter is not a dict")
for key in extra_dict:
if self.argument_dict.has_key(key):
raise ValueError("Extra parameter name %r cannot be used.\n%s" % (
key, extra_dict))
self.argument_dict[key] = extra_dict[key]
class PromiseLauncher(object): class PromiseLauncher(object):
...@@ -189,27 +314,7 @@ class PromiseLauncher(object): ...@@ -189,27 +314,7 @@ class PromiseLauncher(object):
if not os.path.exists(self.promise_output_dir): if not os.path.exists(self.promise_output_dir):
mkdir_p(self.promise_output_dir) mkdir_p(self.promise_output_dir)
def _loadPromiseModule(self, promise_name): def _getErrorPromiseResult(self, promise_process, promise_name, message,
"""Load a promise from promises directory."""
if re.match(r'[a-zA-Z_]', promise_name) is None:
self.logger.error("Promise plugin name %r is not valid" % promise_name)
promise_module = importlib.import_module(os.path.splitext(promise_name)[0])
if not hasattr(promise_module, "RunPromise"):
raise AttributeError("Class RunPromise not found in promise" \
"%s" % promise_name)
if not interface.IPromise.implementedBy(promise_module.RunPromise):
raise RuntimeError("RunPromise class in %s must implements 'IPromise'" \
" interface. zope_interface.implements(interface.IPromise) is" \
" missing ?" % promise_name)
if not issubclass(promise_module.RunPromise, GenericPromise):
raise RuntimeError("RunPromise class is not a subclass of" \
"GenericPromise class.")
return promise_module
def _getErrorPromiseResult(self, promise_instance, promise_name, message,
execution_time=0): execution_time=0):
if self.check_anomaly: if self.check_anomaly:
result = AnomalyResult(problem=True, message=message) result = AnomalyResult(problem=True, message=message)
...@@ -219,7 +324,7 @@ class PromiseLauncher(object): ...@@ -219,7 +324,7 @@ class PromiseLauncher(object):
item=result, item=result,
path=os.path.join(self.promise_folder, promise_name), path=os.path.join(self.promise_folder, promise_name),
name=promise_name, name=promise_name,
title=promise_instance.getTitle(), title=promise_process.getPromiseTitle(),
execution_time=execution_time execution_time=execution_time
) )
...@@ -257,7 +362,8 @@ class PromiseLauncher(object): ...@@ -257,7 +362,8 @@ class PromiseLauncher(object):
)) ))
return result return result
def _launchPromise(self, promise_name, argument_dict, promise_module=None): def _launchPromise(self, promise_name, promise_path, argument_dict,
wrap_process=False):
""" """
Launch the promise and save the result. If promise_module is None, Launch the promise and save the result. If promise_module is None,
the promise will be run with the promise process wap module. the promise will be run with the promise process wap module.
...@@ -267,38 +373,34 @@ class PromiseLauncher(object): ...@@ -267,38 +373,34 @@ class PromiseLauncher(object):
""" """
self.logger.info("Checking promise %s..." % promise_name) self.logger.info("Checking promise %s..." % promise_name)
try: try:
if promise_module is None: promise_process = PromiseProcess(
promise_instance = WrapPromise(argument_dict) self.partition_folder,
else: promise_name,
promise_instance = promise_module.RunPromise(argument_dict) promise_path,
if not self.force and not promise_instance.isPeriodicityMatch(): argument_dict,
result = self._loadPromiseResult(promise_instance.getTitle()) logger=self.logger,
check_anomaly=self.check_anomaly,
allow_bang=not (self.bang_called or self.dry_run) and self.check_anomaly,
uid=self.uid,
gid=self.gid,
wrap=wrap_process,
)
if not self.force and not promise_process.isPeriodicityMatch():
# we won't start the promise process, just get the latest result
result = self._loadPromiseResult(promise_process.getPromiseTitle())
if result is not None: if result is not None:
if result.item.hasFailed(): if result.item.hasFailed():
self.logger.error(result.item.message) self.logger.error(result.item.message)
return True return True
return False return False
promise_instance.setPromiseRunTimestamp() promise_process.start()
except Exception: except Exception:
# only print traceback to not prevent run other promises # only print traceback to not prevent run other promises
self.logger.error(traceback.format_exc()) self.logger.error(traceback.format_exc())
self.logger.warning("Promise %s skipped." % promise_name) self.logger.warning("Promise %s skipped." % promise_name)
return True return True
promise_process = PromiseRunner(
promise_instance,
check_anomaly=self.check_anomaly,
allow_bang=not (self.bang_called or self.dry_run) and self.check_anomaly,
uid=self.uid,
gid=self.gid,
cwd=self.partition_folder,
logger=self.logger
)
# set deamon to True, so promise process will be terminated if parent exit
promise_process.daemon = True
promise_process.start()
queue_item = None queue_item = None
sleep_time = 0.1 sleep_time = 0.1
increment_limit = int(self.promise_timeout / sleep_time) increment_limit = int(self.promise_timeout / sleep_time)
...@@ -345,7 +447,7 @@ class PromiseLauncher(object): ...@@ -345,7 +447,7 @@ class PromiseLauncher(object):
promise_process.join() # wait for process to terminate promise_process.join() # wait for process to terminate
message = 'Promise timed out after %s seconds' % self.promise_timeout message = 'Promise timed out after %s seconds' % self.promise_timeout
queue_item = self._getErrorPromiseResult( queue_item = self._getErrorPromiseResult(
promise_instance, promise_process,
promise_name=promise_name, promise_name=promise_name,
message=message, message=message,
execution_time=execution_time execution_time=execution_time
...@@ -353,7 +455,7 @@ class PromiseLauncher(object): ...@@ -353,7 +455,7 @@ class PromiseLauncher(object):
if queue_item is None: if queue_item is None:
queue_item = self._getErrorPromiseResult( queue_item = self._getErrorPromiseResult(
promise_instance, promise_process,
promise_name=promise_name, promise_name=promise_name,
message="No output returned by the promise", message="No output returned by the promise",
execution_time=execution_time execution_time=execution_time
...@@ -393,40 +495,25 @@ class PromiseLauncher(object): ...@@ -393,40 +495,25 @@ class PromiseLauncher(object):
} }
if os.path.exists(self.promise_folder) and os.path.isdir(self.promise_folder): if os.path.exists(self.promise_folder) and os.path.isdir(self.promise_folder):
# if there is no __init__ file, add it
init_file = os.path.join(self.promise_folder, '__init__.py')
if not os.path.exists(init_file):
with open(init_file, 'w') as f:
f.write("")
os.chmod(init_file, 0644)
if sys.path[0] != self.promise_folder:
sys.path[0:0] = [self.promise_folder]
promise_list = []
# load all promises so we can catch import errors before launch them
for promise_name in os.listdir(self.promise_folder): for promise_name in os.listdir(self.promise_folder):
if promise_name.startswith('__init__') or \ if promise_name.startswith('__init__') or \
not promise_name.endswith('.py'): not promise_name.endswith('.py'):
continue continue
if self.run_only_promise_list is not None and not \ if self.run_only_promise_list is not None and not \
promise_name in self.run_only_promise_list: promise_name in self.run_only_promise_list:
continue continue
promise_list.append((promise_name,
self._loadPromiseModule(promise_name)))
for name, module in promise_list: promise_path = os.path.join(self.promise_folder, promise_name)
promise_path = os.path.join(self.promise_folder, name)
config = { config = {
'path': promise_path, 'path': promise_path,
'name': name 'name': promise_name
} }
if module.__file__ != promise_path:
# cached module need to be updated
module = reload(module)
config.update(base_config) config.update(base_config)
if self._launchPromise(name, config, module) and not failed_promise_name: if self._launchPromise(promise_name, promise_path, config) and \
failed_promise_name = name not failed_promise_name:
failed_promise_name = promise_name
if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \ if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \
and os.path.isdir(self.legacy_promise_folder): and os.path.isdir(self.legacy_promise_folder):
...@@ -444,7 +531,11 @@ class PromiseLauncher(object): ...@@ -444,7 +531,11 @@ class PromiseLauncher(object):
} }
config.update(base_config) config.update(base_config)
# We will use promise wrapper to run this # We will use promise wrapper to run this
if self._launchPromise(promise_name, config) and not failed_promise_name: result_state = self._launchPromise(promise_name,
promise_path,
config,
wrap_process=True)
if result_state and not failed_promise_name:
failed_promise_name = promise_name failed_promise_name = promise_name
stat_info = os.stat(self.partition_folder) stat_info = os.stat(self.partition_folder)
......
slapos/grid/promise/generic.py
View file @ 4b147eae
...@@ -43,6 +43,9 @@ PROMISE_STATE_FOLDER_NAME = '.slapgrid/promise' ...@@ -43,6 +43,9 @@ PROMISE_STATE_FOLDER_NAME = '.slapgrid/promise'
PROMISE_RESULT_FOLDER_NAME = '.slapgrid/promise/result' PROMISE_RESULT_FOLDER_NAME = '.slapgrid/promise/result'
PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log' PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log'
PROMISE_PARAMETER_NAME = 'extra_config_dict'
PROMISE_PERIOD_FILE_NAME = '%s.periodicity'
class BaseResult(object): class BaseResult(object):
def __init__(self, problem=False, message=None, date=None): def __init__(self, problem=False, message=None, date=None):
self.__problem = problem self.__problem = problem
...@@ -131,23 +134,21 @@ class GenericPromise(object): ...@@ -131,23 +134,21 @@ class GenericPromise(object):
self.__log_folder = self.__config.pop('log-folder', None) self.__log_folder = self.__config.pop('log-folder', None)
self.__partition_folder = self.__config.pop('partition-folder', None) self.__partition_folder = self.__config.pop('partition-folder', None)
self.__periodicity = self.__config.pop('periodicity', 2)
self.__debug = self.__config.pop('debug', True) self.__debug = self.__config.pop('debug', True)
self.__name = self.__config.pop('name', None) self.__name = self.__config.pop('name', None)
self.__promise_path = self.__config.pop('path', None) self.__promise_path = self.__config.pop('path', None)
self.__queue = self.__config.pop('queue', None) self.__queue = self.__config.pop('queue', None)
self.__logger_buffer = None self.__logger_buffer = None
self.__periodicity_file = os.path.join(
self.__partition_folder,
PROMISE_STATE_FOLDER_NAME,
PROMISE_PERIOD_FILE_NAME % self.__name)
self.setPeriodicity(self.__config.pop('periodicity', 2))
self.__transaction_id = '%s-%s' % (int(time.time()), random.randint(100, 999)) self.__transaction_id = '%s-%s' % (int(time.time()), random.randint(100, 999))
self._validateConf() self._validateConf()
self._configureLogger() self._configureLogger()
for key, value in config.items():
setattr(self, key.replace('-', '_'), value)
self.__timestamp_file = os.path.join(self.__partition_folder,
PROMISE_STATE_FOLDER_NAME,
self.__name)
def _configureLogger(self): def _configureLogger(self):
self.logger = logging.getLogger(self.__name) self.logger = logging.getLogger(self.__name)
...@@ -213,41 +214,12 @@ class GenericPromise(object): ...@@ -213,41 +214,12 @@ class GenericPromise(object):
if minute <= 0: if minute <= 0:
raise ValueError("Cannot set promise periodicity to a value less than 1") raise ValueError("Cannot set promise periodicity to a value less than 1")
self.__periodicity = minute self.__periodicity = minute
with open(self.__periodicity_file, 'w') as f:
f.write('%s' % minute)
def getPeriodicity(self): def getPeriodicity(self):
return self.__periodicity return self.__periodicity
def isPeriodicityMatch(self):
"""
Return True if promise should be run now, considering given the execution
periodicity in minutes
"""
if os.path.exists(self.__timestamp_file) and \
os.stat(self.__timestamp_file).st_size:
with open(self.__timestamp_file) as f:
try:
latest_timestamp = float(f.read())
current_timediff = (time.time() - latest_timestamp) / 60.0
if current_timediff >= self.__periodicity:
return True
self.logger.debug("Skip Promise %r. periodicity=%s, time_diff=%s" % (
self.__name, self.__periodicity, current_timediff))
except ValueError:
# if the file is broken, run the promise and regenerate it
return True
else:
return False
return True
def setPromiseRunTimestamp(self):
"""
Save the promise execution timestamp
"""
state_directory = os.path.dirname(self.__timestamp_file)
mkdir_p(state_directory)
with open(self.__timestamp_file, 'w') as f:
f.write(str(time.time()))
def __bang(self, message): def __bang(self, message):
""" """
Call bang if requested Call bang if requested
......
slapos/tests/promise.py
View file @ 4b147eae
# -*- coding: utf-8 -*-
############################################################################## ##############################################################################
# #
# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved. # Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
...@@ -31,13 +32,15 @@ import sys ...@@ -31,13 +32,15 @@ import sys
import time import time
import json import json
import random import random
import logging
from datetime import datetime, timedelta from datetime import datetime, timedelta
import Queue import Queue
from zope import interface as zope_interface from zope import interface as zope_interface
from slapos.grid.promise import interface, PromiseLauncher, PromiseError from slapos.grid.promise import interface, PromiseLauncher, PromiseProcess, PromiseError
from slapos.grid.promise.generic import (GenericPromise, TestResult, AnomalyResult, from slapos.grid.promise.generic import (GenericPromise, TestResult, AnomalyResult,
PromiseQueueResult, PROMISE_STATE_FOLDER_NAME, PromiseQueueResult, PROMISE_STATE_FOLDER_NAME,
PROMISE_RESULT_FOLDER_NAME) PROMISE_RESULT_FOLDER_NAME,
PROMISE_PARAMETER_NAME)
class TestSlapOSPromiseMixin(unittest.TestCase): class TestSlapOSPromiseMixin(unittest.TestCase):
...@@ -96,6 +99,35 @@ class TestSlapOSPromiseMixin(unittest.TestCase): ...@@ -96,6 +99,35 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
if save_method: if save_method:
self.launcher._savePromiseResult = save_method self.launcher._savePromiseResult = save_method
def genPromiseConfigDict(self, promise_name):
return {
'log-folder': None,
'partition-folder': self.partition_dir,
'master-url': "https://master.url.com",
'partition-cert': "",
'partition-key': "",
'partition-id': self.partition_id,
'computer-id': self.computer_id,
'debug': False,
'name': promise_name,
'path': os.path.join(self.plugin_dir, promise_name),
'queue': Queue.Queue(),
}
def createPromiseProcess(self, promise_name, check_anomaly=False, wrap=False):
logging.basicConfig()
promise_path = os.path.join(self.plugin_dir, promise_name)
return PromiseProcess(
self.partition_dir,
promise_name,
promise_path,
logger=logging.getLogger('grid.promise'),
argument_dict=self.genPromiseConfigDict(promise_name),
check_anomaly=check_anomaly,
wrap=wrap,
)
def writeFile(self, path, content, mode=0644): def writeFile(self, path, content, mode=0644):
with open(path, 'w') as f: with open(path, 'w') as f:
f.write(content) f.write(content)
...@@ -140,20 +172,21 @@ class TestSlapOSPromiseLauncher(TestSlapOSPromiseMixin): ...@@ -140,20 +172,21 @@ class TestSlapOSPromiseLauncher(TestSlapOSPromiseMixin):
def test_promise_match_interface(self): def test_promise_match_interface(self):
promise_name = 'my_promise.py' promise_name = 'my_promise.py'
self.configureLauncher()
self.generatePromiseScript(promise_name)
self.writeInit() self.writeInit()
self.generatePromiseScript(promise_name)
promise_process = self.createPromiseProcess(promise_name)
promise_module = self.launcher._loadPromiseModule(promise_name) promise_module = promise_process._loadPromiseModule()
def test_promise_match_interface_bad_name(self): def test_promise_match_interface_bad_name(self):
promise_name = 'my_promise_no_py' promise_name = 'my_promise_no_py'
self.configureLauncher()
self.generatePromiseScript(promise_name)
self.writeInit() self.writeInit()
self.generatePromiseScript(promise_name)
promise_process = self.createPromiseProcess(promise_name)
with self.assertRaises(ImportError): with self.assertRaises(ImportError) as exc:
promise_module = self.launcher._loadPromiseModule(promise_name) promise_module = promise_process._loadPromiseModule()
self.assertEquals(exc.exception.message, 'No module named my_promise_no_py')
def test_promise_match_interface_no_implement(self): def test_promise_match_interface_no_implement(self):
promise_name = 'my_promise_noimplement.py' promise_name = 'my_promise_noimplement.py'
...@@ -169,12 +202,15 @@ class RunPromise(GenericPromise): ...@@ -169,12 +202,15 @@ class RunPromise(GenericPromise):
""" """
promise_path = os.path.join(self.plugin_dir, promise_name) promise_path = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher()
self.writeInit()
self.writeFile(promise_path, promise_content) self.writeFile(promise_path, promise_content)
self.writeInit()
promise_process = self.createPromiseProcess(promise_name)
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError) as exc:
promise_module = self.launcher._loadPromiseModule(promise_name) promise_module = promise_process._loadPromiseModule()
message = "RunPromise class in my_promise_noimplement.py must implements" \
" 'IPromise' interface. zope_interface.implements(interface.IPromise) is missing ?"
self.assertEquals(exc.exception.message, message)
def test_promise_match_interface_no_generic(self): def test_promise_match_interface_no_generic(self):
promise_name = 'my_promise_nogeneric.py' promise_name = 'my_promise_nogeneric.py'
...@@ -193,12 +229,14 @@ class RunPromise(object): ...@@ -193,12 +229,14 @@ class RunPromise(object):
""" """
promise_path = os.path.join(self.plugin_dir, promise_name) promise_path = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher()
self.writeInit()
self.writeFile(promise_path, promise_content) self.writeFile(promise_path, promise_content)
self.writeInit()
promise_process = self.createPromiseProcess(promise_name)
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError) as exc:
promise_module = self.launcher._loadPromiseModule(promise_name) promise_module = promise_process._loadPromiseModule()
self.assertEquals(exc.exception.message, 'RunPromise class is not a subclass of GenericPromise class.')
def test_promise_match_interface_no_sense(self): def test_promise_match_interface_no_sense(self):
promise_name = 'my_promise_nosense.py' promise_name = 'my_promise_nosense.py'
...@@ -218,13 +256,72 @@ class RunPromise(GenericPromise): ...@@ -218,13 +256,72 @@ class RunPromise(GenericPromise):
""" """
promise_path = os.path.join(self.plugin_dir, promise_name) promise_path = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher()
self.writeInit()
self.writeFile(promise_path, promise_content) self.writeFile(promise_path, promise_content)
self.writeInit()
promise_process = self.createPromiseProcess(promise_name)
with self.assertRaises(TypeError): with self.assertRaises(TypeError) as exc:
promise_module = self.launcher._loadPromiseModule(promise_name) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise({}) promise = promise_module.RunPromise({})
self.assertEquals(exc.exception.message,
"Can't instantiate abstract class RunPromise with abstract methods sense")
def test_promise_extra_config(self):
promise_name = 'my_promise_extra.py'
config_dict = {'foo': 'bar', 'my-config': 4522111,
'text': 'developers \ninformation, sample\n\ner'}
promise_content = """from zope import interface as zope_interface
from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise
%(config_name)s = %(config_content)s
class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def sense(self):
pass
""" % {'config_name': PROMISE_PARAMETER_NAME,
'config_content': config_dict}
promise_path = os.path.join(self.plugin_dir, promise_name)
self.writeFile(promise_path, promise_content)
self.writeInit()
promise_process = self.createPromiseProcess(promise_name)
promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(promise_process.argument_dict)
self.assertEquals(promise.getConfig('foo'), 'bar')
self.assertEquals(promise.getConfig('my-config'), 4522111)
self.assertEquals(promise.getConfig('text'), config_dict['text'])
def test_promise_extra_config_reserved_name(self):
promise_name = 'my_promise_extra.py'
config_dict = {'name': 'bar', 'my-config': 4522111}
promise_content = """from zope import interface as zope_interface
from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise
%(config_name)s = %(config_content)s
class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def sense(self):
pass
""" % {'config_name': PROMISE_PARAMETER_NAME,
'config_content': config_dict}
promise_path = os.path.join(self.plugin_dir, promise_name)
self.writeFile(promise_path, promise_content)
self.writeInit()
promise_process = self.createPromiseProcess(promise_name)
with self.assertRaises(ValueError) as exc:
promise_module = promise_process._loadPromiseModule()
self.assertEquals(exc.exception.message, "Extra parameter name 'name' cannot be used.\n%s" % config_dict)
def test_runpromise(self): def test_runpromise(self):
promise_name = 'my_promise.py' promise_name = 'my_promise.py'
...@@ -445,11 +542,8 @@ class RunPromise(GenericPromise): ...@@ -445,11 +542,8 @@ class RunPromise(GenericPromise):
self.launcher.run() self.launcher.run()
self.assertEquals(exc.exception.message, 'Promise %r failed.' % second_promise) self.assertEquals(exc.exception.message, 'Promise %r failed.' % second_promise)
if "my_second_promise" in sys.modules:
# force to reload the module without rerun python # force to reload the module without rerun python
os.system('rm %s/*.pyc' % self.plugin_dir) os.system('rm %s/*.pyc' % self.plugin_dir)
del sys.modules["my_second_promise"]
self.generatePromiseScript(second_promise, success=True) self.generatePromiseScript(second_promise, success=True)
# wait next periodicity # wait next periodicity
time.sleep(2) time.sleep(2)
...@@ -624,10 +718,8 @@ class RunPromise(GenericPromise): ...@@ -624,10 +718,8 @@ class RunPromise(GenericPromise):
second_date = second_result['result']['date'] second_date = second_result['result']['date']
time.sleep(4) time.sleep(4)
if "my_second_promise" in sys.modules:
# force to reload the module without rerun python # force to reload the module without rerun python
os.system('rm %s/*.pyc' % self.plugin_dir) os.system('rm %s/*.pyc' % self.plugin_dir)
del sys.modules["my_second_promise"]
# second_promise is now success # second_promise is now success
self.generatePromiseScript(second_promise, success=True, periodicity=0.05) self.generatePromiseScript(second_promise, success=True, periodicity=0.05)
...@@ -856,7 +948,6 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -856,7 +948,6 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
'partition-folder': self.partition_dir, 'partition-folder': self.partition_dir,
'promise-timeout': timeout, 'promise-timeout': timeout,
'debug': False, 'debug': False,
'slapgrid-mode': False,
'check-anomaly': True, 'check-anomaly': True,
'master-url': "https://master.url.com", 'master-url': "https://master.url.com",
'partition-cert': '', 'partition-cert': '',
...@@ -868,10 +959,23 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -868,10 +959,23 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
'name': self.promise_name 'name': self.promise_name
} }
def createPromiseProcess(self, check_anomaly=False, wrap=False):
logging.basicConfig()
return PromiseProcess(
self.partition_dir,
self.promise_name,
self.promise_path,
logger=logging.getLogger('grid.promise'),
argument_dict=self.promise_config,
check_anomaly=check_anomaly,
wrap=wrap,
)
def test_create_simple_promise(self): def test_create_simple_promise(self):
self.initialisePromise() self.initialisePromise()
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
self.assertEquals(promise.getPeriodicity(), 1) self.assertEquals(promise.getPeriodicity(), 1)
self.assertEquals(promise.getName(), self.promise_name) self.assertEquals(promise.getName(), self.promise_name)
...@@ -899,11 +1003,9 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -899,11 +1003,9 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_anomaly_disabled(self): def test_promise_anomaly_disabled(self):
self.initialisePromise() self.initialisePromise()
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
# disable anomaly call, enable test call
promise.setConfig("check-anomaly", False)
promise.run() promise.run()
result = self.queue.get(True, 1) result = self.queue.get(True, 1)
...@@ -919,8 +1021,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -919,8 +1021,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_with_raise(self): def test_promise_with_raise(self):
promise_content = "raise ValueError('Bad Promise raised')" promise_content = "raise ValueError('Bad Promise raised')"
self.initialisePromise(promise_content) self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
# no raise # no raise
...@@ -934,8 +1036,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -934,8 +1036,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_no_return(self): def test_promise_no_return(self):
promise_content = "return" promise_content = "return"
self.initialisePromise(promise_content) self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
# no raise # no raise
...@@ -949,8 +1051,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -949,8 +1051,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_resultfromlog(self): def test_promise_resultfromlog(self):
promise_content = "self.logger.info('Promise is running...')" promise_content = "self.logger.info('Promise is running...')"
self.initialisePromise(promise_content) self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
date = datetime.now() date = datetime.now()
...@@ -969,8 +1071,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -969,8 +1071,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_resultfromlog_error(self): def test_promise_resultfromlog_error(self):
promise_content = 'self.logger.error("Promise is running...\\nmessage in new line")' promise_content = 'self.logger.error("Promise is running...\\nmessage in new line")'
self.initialisePromise(promise_content) self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
date = datetime.now() date = datetime.now()
...@@ -991,8 +1093,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -991,8 +1093,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
self.log_dir = None self.log_dir = None
promise_content = "self.logger.info('Promise is running...')" promise_content = "self.logger.info('Promise is running...')"
self.initialisePromise(promise_content) self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
date = datetime.now() date = datetime.now()
...@@ -1013,8 +1115,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1013,8 +1115,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_resultfromlog_latest_minutes(self): def test_promise_resultfromlog_latest_minutes(self):
self.initialisePromise(timeout=60) self.initialisePromise(timeout=60)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
# write some random logs # write some random logs
...@@ -1046,8 +1148,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1046,8 +1148,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_resultfromlog_latest_minutes_multilog(self): def test_promise_resultfromlog_latest_minutes_multilog(self):
self.initialisePromise(timeout=60) self.initialisePromise(timeout=60)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
# write some random logs # write some random logs
...@@ -1091,8 +1193,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1091,8 +1193,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_resultfromlog_result_count(self): def test_promise_resultfromlog_result_count(self):
self.initialisePromise() self.initialisePromise()
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
# write some random logs # write some random logs
...@@ -1133,8 +1235,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1133,8 +1235,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_resultfromlog_result_count_many(self): def test_promise_resultfromlog_result_count_many(self):
self.initialisePromise() self.initialisePromise()
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
# write some random logs # write some random logs
...@@ -1182,8 +1284,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1182,8 +1284,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_defaulttest(self): def test_promise_defaulttest(self):
promise_content = 'self.logger.info("Promise is running...\\nmessage in new line")' promise_content = 'self.logger.info("Promise is running...\\nmessage in new line")'
self.initialisePromise(promise_content) self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
promise.sense() promise.sense()
...@@ -1195,8 +1297,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1195,8 +1297,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_defaulttest_failure(self): def test_promise_defaulttest_failure(self):
self.initialisePromise(success=False) self.initialisePromise(success=False)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
promise.sense() promise.sense()
...@@ -1208,8 +1310,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1208,8 +1310,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_defaulttest_error_if_two_fail(self): def test_promise_defaulttest_error_if_two_fail(self):
self.initialisePromise(success=False, timeout=1) self.initialisePromise(success=False, timeout=1)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
promise.sense() promise.sense()
...@@ -1221,6 +1323,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1221,6 +1323,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
self.assertEquals(result.hasFailed(), False) self.assertEquals(result.hasFailed(), False)
self.initialisePromise(success=False, timeout=1) self.initialisePromise(success=False, timeout=1)
promise_process = self.createPromiseProcess()
promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
promise.sense() promise.sense()
result = promise._test(result_count=2, failure_amount=2) result = promise._test(result_count=2, failure_amount=2)
...@@ -1229,6 +1333,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1229,6 +1333,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
# will continue to fail # will continue to fail
self.initialisePromise(success=False, timeout=1) self.initialisePromise(success=False, timeout=1)
promise_process = self.createPromiseProcess()
promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
promise.sense() promise.sense()
result = promise._test(result_count=2, failure_amount=2) result = promise._test(result_count=2, failure_amount=2)
...@@ -1238,8 +1344,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1238,8 +1344,8 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def test_promise_defaulttest_anomaly(self): def test_promise_defaulttest_anomaly(self):
promise_content = 'self.logger.info("Promise is running...\\nmessage in new line")' promise_content = 'self.logger.info("Promise is running...\\nmessage in new line")'
self.initialisePromise(promise_content) self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name) promise_process = self.createPromiseProcess()
reload(promise_module) promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise(self.promise_config) promise = promise_module.RunPromise(self.promise_config)
promise.sense() promise.sense()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7