slapgid.promise: add tests, fixup

slapos/grid/promise/__init__.py

@@ -107,12 +107,11 @@ class GenericPromise(object):
 
     self.__log_folder = self.__config.pop('log-folder', None)
     self.__partition_folder = self.__config.pop('partition-folder', None)
-    self.__check_anomaly = self.__config.pop('check-anomaly', False)
     self.__periodicity = self.__config.pop('periodicity', 2)
     self.__debug = self.__config.pop('debug', True)
     self.__name = self.__config.pop('name', None)
     self.__promise_path = self.__config.pop('path', None)
-    self.queue = self.__config.pop('queue', None)
+    self.__queue = self.__config.pop('queue', None)
     self.__logger_buffer = None
 
     self._validateConf()
@@ -122,6 +121,8 @@ class GenericPromise(object):
 
   def _configureLogger(self):
     self.logger = logging.getLogger(self.__name)
+    for handler in self.logger.handlers:
+      self.logger.removeHandler(handler)
     if self.__log_folder is None:
       # configure logger with StringIO
       import cStringIO
@@ -143,7 +144,7 @@ class GenericPromise(object):
     self.logger.addHandler(logger_handler)
 
   def _validateConf(self):
-    if self.queue is None:
+    if self.__queue is None:
       raise ValueError("Queue object is not set in configuration")
     if self.__name is None:
       raise ValueError("Monitor name is not set in configuration")
@@ -153,8 +154,11 @@ class GenericPromise(object):
     if self.__partition_folder is None:
       raise ValueError("Monitor partition folder is not set in configuration")
 
-  def getConfig(self):
-    return self.__config
+  def getConfig(self, key, default=None):
+    return self.__config.get(key, default)
+
+  def setConfig(self, key, value):
+    self.__config[key] = value
 
   def getTitle(self):
     return self.__title
@@ -271,7 +275,7 @@ class GenericPromise(object):
           if line != "":
             result = regex.match(line)
             if result is not None:
-              if result.groups()[0] < date_string:
+              if result.groups()[0] <= date_string:
                 break
               if not only_failure or \
                   (only_failure and result.groups()[1] == 'ERROR'):
@@ -288,21 +292,13 @@ class GenericPromise(object):
     line_list.reverse()
     return line_list
 
-  def defaultTest(self, latest_minute=2, failure_amount=1, exact_match=True,
-    is_anomaly=False):
+  def defaultTest(self, latest_minute=2, failure_amount=1, is_anomaly=False):
     """
       Test if the latest messages contain `failure_amount` failures.
 
       @param latest_minute: test the result from now to the latest X minutes in
         the past.
       @param failure_amount: fail is this amount of failure is found in result
-      @param exact_match: bool (True|False). 
-        True: 
-          only fail if the number of failure found is equal to
-          `failure_amount`, exactly `failure_amount` promise result are tested
-          starting from the most recent . 
-        False:
-          fail if at least one failure is found. 
       @param is_anomaly: Say if the result is an AnomalyResult of TestResult
     """
 
@@ -315,19 +311,19 @@ class GenericPromise(object):
     )
     result_size = len(latest_result_list)
     if result_size == 0:
-      return module(problem=False, message="No result!")
+      return module(problem=False, message="No result found!")
 
     i = 0
     failure_found = 0
     latest_result_list.reverse()
     # we test at most the `failure_amount` latest results
-    while i < result_size and i < failure_amount:
+    while i < result_size and failure_found < failure_amount:
       if latest_result_list[i][1] == 'ERROR':
         failure_found += 1
         problem = True
       i += 1
 
-    if exact_match and failure_found != failure_amount:
+    if failure_found != failure_amount:
       return module(problem=False, message=latest_result_list[0][2])
     return module(problem=problem, message=latest_result_list[0][2])
 
@@ -354,7 +350,7 @@ class GenericPromise(object):
     except Exception, e:
       # log the result
       self.logger.error(str(e))
-    if self.__check_anomaly:
+    if self.getConfig('check-anomaly', False):
       # run sense, anomaly
       try:
         result = self.anomaly()
@@ -362,8 +358,9 @@ class GenericPromise(object):
           raise ValueError("Promise anomaly method returned 'None'")
       except Exception, e:
         result = AnomalyResult(problem=True, message=str(e))
-      if result.hasFailed() and can_bang:
-        self.__bang("Promise %s is failing" % self.__title)
+      else:
+        if result.hasFailed() and can_bang:
+          self.__bang("Promise %s is failing" % self.__title)
     else:
       # run sense, test
       try:
@@ -377,8 +374,7 @@ class GenericPromise(object):
       self.__logger_buffer.close()
 
     # send the result of this promise
-    # should not raise Queue.Full exception as limit is not set to constructor
-    self.queue.put(PromiseQueueResult(
+    self.__queue.put(PromiseQueueResult(
         path=self.__promise_path,
         name=self.__name,
         title=self.__title,
@@ -395,6 +391,7 @@ class PromiseWrapper(GenericPromise):
 
   def __init__(self, config):
     GenericPromise.__init__(self, config)
+    self.setConfig("check-anomaly", False)
     self.setPeriodicity(minute=2)
 
   @staticmethod
@@ -550,11 +547,13 @@ class PromiseLauncher(object):
     if logger is None:
       self.logger = logging.getLogger(__name__)
       self.logger.setLevel(logging.DEBUG if self.debug else logging.INFO)
-      handler = logging.StreamHandler()
-      handler.setFormatter(
-        logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-      )
-      self.logger.addHandler(handler)
+      if len(self.logger.handlers) == 0 or \
+          not isinstance(self.logger.handlers[0], logging.StreamHandler):
+        handler = logging.StreamHandler()
+        handler.setFormatter(
+          logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        )
+        self.logger.addHandler(handler)
     else:
       self.logger = logger
 
@@ -580,8 +579,7 @@ class PromiseLauncher(object):
         try:
           latest_timestamp = float(f.read())
           current_timediff = (time.time() - latest_timestamp) / 60.0
-          margin_error = 0.15 # 0.15 seconds less is accepted
-          if current_timediff + margin_error >= periodicity:
+          if current_timediff >= periodicity:
             return True
           self.logger.debug("Skip Promise %r. periodicity=%s, time_diff=%s" % (
               promise_name, periodicity, current_timediff))
@@ -679,10 +677,10 @@ class PromiseLauncher(object):
         # process is gone
         pass
     for current_increment in range(0, increment_limit):
-      execution_time = current_increment * sleep_time
+      execution_time = (current_increment + 1) * sleep_time
       if not promise_process.is_alive():
         try:
-          queue_item = self.queue_result.get(False, 2)
+          queue_item = self.queue_result.get(True, 1)
         except Queue.Empty:
           # no result found in process result Queue
           if self.save_method is None:
@@ -751,6 +749,7 @@ class PromiseLauncher(object):
       'partition-folder': self.partition_folder,
       'debug': self.debug,
       'slapgrid-mode': self.slapgrid_mode,
+      'check-anomaly': self.check_anomaly,
       'master-url': self.master_url,
       'partition-cert': self.partition_cert,
       'partition-key': self.partition_key,
@@ -781,13 +780,20 @@ class PromiseLauncher(object):
         promise_list.append((promise_name,
                              self._loadPromiseModule(promise_name)))
 
-      for promise in promise_list:
+      for name, module in promise_list:
+        promise_path = os.path.join(self.promise_folder, name)
         config = {
-          'path': os.path.join(self.promise_folder, promise[0]),
-          'name': promise[0]
+          'path': promise_path,
+          'name': name
         }
+        if module.__file__ != promise_path:
+          # cached module need to be updated
+          module = reload(module)
+        #if os.path.getmtime(promise_path) > os.path.getmtime('%sc' % promise_path):
+        #  module = reload(module)
+
         config.update(base_config)
-        self._launchPromise(promise_name, config, promise[1])
+        self._launchPromise(name, config, module)
 
     if not self.run_only_promise_list and os.path.exists(self.old_promise_folder) \
         and os.path.isdir(self.old_promise_folder):