grid.promise: return previous execution result if promise is skipped because of periodicity

slapos/grid/promise/__init__.py

@@ -44,8 +44,7 @@ from slapos.grid.utils import dropPrivileges
 from slapos.grid.promise import interface
 from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult,
                                          AnomalyResult, TestResult,
-                                         PROMISE_RESULT_FOLDER_NAME,
-                                         PROMISE_STATE_FOLDER_NAME)
+                                         PROMISE_RESULT_FOLDER_NAME)
 from slapos.grid.promise.wrapper import WrapPromise
 
 
@@ -183,6 +182,13 @@ class PromiseLauncher(object):
     self.queue_result = MQueue()
     self.bang_called = False
 
+    self.promise_output_dir = os.path.join(
+      self.partition_folder,
+      PROMISE_RESULT_FOLDER_NAME
+    )
+    if not os.path.exists(self.promise_output_dir):
+      mkdir_p(self.promise_output_dir)
+
   def _loadPromiseModule(self, promise_name):
     """Load a promise from promises directory."""
 
@@ -222,34 +228,55 @@ class PromiseLauncher(object):
       self.logger.error('Bad result: %s is not type of PromiseQueueResult...' % result)
       return
 
-    promise_output_dir = os.path.join(
-      self.partition_folder,
-      PROMISE_RESULT_FOLDER_NAME
-    )
     promise_output_file = os.path.join(
-      promise_output_dir,
+      self.promise_output_dir,
       "%s.status.json" % result.title
     )
     promise_tmp_file = '%s.tmp' % promise_output_file
 
-    if not os.path.exists(promise_output_dir):
-      mkdir_p(promise_output_dir)
-
     with open(promise_tmp_file, "w") as outputfile:
       json.dump(result.serialize(), outputfile)
     os.rename(promise_tmp_file, promise_output_file)
 
+  def _loadPromiseResult(self, promise_title):
+    promise_output_file = os.path.join(
+      self.promise_output_dir,
+      "%s.status.json" % promise_title
+    )
+    result = None
+    if os.path.exists(promise_output_file):
+      with open(promise_output_file) as f:
+        try:
+          result = PromiseQueueResult()
+          result.load(json.loads(f.read()))
+        except ValueError, e:
+          result = None
+          self.logger.warn('Bad promise JSON result at %r: %s' % (
+            promise_output_file,
+            e
+          ))
+    return result
+
   def _launchPromise(self, promise_name, argument_dict, promise_module=None):
     """
-      Launch the promise and save the result if `self.save_method` is not None
-        If no save method is set, raise PromiseError in case of failure
+      Launch the promise and save the result. If promise_module is None,
+      the promise will be run with the promise process wap module.
+
+      If the promise periodicity doesn't match, the previous promise result is
+      checked.
     """
+    self.logger.info("Checking promise %s..." % promise_name)
     try:
       if promise_module is None:
         promise_instance = WrapPromise(argument_dict)
       else:
         promise_instance = promise_module.RunPromise(argument_dict)
       if not self.force and not promise_instance.isPeriodicityMatch():
+        result = self._loadPromiseResult(promise_instance.getTitle())
+        if result is not None:
+          if result.item.hasFailed():
+            self.logger.error(result.item.message)
+            return True
         return False
       promise_instance.setPromiseRunTimestamp()
     except Exception:
@@ -268,7 +295,6 @@ class PromiseLauncher(object):
       logger=self.logger
     )
 
-    self.logger.info("Checking promise %s..." % promise_name)
     # set deamon to True, so promise process will be terminated if parent exit
     promise_process.daemon = True
     promise_process.start()

slapos/grid/promise/generic.py

@@ -54,8 +54,8 @@ class BaseResult(object):
   def hasFailed(self):
     return self.__problem
 
-  @property
-  def type(self):
+  @staticmethod
+  def type():
     return "Base Result"
 
   @property
@@ -68,19 +68,20 @@ class BaseResult(object):
 
 class TestResult(BaseResult):
 
-  @property
-  def type(self):
+  @staticmethod
+  def type():
     return "Test Result"
 
 class AnomalyResult(BaseResult):
 
-  @property
-  def type(self):
+  @staticmethod
+  def type():
     return "Anomaly Result"
 
 class PromiseQueueResult(object):
 
-  def __init__(self, path, name, title, item, execution_time=0):
+  def __init__(self, path=None, name=None, title=None,
+               item=None, execution_time=0):
     self.path = path
     self.name = name
     self.item = item
@@ -94,13 +95,32 @@ class PromiseQueueResult(object):
       'path': self.path,
       'execution-time': self.execution_time,
       'result': {
-        'type': self.item.type,
+        'type': self.item.type(),
         'failed': self.item.hasFailed(),
         'date': self.item.date.strftime('%Y-%m-%dT%H:%M:%S'),
         'message': self.item.message
       }
     }
 
+  def load(self, data):
+    if data['result']['type'] == AnomalyResult.type():
+      self.item = AnomalyResult(
+        problem=data['result']['failed'],
+        message=data['result']['message'],
+        date=datetime.strptime(data['result']['date'], '%Y-%m-%dT%H:%M:%S'))
+    elif data['result']['type'] == TestResult.type():
+      self.item = TestResult(
+        problem=data['result']['failed'],
+        message=data['result']['message'],
+        date=datetime.strptime(data['result']['date'], '%Y-%m-%dT%H:%M:%S'))
+    else:
+      raise ValueError('Unknown result type: %r' % data['result']['type'])
+
+    self.title = data['title']
+    self.name = data['name']
+    self.path = data['path']
+    self.execution_time = data['execution-time']
+
 class GenericPromise(object):
 
   # Abstract class

slapos/tests/promise.py

@@ -533,6 +533,112 @@ class RunPromise(GenericPromise):
     self.launcher.run()
     self.assertEquals(self.counter, 2)
 
+  def test_runpromise_with_periodicity_result_failed(self):
+    first_promise = 'my_first_promise.py'
+    second_promise = 'my_second_promise.py'
+    first_state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME,
+                                    'my_first_promise.status.json')
+    second_state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME,
+                                     'my_second_promise.status.json')
+
+    self.configureLauncher()
+    # ~2 seconds
+    self.generatePromiseScript(first_promise, success=True, periodicity=0.03)
+    # ~3 seconds
+    self.generatePromiseScript(second_promise, success=False, periodicity=0.05)
+
+    with self.assertRaises(PromiseError) as exc:
+      self.launcher.run()
+    self.assertEquals(exc.exception.message, 'Promise %r failed.' % second_promise)
+
+    self.assertTrue(os.path.exists(first_state_file))
+    self.assertTrue(os.path.exists(second_state_file))
+    first_result = json.load(open(first_state_file))
+    second_result = json.load(open(second_state_file))
+    self.assertEquals(first_result['name'], first_promise)
+    self.assertEquals(second_result['name'], second_promise)
+    first_date = first_result['result']['date']
+    second_date = second_result['result']['date']
+
+    self.configureLauncher()
+    time.sleep(2)
+    with self.assertRaises(PromiseError) as exc:
+      self.launcher.run() # only my_first_promise will run but second_promise still failing
+    self.assertEquals(exc.exception.message, 'Promise %r failed.' % second_promise)
+
+    first_result = json.load(open(first_state_file))
+    second_result = json.load(open(second_state_file))
+    self.assertNotEquals(first_result['result']['date'], first_date)
+    self.assertEquals(second_result['result']['date'], second_date)
+    first_date = first_result['result']['date']
+
+    time.sleep(3)
+    self.configureLauncher()
+    with self.assertRaises(PromiseError) as exc:
+      self.launcher.run()
+    self.assertEquals(exc.exception.message, 'Promise %r failed.' % second_promise)
+
+    first_result = json.load(open(first_state_file))
+    second_result = json.load(open(second_state_file))
+    self.assertNotEquals(first_result['result']['date'], first_date)
+    self.assertNotEquals(second_result['result']['date'], second_date)
+
+  def test_runpromise_with_periodicity_result_failed_and_ok(self):
+    first_promise = 'my_first_promise.py'
+    second_promise = 'my_second_promise.py'
+    first_state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME,
+                                    'my_first_promise.status.json')
+    second_state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME,
+                                     'my_second_promise.status.json')
+
+    self.configureLauncher()
+    # ~2 seconds
+    self.generatePromiseScript(first_promise, success=True, periodicity=0.03)
+    # ~3 seconds
+    self.generatePromiseScript(second_promise, success=False, periodicity=0.05)
+
+    with self.assertRaises(PromiseError) as exc:
+      self.launcher.run()
+    self.assertEquals(exc.exception.message, 'Promise %r failed.' % second_promise)
+
+    self.assertTrue(os.path.exists(first_state_file))
+    self.assertTrue(os.path.exists(second_state_file))
+    first_result = json.load(open(first_state_file))
+    second_result = json.load(open(second_state_file))
+    self.assertEquals(first_result['name'], first_promise)
+    self.assertEquals(second_result['name'], second_promise)
+    first_date = first_result['result']['date']
+    second_date = second_result['result']['date']
+
+    self.configureLauncher()
+    time.sleep(2)
+    with self.assertRaises(PromiseError) as exc:
+      self.launcher.run() # only my_first_promise will run but second_promise still failing
+    self.assertEquals(exc.exception.message, 'Promise %r failed.' % second_promise)
+
+    first_result = json.load(open(first_state_file))
+    second_result = json.load(open(second_state_file))
+    self.assertNotEquals(first_result['result']['date'], first_date)
+    self.assertEquals(second_result['result']['date'], second_date)
+    first_date = first_result['result']['date']
+    second_date = second_result['result']['date']
+
+    time.sleep(4)
+    if "my_second_promise" in sys.modules:
+      # force to reload the module without rerun python
+      os.system('rm %s/*.pyc' % self.plugin_dir)
+      del sys.modules["my_second_promise"]
+
+    # second_promise is now success
+    self.generatePromiseScript(second_promise, success=True, periodicity=0.05)
+    self.configureLauncher()
+    self.launcher.run() # now all succeed
+
+    first_result = json.load(open(first_state_file))
+    second_result = json.load(open(second_state_file))
+    self.assertNotEquals(first_result['result']['date'], first_date)
+    self.assertNotEquals(second_result['result']['date'], second_date)
+
   def test_runpromise_force(self):
     first_promise = 'my_first_promise.py'
     second_promise = 'my_second_promise.py'