grid.promise: on promise timeout fail only if the problem is occurring a second time

If a promise timeout or return no message, return promise result with
error if the problem is happening for a second time or if there was
already an error. The goal is to reduce temporary failure.

slapos/grid/promise/__init__.py

@@ -315,12 +315,19 @@ class PromiseLauncher(object):
       mkdir_p(self.promise_output_dir)
       self._updateFolderOwner()
 
-  def _getErrorPromiseResult(self, promise_process, promise_name, promise_path,
+  def _generatePromiseResult(self, promise_process, promise_name, promise_path,
       message, execution_time=0):
+    problem = False
+    result = self._loadPromiseResult(promise_process.getPromiseTitle())
+    if result is not None and (result.item.hasFailed() or
+        'error' in result.item.message.lower()):
+      # generate failure if latest promise result was error
+      # If a promise timeout it will return failure if the timeout occur again
+      problem = True
     if self.check_anomaly:
-      result = AnomalyResult(problem=True, message=message)
+      result = AnomalyResult(problem=problem, message=message)
     else:
-      result = TestResult(problem=True, message=message)
+      result = TestResult(problem=problem, message=message)
     return PromiseQueueResult(
       item=result,
       path=promise_path,
@@ -469,8 +476,8 @@ class PromiseLauncher(object):
         self.logger.info("Killing process %s..." % promise_name)
         killProcessTree(promise_process.pid, self.logger)
 
-      message = 'Promise timed out after %s seconds' % self.promise_timeout
-      queue_item = self._getErrorPromiseResult(
+      message = 'Error: Promise timed out after %s seconds' % self.promise_timeout
+      queue_item = self._generatePromiseResult(
         promise_process,
         promise_name=promise_name,
         promise_path=promise_path,
@@ -479,11 +486,11 @@ class PromiseLauncher(object):
       )
 
     if queue_item is None:
-      queue_item = self._getErrorPromiseResult(
+      queue_item = self._generatePromiseResult(
         promise_process,
         promise_name=promise_name,
         promise_path=promise_path,
-        message="No output returned by the promise",
+        message="Error: No output returned by the promise",
         execution_time=execution_time
       )
 

slapos/tests/promise.py

@@ -912,9 +912,54 @@ exit 0
     self.launcher.run()
     self.assertEquals(self.counter, 2)
 
+  def writeLatestPromiseResult(self):
+    state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME, 'my_promise.status.json')
+    result_string = """{
+  "result": {
+    "failed": true,
+    "message": "error",
+    "date": "%s",
+    "type": "Test Result"
+  },
+  "path": "%s/my_promise.py",
+  "name": "my_promise.py",
+  "execution-time": 0.1,
+  "title": "my_promise"
+}""" % (datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S'), self.plugin_dir)
+
+    if not os.path.exists(os.path.dirname(state_file)):
+      os.makedirs(os.path.dirname(state_file))
+    with open(state_file, 'w') as f:
+      f.write(result_string)
+
+  def writeLatestBashPromiseResult(self, name="my_bash_promise"):
+    state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME,
+                              '%s.status.json' % name)
+    result_string = """{
+  "result": {
+    "failed": true,
+    "message": "error",
+    "date": "%(date)s",
+    "type": "Test Result"
+  },
+  "path": "%(folder)s/%(name)s",
+  "name": "%(name)s",
+  "execution-time": 0.1,
+  "title": "%(name)s"
+}""" % {'date': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S'),
+        'folder': self.legacy_promise_dir,
+        'name': name}
+
+    if not os.path.exists(os.path.dirname(state_file)):
+      os.makedirs(os.path.dirname(state_file))
+    with open(state_file, 'w') as f:
+      f.write(result_string)
+
   def test_runpromise_will_timeout(self):
     self.called = False
     promise_name = 'my_promise.py'
+    self.writeLatestPromiseResult()
+
     def test_method(result):
       self.called = True
       self.assertTrue(isinstance(result, PromiseQueueResult))
@@ -937,6 +982,7 @@ exit 0
   def test_runpromise_wrapped_will_timeout(self):
     promise_name = "my_bash_promise"
     promise_path = os.path.join(self.legacy_promise_dir, promise_name)
+    self.writeLatestBashPromiseResult()
     self.called = False
     with open(promise_path, 'w') as f:
       f.write("""#!/bin/bash
@@ -968,6 +1014,8 @@ echo "success"
     first_promise_path = os.path.join(self.legacy_promise_dir, first_promise)
     second_promise = "my_second_bash_promise"
     second_promise_path = os.path.join(self.legacy_promise_dir, second_promise)
+    self.writeLatestBashPromiseResult(first_promise)
+    self.writeLatestBashPromiseResult(second_promise)
 
     def createPromise(promise_path):
       with open(promise_path, 'w') as f:
@@ -986,6 +1034,26 @@ exit 1
     with self.assertRaises(PromiseError):
       self.launcher.run()
 
+  def test_runpromise_fail_if_timeout_twice(self):
+    promise_name = 'my_promise.py'
+
+    self.configureLauncher(timeout=1)
+    self.generatePromiseScript(promise_name, success=True, content="""import time
+    time.sleep(20)""", periodicity=0.01)
+
+    # timeout for the first time, no raise
+    self.launcher.run()
+
+    # run promise will timeout and raise
+    time.sleep(1)
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+
+    # run promise will continue to raise
+    time.sleep(1)
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+
 class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
 
   def initialisePromise(self, promise_content="", success=True, timeout=60):
@@ -1410,3 +1478,4 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
 
 if __name__ == '__main__':
   unittest.main()
+