monitor: reimplement monitor following the new promise design in slapgrid

promise are run by slapgrid now, monitor will collect result and generate final files
monitor runpromise script can be used to test a promise (which is implemented on the new promise design)
remove all generated script and logrotate code from monitor.py, they will be added in buildout

setup.py

@@ -52,6 +52,7 @@ setup(name=name,
         'PyRSS2Gen',
         'dnspython',
         'requests',
+        'jsonschema',
       ] + additional_install_requires,
       extras_require = {
         'lampconfigure':  ["mysqlclient"], #needed for MySQL Database access

slapos/monitor/collect.py

@@ -69,7 +69,7 @@ class ResourceCollect:
     assert os.path.exists(db_path)
     if db_path.endswith("collector.db"):
       db_path = db_path[:-len("collector.db")]
-    self.db = Database(db_path)
+    self.db = Database(db_path, create=False)
     self.consumption_utils = ConsumptionReportBase(self.db)
 
   def has_table(self, name):
@@ -255,7 +255,18 @@ def main():
     initMemoryDataFile(mem_file)
     initIODataFile(io_file)
     with open(status_file, "w") as status_file:
-      status_file.write('{"cpu_time": 0, "cpu_percent": 0, "memory_rss": 0, "memory_percent": 0, "io_rw_counter": 0, "date": "", "total_process": 0, "disk_used": 0, "io_cycles_counter": 0, "cpu_num_threads": 0}')
+      status_file.write(json.dumps({
+        "cpu_time": 0,
+        "cpu_percent": 0,
+        "memory_rss": 0,
+        "memory_percent": 0,
+        "io_rw_counter": 0,
+        "date": "",
+        "total_process": 0,
+        "disk_used": 0,
+        "io_cycles_counter": 0,
+        "cpu_num_threads": 0
+      }))
     with open(resource_file, "w") as resource_file:
       resource_file.write('[]')
     exit(1)

slapos/monitor/globalstate.py

@@ -14,7 +14,7 @@ import PyRSS2Gen
 def getKey(item):
   return item.source.name
 
-class monitorFeed(object):
+class MonitorFeed(object):
 
   def __init__(self, instance_name, hosting_name,
       public_url, private_url, feed_url):
@@ -26,23 +26,24 @@ class monitorFeed(object):
     self.private_url = private_url
     self.feed_url = feed_url
 
-  def appendItem(self, item_dict):
-    event_time = datetime.utcfromtimestamp(item_dict['change-time'])
-    description = item_dict.get('message', '')
+  def appendItem(self, item_dict, has_string=""):
+    event_date = item_dict['result']['change-date']
+    report_date = item_dict['result']['date']
+    description = item_dict['result'].get('message', '')
     rss_item = PyRSS2Gen.RSSItem(
       categories = [item_dict['status']],
       source = PyRSS2Gen.Source(item_dict['title'], self.public_url),
       title = '[%s] %s' % (item_dict['status'], item_dict['title']),
-      description = "%s: %s\n\n%s" % (event_time, item_dict['status'], description),
+      description = "\n%s" % (description,),
       link = self.private_url,
-      pubDate = event_time,
-      guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s, %s" % (self.hosting_name,
-                              item_dict['title'], event_time)),
+      pubDate = event_date,
+      guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s, %s, %s" % (self.hosting_name,
+                              item_dict['title'], has_string, event_date)),
                             isPermaLink=False)
     )
     self.rss_item_list.append(rss_item)
 
-  def genrss(self, output_file):
+  def generateRSS(self, output_file):
     ### Build the rss feed
     # try to keep the list in the same order
     sorted(self.rss_item_list, key=getKey)
@@ -57,12 +58,6 @@ class monitorFeed(object):
     with open(output_file, 'w') as frss:
       frss.write(rss_feed.to_xml())
 
-def softConfigGet(config, *args, **kwargs):
-  try:
-    return config.get(*args, **kwargs)
-  except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
-    return ""
-
 def generateStatisticsData(stat_file_path, content):
   # csv document for success/error statictics
   if not os.path.exists(stat_file_path) or os.stat(stat_file_path).st_size == 0:
@@ -89,63 +84,148 @@ def generateStatisticsData(stat_file_path, content):
       fstat.seek(position)
       fstat.write('%s}' % ',"{}"]'.format(current_state))
 
-def run(args_list):
-  monitor_file, instance_file = args_list
-
-  monitor_config = ConfigParser.ConfigParser()
-  monitor_config.read(monitor_file)
-
-  base_folder = monitor_config.get('monitor', 'private-folder')
-  status_folder = monitor_config.get('monitor', 'public-folder')
-  base_url = monitor_config.get('monitor', 'base-url')
-  related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
-  statistic_folder = os.path.join(base_folder, 'documents')
-  # need webdav to update parameters
-  parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
-  feed_output = os.path.join(status_folder, 'feed')
-  public_url = "%s/share/public/" % base_url
-  private_url = "%s/share/private/" % base_url
-  feed_url = "%s/public/feed" % base_url
+def writeDocumentList(folder_path):
+  # Save document list in a file called _document_list
+  public_document_list = [os.path.splitext(file)[0]
+                for file in os.listdir(folder_path) if file.endswith('.json')]
 
-  error = success = 0
-  status = 'OK'
-  global_state_file = os.path.join(base_folder, 'monitor.global.json')
-  public_state_file = os.path.join(status_folder, 'monitor.global.json')
+  with open(os.path.join(folder_path, '_document_list'), 'w') as lfile:
+    lfile.write('\n'.join(public_document_list))
 
+def generateMonitoringData(config, public_folder, private_folder, public_url,
+    private_url, feed_url):
+  feed_output = os.path.join(public_folder, 'feed')
   # search for all status files
-  file_list = filter(os.path.isfile,
-      glob.glob("%s/*.status.json" % status_folder)
+  file_list = filter(
+    os.path.isfile,
+    glob.glob("%s/promise/*.status.json" % public_folder)
   )
-  if os.path.exists(instance_file):
-    config = ConfigParser.ConfigParser()
-    config.read(instance_file)
-  else:
-    raise Exception("Cannot read instance configuration at %s" % instance_file)
 
-  report_date = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
-  monitor_feed = monitorFeed(
-    config.get('instance', 'name'),
-    config.get('instance', 'root-name'),
+  promises_status_file = os.path.join(private_folder, '_promise_status')
+  previous_state_dict = {}
+  new_state_dict = {}
+  error = success = 0
+  monitor_feed = MonitorFeed(
+    config.get('monitor', 'title'),
+    config.get('monitor', 'root-title'),
     public_url,
     private_url,
     feed_url)
+
+  if os.path.exists(promises_status_file):
+    with open(promises_status_file) as f:
+      try:
+        previous_state_dict = json.loads(f.read())
+      except ValueError:
+        pass
+
   for file in file_list:
     try:
       with open(file, 'r') as temp_file:
         tmp_json = json.loads(temp_file.read())
-    except ValueError:
-      # bad json file ?
-      continue
-    if tmp_json['status'] == 'ERROR':
+
+      if tmp_json['result']['failed']:
+        promise_status = "ERROR"
         error += 1
-    elif tmp_json['status'] == 'OK':
+      else:
+        promise_status = "OK"
         success += 1
-    monitor_feed.appendItem(tmp_json)
+      tmp_json['result']['change-date'] = tmp_json['result']['date']
+      if previous_state_dict.has_key(tmp_json['name']):
+        status, change_date, _ = previous_state_dict[tmp_json['name']]
+        if promise_status == status:
+          tmp_json['result']['change-date'] = change_date
+
+      tmp_json['status'] = promise_status
+      message_hash = hashlib.md5(tmp_json['result'].get('message', '')).hexdigest()
+      new_state_dict[tmp_json['name']] = [
+        promise_status,
+        tmp_json['result']['change-date'],
+        message_hash
+      ]
+      monitor_feed.appendItem(tmp_json, message_hash)
+      savePromiseHistory(
+        tmp_json['title'],
+        tmp_json,
+        previous_state_dict.get(tmp_json['name']),
+        public_folder
+      )
+    except ValueError, e:
+      # bad json file
+      print "ERROR: Bad json file at: %s\n%s" % (file, str(e))
+      continue
+
+  with open(promises_status_file, "w") as f:
+    json.dump(new_state_dict, f)
 
+  monitor_feed.generateRSS(feed_output)
+  return error, success
+
+def savePromiseHistory(promise_name, state_dict, previous_state_list,
+    history_folder):
+  if not os.path.exists(history_folder) and os.path.isdir(history_folder):
+    self.logger.warning('Bad promise history folder, history is not saved...')
+    return
+
+  history_file = os.path.join(
+    history_folder,
+    '%s.history.json' % promise_name
+  )
+
+  # Remove useless informations
+  result = state_dict.pop('result')
+  state_dict.update(result)
+  state_dict.pop('path', '')
+  state_dict.pop('type', '')
+  if not os.path.exists(history_file) or not os.stat(history_file).st_size:
+    with open(history_file, 'w') as f:
+      data_dict = {
+        "date": time.time(),
+        "data": [state_dict]
+      }
+      json.dump(data_dict, f)
+  else:
+    if previous_state_list is not None:
+      _, change_date, checksum = previous_state_list
+      current_sum = hashlib.md5(state_dict.get('message', '')).hexdigest()
+      if state_dict['change-date'] == change_date and \
+          current_sum == checksum:
+        # Only save the changes and not the same info
+        return
+
+    state_dict.pop('title', '')
+    state_dict.pop('name', '')
+    with open (history_file, mode="r+") as f:
+      f.seek(0,2)
+      f.seek(f.tell() -2)
+      f.write('%s}' % ',{}]'.format(json.dumps(state_dict)))
+
+def run(monitor_conf_file):
+
+  config = ConfigParser.ConfigParser()
+  config.read(monitor_conf_file)
+
+  base_folder = config.get('monitor', 'private-folder')
+  status_folder = config.get('monitor', 'public-folder')
+  base_url = config.get('monitor', 'base-url')
+  related_monitor_list = config.get("monitor", "monitor-url-list").split()
+  statistic_folder = os.path.join(base_folder, 'documents')
+  # need webdav to update parameters
+  parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
+
+  public_url = "%s/share/public/" % base_url
+  private_url = "%s/share/private/" % base_url
+  feed_url = "%s/public/feed" % base_url
+  status = 'OK'
+  global_state_file = os.path.join(base_folder, 'monitor.global.json')
+  public_state_file = os.path.join(status_folder, 'monitor.global.json')
+  report_date = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
+
+  error, success = generateMonitoringData(config, status_folder, base_folder,
+                                          public_url, private_url, feed_url)
   if error:
     status = 'ERROR'
 
-  monitor_feed.genrss(feed_output)
   global_state_dict = dict(
     status=status,
     state={
@@ -157,7 +237,8 @@ def run(args_list):
     date=report_date,
     _links={"rss_url": {"href": feed_url},
             "public_url": {"href": public_url},
-            "private_url": {"href": private_url}
+            "private_url": {"href": private_url},
+            "related_monitor": []
           },
     data={'state': 'monitor_state.data',
           'process_state': 'monitor_process_resource.status',
@@ -165,25 +246,19 @@ def run(args_list):
           'memory_resource': 'monitor_resource_memory.data',
           'io_resource': 'monitor_resource_io.data',
           'monitor_process_state': 'monitor_resource.status'},
-    _embedded={},
+    title=config.get('monitor', 'title'),
+    specialise_title=config.get('monitor', 'root-title'),
+    aggregate_reference=config.get('promises', 'computer-id'),
+    ipv4=config.get('promises', 'ipv4'),
+    ipv6=config.get('promises', 'ipv6'),
+    software_release=config.get('promises', 'software-release'),
+    software_type=config.get('promises', 'software-type'),
+    partition_id=config.get('promises', 'partition-id'),
   )
 
-  instance_dict = {}
-  global_state_dict['title'] = config.get('instance', 'name')
-  global_state_dict['specialise_title'] = config.get('instance', 'root-name')
-  global_state_dict['aggregate_reference'] = config.get('instance', 'computer')
   if not global_state_dict['title']:
     global_state_dict['title'] = 'Instance Monitoring'
 
-  instance_dict['computer'] = config.get('instance', 'computer')
-  instance_dict['ipv4'] = config.get('instance', 'ipv4')
-  instance_dict['ipv6'] = config.get('instance', 'ipv6')
-  instance_dict['software-release'] = config.get('instance', 'software-release')
-  instance_dict['software-type'] = config.get('instance', 'software-type')
-  instance_dict['partition'] = config.get('instance', 'partition')
-
-  global_state_dict['_embedded'].update({'instance' : instance_dict})
-
   if related_monitor_list:
     global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/public" % url}
                           for url in related_monitor_list]
@@ -197,9 +272,9 @@ def run(args_list):
     status=status,
     date=report_date,
     _links={'monitor': {'href': '%s/share/private/' % base_url}},
-    title=global_state_dict.get('title', '')
+    title=global_state_dict.get('title', ''),
+    specialise_title=global_state_dict.get('specialise_title', ''),
   )
-  public_state_dict['specialise_title'] = global_state_dict.get('specialise_title', '')
   public_state_dict['_links']['related_monitor'] = global_state_dict['_links'].get('related_monitor', [])
 
   with open(global_state_file, 'w') as fglobal:
@@ -208,31 +283,20 @@ def run(args_list):
   with open(public_state_file, 'w') as fpglobal:
     fpglobal.write(json.dumps(public_state_dict))
 
-  # Save document list in a file called _document_list
-  public_document_list = [os.path.splitext(file)[0]
-                for file in os.listdir(status_folder) if file.endswith('.json')]
-  private_document_list = [os.path.splitext(file)[0]
-                  for file in os.listdir(base_folder) if file.endswith('.json')]
-  data_document_list = [os.path.splitext(file)[0]
-              for file in os.listdir(statistic_folder) if file.endswith('.json')]
-
-  with open(os.path.join(status_folder, '_document_list'), 'w') as lfile:
-    lfile.write('\n'.join(public_document_list))
-
-  with open(os.path.join(base_folder, '_document_list'), 'w') as lfile:
-    lfile.write('\n'.join(private_document_list))
-
-  with open(os.path.join(statistic_folder, '_document_list'), 'w') as lfile:
-    lfile.write('\n'.join(data_document_list))
+  # write list of files
+  writeDocumentList(status_folder)
+  writeDocumentList(base_folder)
+  writeDocumentList(statistic_folder)
 
   generateStatisticsData(
     os.path.join(statistic_folder, 'monitor_state.data.json'),
-    global_state_dict)
+    global_state_dict
+  )
 
   return 0
 
 def main():
-  if len(sys.argv) < 3:
-    print("Usage: %s <monitor_conf_path> <instance_conf_path>" % sys.argv[0])
+  if len(sys.argv) < 2:
+    print("Usage: %s <monitor_conf_path>" % sys.argv[0])
     sys.exit(2)
-  sys.exit(run(sys.argv[1:]))
+  sys.exit(run(sys.argv[1]))

slapos/monitor/monitor.py

@@ -36,7 +36,7 @@ def parseArguments():
   Parse arguments for monitor instance.
   """
   parser = argparse.ArgumentParser()
-  parser.add_argument('--config_file',
+  parser.add_argument('-c', '--config-file', required=True,
                       default='monitor.cfg',
                       help='Monitor Configuration file')
 
@@ -67,6 +67,7 @@ def createSymlink(source, destination):
 class Monitoring(object):
 
   def __init__(self, configuration_file):
+    self._config_file = configuration_file
     config = self.loadConfig([configuration_file])
 
     # Set Monitor variables
@@ -74,19 +75,11 @@ class Monitoring(object):
     self.root_title = config.get("monitor", "root-title")
     self.service_pid_folder = config.get("monitor", "service-pid-folder")
     self.crond_folder = config.get("monitor", "crond-folder")
-    self.logrotate_d = config.get("monitor", "logrotate-folder")
-    self.promise_runner = config.get("monitor", "promise-runner")
-    self.promise_folder = config.get("monitor", "promise-folder")
     self.public_folder = config.get("monitor", "public-folder")
     self.private_folder = config.get("monitor", "private-folder")
-    self.collector_db  = config.get("monitor", "collector-db")
-    self.collect_script = config.get("monitor", "collect-script")
-    self.statistic_script = config.get("monitor", "statistic-script")
     self.webdav_folder = config.get("monitor", "webdav-folder")
-    self.report_script_folder = config.get("monitor", "report-folder")
     self.webdav_url = '%s/share' % config.get("monitor", "base-url")
     self.public_url = '%s/public' % config.get("monitor", "base-url")
-    self.python = config.get("monitor", "python") or "python"
     self.public_path_list = config.get("monitor", "public-path-list").split()
     self.private_path_list = config.get("monitor", "private-path-list").split()
     self.monitor_url_list = config.get("monitor", "monitor-url-list").split()
@@ -94,25 +87,14 @@ class Monitoring(object):
     # Use this file to write knowledge0_cfg required by webrunner
     self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
     self.pid_file = config.get("monitor", "pid-file")
-    self.monitor_promise_folder = softConfigGet(config, "monitor",
-                                                "monitor-promise-folder")
-    self.promise_timeout_file = softConfigGet(config, "monitor",
-                                                "promises-timeout-file")
-    self.nice_command = softConfigGet(config, "monitor",
-                                                "nice-cmd")
+    self.promise_output_file = config.get("monitor", "promise-output-file")
+    self.promise_folder = config.get("promises", 'promise-folder')
+    self.legacy_promise_folder = config.get("promises", 'legacy-promise-folder')
+    self.promise_output = config.get("promises", 'output-folder')
 
     self.config_folder = os.path.join(self.private_folder, 'config')
-    self.report_folder = self.private_folder
-    self.data_folder = os.path.join(self.private_folder, 'documents')
-    self.log_folder = os.path.join(self.private_folder, 'monitor-log')
-
-    self.promise_output_file = config.get("monitor", "promise-output-file")
+    self.data_folder = config.get("monitor", "document-folder")
     self.bootstrap_is_ok = True
-    self.randomsleep = config.get("monitor", "randomsleep")
-
-    if self.nice_command:
-      # run monitor promises script with low priority
-      self.promise_runner = '%s %s' % (self.nice_command, self.promise_runner)
 
   def loadConfig(self, pathes, config=None):
     if config is None:
@@ -224,7 +206,7 @@ class Monitoring(object):
     success = False
     monitor_title = 'Unknown Instance'
     try:
-      # Timeout after 20 seconds to not stall on download
+      # Timeout after 20 seconds
       timeout = 20
       # XXX - working here with public url
       if hasattr(ssl, '_create_unverified_context'):
@@ -247,28 +229,6 @@ class Monitoring(object):
     self.bootstrap_is_ok = success
     return monitor_title
 
-  def getReportInfoFromFilename(self, filename):
-    splited_filename = filename.split('_every_')
-    possible_time_list = ['hour', 'minute']
-    if len(splited_filename) == 1:
-      return (filename, "* * * * *")
-
-    run_time = splited_filename[1].split('_')
-    report_name = splited_filename[0]
-    if len(run_time) != 2 or not run_time[1] in possible_time_list:
-      return (report_name, "* * * * *")
-
-    try:
-      value = int(run_time[0])
-    except ValueError:
-      print "Warning: Bad report filename: %s" % filename
-      return (report_name, "* * * * *")
-
-    if run_time[1] == 'hour':
-      return (report_name, "11 */%s * * *" % value)
-    if run_time[1] == 'minute':
-      return (report_name, "*/%s * * * *" % value)
-
   def configureFolders(self):
     # create symlinks from monitor.conf
     self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
@@ -278,19 +238,8 @@ class Monitoring(object):
     self.createSymlinksFromConfig(self.webdav_folder, [self.private_folder])
 
     config_jio_folder = os.path.join(self.config_folder, '.jio_documents')
-
-    mkdirAll(self.data_folder)
     mkdirAll(config_jio_folder)
 
-    # Cleanup private folder, remove files that should not be synced from private root dir
-    cleanup_file_list = glob.glob("%s/*.history.json" % self.private_folder)
-    cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.private_folder))
-    for file in cleanup_file_list:
-      try:
-        os.unlink(file)
-      except OSError:
-        print "failed to remove file %s. Ignoring..." % file
-
   def makeConfigurationFiles(self):
     config_folder = os.path.join(self.config_folder, '.jio_documents')
     parameter_config_file = os.path.join(config_folder, 'config.parameters.json')
@@ -353,113 +302,43 @@ class Monitoring(object):
     with open(output_file, 'w') as wfile:
       wfile.write(opml_content)
 
-  def generateLogrotateEntry(self, name, file_list, option_list):
-    """
-      Will add a new entry in logrotate.d folder. This can help to rotate data file daily
-    """
-    content = "%(logfiles)s {\n%(options)s\n}\n" % {
-                'logfiles': ' '.join(file_list),
-                'options': '\n'.join(option_list)
-              }
-    file_path = os.path.join(self.logrotate_d, name)
-    with open(file_path, 'w') as flog:
-      flog.write(content)
-
-  def generateReportCronEntries(self):
-    cron_line_list = []
-
-    report_name_list = [name.replace('.report.json', '')
-      for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
-
-    for filename in os.listdir(self.report_script_folder):
-      report_script = os.path.join(self.report_script_folder, filename)
-      if os.path.isfile(report_script) and os.access(report_script, os.X_OK):
-        report_name, frequency = self.getReportInfoFromFilename(filename)
-        report_json_name = "%s.report.json" % report_name
-
-        report_cmd_line = [
-          frequency,
-          self.promise_runner,
-          '--pid_path "%s"' % os.path.join(self.service_pid_folder,
-            "%s.pid" % filename),
-          '--output "%s"' % os.path.join(self.data_folder, report_json_name),
-          '--promise_script "%s"' % report_script,
-          '--promise_name "%s"' % report_name,
-          '--monitor_url "%s/private/"' % self.webdav_url, # XXX hardcoded,
-          '--history_folder "%s"' % self.data_folder,
-          '--instance_name "%s"' % self.title,
-          '--hosting_name "%s"' % self.root_title,
-          '--promise_type "report"',
-          '--log_file "%s.report.log"' % os.path.join(self.log_folder, report_name)]
-
-        cron_line_list.append(' '.join(report_cmd_line))
-
-      if report_name in report_name_list:
-        report_name_list.pop(report_name_list.index(report_name))
-
-    # cleanup removed report json result
-    if report_name_list != []:
-      for report_name in report_name_list:
-        result_path = os.path.join(self.report_folder, '%s.report.json' % report_name)
-        if os.path.exists(result_path):
+  def cleanupMonitorDeprecated(self):
+    # Monitor report feature is removed
+    cleanup_file_list = glob.glob("%s/*.history.json" % self.private_folder)
+    cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.private_folder))
+    cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.data_folder))
+    cleanup_file_list.extend(glob.glob("%s/*.history.json" % self.data_folder))
+    cleanup_file_list.extend(glob.glob("%s/*.status.json" % self.public_folder))
+    cleanup_file_list.append(self.crond_folder + "/monitor-reports")
+    cleanup_file_list.append(self.crond_folder + "/monitor-promises")
+
+    for file in cleanup_file_list:
       try:
-            os.unlink(result_path)
+        if os.path.exists(file):
+          os.unlink(file)
       except OSError, e:
-            print "Error: Failed to delete %s" % result_path, str(e)
-            pass
+        print "failed to remove file %s." % file, str(e)
 
-    with open(self.crond_folder + "/monitor-reports", "w") as freport:
-      freport.write("\n".join(cron_line_list))
-
-  def generateServiceCronEntries(self):
-    # XXX only if at least one configuration file is modified, then write in the cron
-
-    service_name_list = [name.replace('.status.json', '')
-      for name in os.listdir(self.public_folder) if name.endswith('.status.json')]
-
-    promise_cmd_line = [
-      "* * * * *",
-      self.randomsleep + " 60 &&", # Sleep between 1 to 60 seconds
-      self.promise_runner,
-      '--pid_path "%s"' % os.path.join(self.service_pid_folder,
-        "monitor-promises.pid"),
-      '--output "%s"' % self.public_folder,
-      '--promise_folder "%s"' % self.promise_folder,
-      '--timeout_file "%s"' % self.promise_timeout_file,
-      '--monitor_promise_folder "%s"' % self.monitor_promise_folder,
-      '--monitor_url "%s/private/"' % self.webdav_url, # XXX hardcoded,
-      '--history_folder "%s"' % self.public_folder,
-      '--instance_name "%s"' % self.title,
-      '--hosting_name "%s"' % self.root_title,
-      '--log_file "%s.log"' % os.path.join(self.log_folder,
-                                           self.title.replace(' ', '_'))]
-
-    registered_promise_list = os.listdir(self.promise_folder)
-    registered_promise_list.extend(os.listdir(self.monitor_promise_folder))
-    delete_promise_list = []
-    for service_name in service_name_list:
-      if not service_name in registered_promise_list:
-        delete_promise_list.append(service_name)
-
-    if delete_promise_list != []:
-      # XXX Some service was removed, delete his status file so monitor will not consider the status anymore
-      for service_name in delete_promise_list:
-        status_path = os.path.join(self.public_folder, '%s.status.json' % service_name)
+    # cleanup result of promises that was removed
+    promise_list = os.listdir(self.legacy_promise_folder)
+    for name in os.listdir(self.promise_folder):
+      if name.startswith('__init__'):
+        continue
+      promise_list.append(os.path.splitext(name)[0])
+    for name in os.listdir(self.promise_output):
+      if not name.endswith('.status.json'):
+        continue
+      try:
+        position = promise_list.index(name.replace('.status.json', ''))
+      except ValueError:
+        status_path = os.path.join(self.promise_output, name)
         if os.path.exists(status_path):
           try:
             os.unlink(status_path)
           except OSError, e:
             print "Error: Failed to delete %s" % status_path, str(e)
-            pass
-
-    with open(self.crond_folder + "/monitor-promises", "w") as fp:
-      fp.write(' '.join(promise_cmd_line))
-
-  def addCronEntry(self, name, frequency, command):
-    entry_line = '%s %s' % (frequency, command)
-    cron_entry_file = os.path.join(self.crond_folder, name)
-    with open(cron_entry_file, "w") as cronf:
-      cronf.write(entry_line)
+      else:
+        promise_list.pop(position)
 
   def bootstrapMonitor(self):
 
@@ -476,57 +355,12 @@ class Monitoring(object):
     self.generateOpmlFile(self.monitor_url_list,
       os.path.join(self.public_folder, 'feeds'))
 
-    # put promises to a cron file
-    self.generateServiceCronEntries()
-
-    # put report script to cron
-    self.generateReportCronEntries()
+    # cleanup deprecated entries
+    self.cleanupMonitorDeprecated()
 
     # Generate parameters files and scripts
     self.makeConfigurationFiles()
 
-    # Rotate monitor data files
-    option_list = [
-      'daily', 'nocreate', 'olddir %s' % self.data_folder, 'rotate 5',
-      'nocompress', 'missingok', 'extension .json', 'dateext',
-      'dateformat -%Y-%m-%d', 'notifempty'
-    ]
-    file_list = [
-      "%s/*.data.json" % self.private_folder,
-      "%s/*.data.json" % self.data_folder]
-    self.generateLogrotateEntry('monitor.data', file_list, option_list)
-
-    # Rotate public history status file, delete data of previous days
-    option_list = [
-      'daily', 'nocreate', 'rotate 0',
-      'nocompress', 'notifempty', 'prerotate',
-      '   %s --history_folder %s' % (self.statistic_script, self.public_folder), 
-      'endscript'
-    ]
-    file_list = ["%s/*.history.json" % self.public_folder]
-    self.generateLogrotateEntry('monitor.service.status', file_list, option_list)
-
-    # Rotate monitor log files
-    option_list = [
-      'daily', 'dateext', 'create', 'rotate 180',
-      'compress', 'delaycompress', 'notifempty',
-      'missingok'
-    ]
-    file_list = ["%s/*.log" % self.log_folder]
-    self.generateLogrotateEntry('monitor.promise.log', file_list, option_list)
-
-    # Add cron entry for SlapOS Collect
-    command = self.randomsleep + " 60 && " # Random sleep between 1 to 60 seconds
-    if self.nice_command:
-      # run monitor collect with low priority
-      command += '%s ' % self.nice_command
-    command += "%s --output_folder %s --collector_db %s --pid_file %s" % (
-        self.collect_script,
-        self.data_folder,
-        self.collector_db,
-        os.path.join(self.service_pid_folder, "monitor-collect.pid"))
-    self.addCronEntry('monitor_collect', '* * * * *', command)
-
     # Write an empty file when monitor bootstrap went until the end
     if self.bootstrap_is_ok:
       with open(self.promise_output_file, 'w') as promise_file:

slapos/monitor/runpromise.py

@@ -13,147 +13,103 @@ import glob
 import argparse
 import traceback
 import logging
-from slapos.grid.utils import checkPromiseList as _checkPromiseList
+import ConfigParser
+from slapos.grid.promise import PromiseLauncher, PromiseQueueResult, PromiseError
+from slapos.grid.promise.generic import PROMISE_LOG_FOLDER_NAME
+from slapos.util import mkdir_p
 
 # Promise timeout after 20 seconds by default
 promise_timeout = 20
 
-def parseArguments():
+def getArgumentParser():
   """
   Parse arguments for monitor collector instance.
   """
   parser = argparse.ArgumentParser()
-  parser.add_argument('--pid_path',
+  parser.add_argument('-c', '--config', dest='config_file',
+                      help='The Path of configuration file to load.')
+  parser.add_argument('-p', '--partition-folder',
+                      help='Base path of the partition.')
+  parser.add_argument('-L', '--log-folder',
+                      help='Folder where promises will write logs.')
+  parser.add_argument('--console',
+                      help='Log to console too',
+                      default=False, action='store_true')
+  parser.add_argument('-a', '--check-anomaly',
+                      help='Enable check of anomaly on promises.',
+                      default=False, action='store_true')
+  parser.add_argument('-D', '--debug',
+                      help='Configure loggin in debug mode.',
+                      default=False, action='store_true')
+  parser.add_argument('--master-url',
+                      help='SlapOS Master Service URL.')
+  parser.add_argument('--partition-cert',
+                      help='Computer partition certificate file path.')
+  parser.add_argument('--partition-key',
+                      help='Computer partition key file path.')
+  parser.add_argument('--partition-id',
+                      help='Computer partition ID.')
+  parser.add_argument('--computer-id',
+                      help='Computer ID.')
+  parser.add_argument('--pid-path',
                       help='Path where the pid of this process will be writen.')
-  parser.add_argument('--output',
-                      help='The Path of file where Json result of this promise will be saved.')
-  parser.add_argument('--promise_script',
-                      help='Promise script to execute.')
-  parser.add_argument('--promise_folder',
-                      help='Folder where to find promises to execute. Hide --promise_script and --promise_name')
-  parser.add_argument('--monitor_promise_folder',
-                      help='Folder where to find Custom monitor promises to execute.')
-  parser.add_argument('--promise_name',
-                      help='Title to give to this promise.')
-  parser.add_argument('--timeout_file',
-                      default='',
-                      help='File containing Max timeout for each promise run.')
-  parser.add_argument('--promise_type',
-                      default='status',
-                      help='Type of promise to execute. [status, report].')
-  parser.add_argument('--monitor_url',
-                      help='Monitor Instance website URL.')
-  parser.add_argument('--history_folder',
-                      help='Path where old result file will be placed before generate a new json result file.')
-  parser.add_argument('--instance_name',
-                      default='UNKNOWN Software Instance',
-                      help='Software Instance name.')
-  parser.add_argument('--hosting_name',
-                      default='UNKNOWN Hosting Subscription',
-                      help='Hosting Subscription name.')
-  parser.add_argument('--log_file',
-                      help='Path of log file.')
+  parser.add_argument('--run-only',
+                      help='List of python promises to run separates by space.',
+                      default="")
+  parser.add_argument('-f', '--force',
+                      help='Force to run promise without consider the periodicity.',
+                      default=False, action='store_true')
+  parser.add_argument('--dry-run',
+                      help='Only run promise without save result.',
+                      default=False, action='store_true')
+  parser.add_argument('--promise-timeout', default=20, type=int,
+                      help='Maximum promise execution time.')
 
   return parser
 
 
-class RunPromise(object):
+class MonitorPromiseLauncher(object):
 
   def __init__(self, config_parser):
     self.config = config_parser
-    self.logger = logging.getLogger("RunPromise")
-    self.logger.setLevel(logging.DEBUG)
+    if self.config.config_file:
+      self._loadConfigFromFile(self.config.config_file)
+    self.logger = logging.getLogger("Monitor")
+    self.logger.setLevel(logging.DEBUG if self.config.debug else logging.INFO)
 
-    if not self.config.log_file:
+    if not self.config.log_folder or self.config.console:
       if len(self.logger.handlers) == 0 or \
           not isinstance(self.logger.handlers[0], logging.StreamHandler):
-        self.logger.addHandler(logging.StreamHandler())
-    else:
-      file_handler = logging.FileHandler(self.config.log_file)
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+        self.logger.addHandler(handler)
+    if self.config.log_folder:
+      log_file = os.path.join(self.config.log_folder,
+        '%s.log' % self.config.title.replace(' ', '_'))
+      file_handler = logging.FileHandler(log_file)
       file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
       self.logger.addHandler(file_handler)
 
-    self.promise_timeout = promise_timeout
-    if self.config.timeout_file and \
-            os.path.exists(self.config.timeout_file):
-      with open(self.config.timeout_file) as tf:
-        timeout = tf.read().strip()
-        if timeout.isdigit():
-          self.promise_timeout = int(timeout)
-        else:
-          self.logger.warning("%s it not a valid promise-timeout value" % timeout)
-
-  def runPromise(self):
 
-    if self.config.promise_folder:
-      # run all promises from the given folder in a synchronous way
-      return self.runPromiseSynchronous()
+  def _loadConfigFromFile(self, config_file):
+    config = ConfigParser.ConfigParser()
+    config.read([config_file])
+    known_key_list = ['partition-cert', 'partition-key', 'partition-id',
+                      'pid-path', 'computer-id', 'check-anomaly',
+                      'master-url', 'partition-folder', 'promise-timeout']
  
-    return self.runSinglePromise()
+    if config.has_section('promises'):
+      for key, value in config.items('promises'):
+        key_add = key.replace('-', '_')
+        if key in known_key_list and \
+            getattr(self.config, key_add, None) is None:
+          setattr(self.config, key_add, value)
 
-  def runSinglePromise(self):
-    """
-      run a single promise in a new process
-    """
-    if os.path.exists(self.config.pid_path):
-      with open(self.config.pid_path, "r") as pidfile:
-        try:
-          pid = int(pidfile.read(6))
-        except ValueError:
-          pid = None
-        if pid and os.path.exists("/proc/" + str(pid)):
-          self.logger.warning("A process is already running with pid " + str(pid))
-          return 1
-    start_date = datetime.utcnow()
-    with open(self.config.pid_path, "w") as pidfile:
-      self.logger.info("Running script %s..." % self.config.promise_script)
-      process = self.executeCommand(self.config.promise_script)
-      ps_process = psutil.Process(process.pid)
-      pidfile.write(str(process.pid))
-
-    status_json = self.generateStatusJsonFromProcess(
-      process,
-      start_date=start_date,
-      title=self.config.promise_name
-    )
-
-    status_json['_links'] = {"monitor": {"href": self.config.monitor_url}}
-    status_json['title'] = self.config.promise_name
-    status_json['instance'] = self.config.instance_name
-    status_json['hosting_subscription'] = self.config.hosting_name
-    status_json['type'] = self.config.promise_type
-    status_json['portal-type'] = "promise" if \
-        self.config.promise_type == "status" else self.config.promise_type
-
-    # Save the lastest status change date (needed for rss)
-    status_json['change-time'] = ps_process.create_time()
-    if os.path.exists(self.config.output) and os.stat(self.config.output).st_size:
-      with open(self.config.output) as f:
-        try:
-          last_result = json.loads(f.read())
-          if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
-            status_json['change-time'] = last_result['change-time']
-        except ValueError:
-          pass
-
-    self.updateStatusHistoryFolder(
-      self.config.promise_name,
-      self.config.output,
-      self.config.history_folder,
-      self.config.promise_type
-    )
-    # write the promise in a tmp file the move to promise output file
-    # this reduce conflict error on read/write at sametime
-    output_tmp = '%s.tmp' % self.config.output
-    with open(output_tmp, "w") as outputfile:
-      json.dump(status_json, outputfile)
-    os.rename(output_tmp, self.config.output)
-    os.remove(self.config.pid_path)
-
-  def runPromiseSynchronous(self):
+  def start(self):
     """
       run all promises in sequential ways
     """
+    if self.config.pid_path:
       if os.path.exists(self.config.pid_path):
         # Check if another run promise is running
         with open(self.config.pid_path) as fpid:
@@ -168,171 +124,49 @@ class RunPromise(object):
       with open(self.config.pid_path, 'w') as fpid:
         fpid.write(str(os.getpid()))
 
-    promise_folder_list = [self.config.promise_folder]
-    if self.config.monitor_promise_folder:
-      promise_folder_list.append(self.config.monitor_promise_folder)
-    status_list = self.checkPromisesList(promise_folder_list)
-    promises_status_file = os.path.join(self.config.output, '_promise_status')
-    previous_state_dict = {}
-    new_state_dict = {}
-    base_dict = {
-      '_links': {"monitor": {"href": self.config.monitor_url}},
-      'instance': self.config.instance_name,
-      'hosting_subscription': self.config.hosting_name,
-    }
-
-    if os.path.exists(promises_status_file):
-      with open(promises_status_file) as f:
-        try:
-          previous_state_dict = json.loads(f.read())
-        except ValueError:
-          pass
-
-    for status_dict in status_list:
-      status_dict.update(base_dict)
-      if previous_state_dict.has_key(status_dict['title']):
-        status, change_time = previous_state_dict[status_dict['title']].split('#')
-        if status_dict['status'] == status:
-          status_dict['change-time'] = float(change_time)
-
-      promise_result_file = os.path.join(self.config.output, 
-                                         "%s.status.json" % status_dict['title'])
-      # write the promise in a tmp file the move to promise output file
-      # this reduce conflict error on read/write at sametime
-      promise_tmp_file = '%s.tmp' % promise_result_file
-      with open(promise_tmp_file, "w") as outputfile:
-        json.dump(status_dict, outputfile)
-      os.rename(promise_tmp_file, promise_result_file)
-
-      new_state_dict[status_dict['title']] = '%s#%s' % (status_dict['status'],
-                                                        status_dict['change-time'])
-      self.updateStatusHistoryFolder(
-        status_dict['title'],
-        promise_result_file,
-        self.config.history_folder,
-        'status'
-      )
-
-    with open(promises_status_file, "w") as outputfile:
-      json.dump(new_state_dict, outputfile)
-
-    os.remove(self.config.pid_path)
-
-  def updateStatusHistoryFolder(self, name, status_file, history_folder, promise_type):
-    history_path = os.path.join(history_folder)
-    if not os.path.exists(status_file):
-      return
-    if not os.path.exists(history_folder):
-      return
-    if not os.path.exists(history_path):
-      try:
-        os.makedirs(history_path)
-      except OSError, e:
-        if e.errno == os.errno.EEXIST and os.path.isdir(history_path):
-          pass
-        else: raise
-    with open(status_file, 'r') as sf:
-      try:
-        status_dict = json.loads(sf.read())
-      except ValueError:
-        traceback.print_exc()
-        return
-
-    if promise_type == 'status':
-      filename = '%s.history.json' % name
-      history_file = os.path.join(history_path, filename)
-      # Remove links from history (not needed)
-      status_dict.pop('_links', None)
-      if not os.path.exists(history_file) or \
-        (os.path.exists(history_file) and not os.stat(history_file).st_size):
-        with open(history_file, 'w') as f_history:
-          data_dict = {
-            "date": time.time(),
-            "data": [status_dict]
+    if not self.config.partition_folder:
+      raise ValueError("Partition folder is not specified")
+
+    parameter_dict = {
+      'promise-timeout': self.config.promise_timeout or promise_timeout,
+      'promise-folder': os.path.join(self.config.partition_folder, 'etc', 'plugin'),
+      'legacy-promise-folder': os.path.join(self.config.partition_folder, 'etc', 'promise'),
+      'partition-folder': self.config.partition_folder,
+      'master-url': self.config.master_url,
+      'partition-cert': self.config.partition_cert,
+      'partition-key': self.config.partition_key,
+      'partition-id': self.config.partition_id,
+      'computer-id': self.config.computer_id,
+      'debug': self.config.debug,
+      'check-anomaly': self.config.check_anomaly,
+      'force': self.config.force,
+      'run-only-promise-list': [x for x in self.config.run_only.split(' ') if x]
     }
-          f_history.write(json.dumps(data_dict))
+    if self.config.log_folder:
+      parameter_dict['log-folder'] = self.config.log_folder
     else:
-        # Remove useless informations
-        status_dict.pop('hosting_subscription', '')
-        status_dict.pop('title', '')
-        status_dict.pop('instance', '')
-        status_dict.pop('type', '')
-        status_dict.pop('portal-type', '')
-
-        with open (history_file, mode="r+") as f_history:
-          f_history.seek(0,2)
-          position = f_history.tell() -2
-          f_history.seek(position)
-          #f_history.write(',%s]}' % str(status_dict))
-          f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
-    elif promise_type == 'report':
-      # keep_item_amount = 3
-      filename = '%s.history.json' % (
-        name)
-
-      copyfile(status_file, os.path.join(history_path, filename))
-
-  def generateStatusJsonFromProcess(self, process, start_date=None, title=None):
-    stdout, stderr = process.communicate()
-    status_json = {}
-    if process.returncode != 0:
-      status_json["returncode"] = process.returncode
-      status_json["status"] = "ERROR"
-      if not stdout:
-        status_json["message"] = stderr
-      else:
-        status_json["message"] = stdout
-    else:
-      status_json["status"] = "OK" 
-      status_json["message"] = stdout
-      status_json["returncode"] = 0
-
-    if start_date:
-      date_diff = (datetime.utcnow() - start_date)
-      status_json["start-date"] = start_date.strftime('%Y-%m-%dT%H:%M:%S')
-      status_json["execution-time"] = date_diff.total_seconds()
-      self.logger.info("Finished execution of %s in %s second(s)." % (
-                    title, date_diff.total_seconds()))
-    if title:
-      status_json["title"] = title
-    return status_json
-
-  def executeCommand(self, args):
-    return subprocess.Popen(
-      args,
-      stdin=None,
-      stdout=subprocess.PIPE,
-      stderr=subprocess.PIPE
+      parameter_dict['log-folder'] = os.path.join(self.config.partition_folder,
+                                                  PROMISE_LOG_FOLDER_NAME)
+      mkdir_p(parameter_dict['log-folder'])
+
+    promise_launcher = PromiseLauncher(
+      config=parameter_dict,
+      logger=self.logger,
+      dry_run=self.config.dry_run
     )
 
-  def checkPromisesList(self, promise_dir_list):
-    """
-      Run all promises found into specified folder
-    """
-    promise_list = []
     self.logger.info("Checking promises...")
-    for promise_dir in promise_dir_list:
-      promise_list.extend(_checkPromiseList(
-        promise_dir,
-        self.promise_timeout,
-        profile=False,
-        raise_on_failure=False,
-        logger=self.logger))
-
-    # Check whether every promise is kept
-    for i in range(0, len(promise_list)):
-      promise_list[i]["status"] = "OK" if promise_list[i]["returncode"] == 0 else "ERROR"
-      promise_list[i]["type"] = "status"
-      promise_list[i]["portal-type"] = "promise"
-      promise_list[i]["change-time"] = time.mktime(promise_list[i]["start-date"].timetuple())
-      promise_list[i]["start-date"] = promise_list[i]["start-date"].strftime('%Y-%m-%dT%H:%M:%S')
-
+    try:
+      promise_launcher.run()
+    except PromiseError, e:
+      #self.logger.exception(e)
+      # error was already logged
+      pass
+    os.remove(self.config.pid_path)
     self.logger.info("Finished promises.")
-    self.logger.info("---")
-    return promise_list
 
 
 def main():
-  arg_parser = parseArguments()
-  promise_runner = RunPromise(arg_parser.parse_args())
-  sys.exit(promise_runner.runPromise())
+  arg_parser = getArgumentParser()
+  promise_runner = MonitorPromiseLauncher(arg_parser.parse_args())
+  sys.exit(promise_runner.start())

slapos/test/monitor/testbootstrap.py

@@ -12,15 +12,18 @@ class MonitorBootstrapTest(unittest.TestCase):
 
   def setUp(self):
     self.base_dir = tempfile.mkdtemp()
+    os.mkdir(os.path.join(self.base_dir, 'plugin'))
     os.mkdir(os.path.join(self.base_dir, 'promise'))
-    os.mkdir(os.path.join(self.base_dir, 'monitor-promise'))
     os.mkdir(os.path.join(self.base_dir, 'public'))
     os.mkdir(os.path.join(self.base_dir, 'private'))
     os.mkdir(os.path.join(self.base_dir, 'cron.d'))
+    os.mkdir(os.path.join(self.base_dir, 'log'))
+    os.mkdir(os.path.join(self.base_dir, 'log-backup'))
     os.mkdir(os.path.join(self.base_dir, 'logrotate.d'))
     os.mkdir(os.path.join(self.base_dir, 'monitor-report'))
     os.mkdir(os.path.join(self.base_dir, 'webdav'))
     os.mkdir(os.path.join(self.base_dir, 'run'))
+    os.mkdir(os.path.join(self.base_dir, 'private/documents'))
     self.writeContent(os.path.join(self.base_dir, 'param'), '12345')
     self.writeContent(os.path.join(self.base_dir, '.monitor_pwd'), 'bcuandjy')
     self.writeContent(os.path.join(self.base_dir, 'test-httpd-cors.cfg'), '')
@@ -28,13 +31,12 @@ class MonitorBootstrapTest(unittest.TestCase):
     self.monitor_config_file = os.path.join(self.base_dir, 'monitor.conf')
 
     self.monitor_config_dict = dict(
+      monitor_conf=self.monitor_config_file,
       base_dir=self.base_dir,
       root_title="Monitor ROOT",
       title="Monitor",
       url_list="",
       base_url="https://monitor.test.com",
-      monitor_promise_folder=os.path.join(self.base_dir, 'monitor-promise'),
-      promise_folder=os.path.join(self.base_dir, 'promise'),
       promise_runner_pid=os.path.join(self.base_dir, 'run', 'monitor-promises.pid'),
       public_folder=os.path.join(self.base_dir, 'public'),
       public_path_list="",
@@ -45,18 +47,18 @@ class MonitorBootstrapTest(unittest.TestCase):
     )
     self.monitor_conf = """[monitor]
 parameter-file-path = %(base_dir)s/knowledge0.cfg
-promise-folder = %(base_dir)s/promise
 service-pid-folder = %(base_dir)s/run
-monitor-promise-folder = %(base_dir)s/monitor-promise
 private-folder = %(base_dir)s/private
 public-folder = %(base_dir)s/public
 public-path-list = %(public_path_list)s
 private-path-list = %(private_path_list)s
 crond-folder = %(base_dir)s/cron.d
 logrotate-folder = %(base_dir)s/logrotate.d
-report-folder = %(base_dir)s/monitor-report
 root-title = %(root_title)s
 pid-file =  %(base_dir)s/monitor.pid
+log-folder = %(base_dir)s/log
+log-backup-folder = %(base_dir)s/log-backup
+document-folder = %(base_dir)s/private/documents
 parameter-list = 
   raw monitor-user admin
   file sample %(base_dir)s/param
@@ -75,6 +77,12 @@ service-pid-folder = %(base_dir)s/run
 promise-output-file = %(base_dir)s/monitor-bootstrap-status
 promise-runner = %(promise_run_script)s
 randomsleep = /bin/echo sleep
+
+[promises]
+output-folder = %(base_dir)s/public
+legacy-promise-folder = %(base_dir)s/promise
+promise-folder = %(base_dir)s/plugin
+partition-folder = %(base_dir)s
 """
 
     self.opml_outline = """<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(base_url)s/public/feed" xmlUrl="%(base_url)s/public/feed" url="%(base_url)s/share/private/" />"""
@@ -87,21 +95,18 @@ randomsleep = /bin/echo sleep
     with open(file_path, 'w') as cfg:
       cfg.write(config)
 
-  def configPromises(self, amount):
+  def configPromiseList(self, amount):
     promise_dir = os.path.join(self.base_dir, 'promise')
+    plugin_dir = os.path.join(self.base_dir, 'plugin')
     promse_content = "/bin/bash echo something"
     for index in range(1, amount+1):
       promise_file = os.path.join(promise_dir, 'monitor_promise-%s' % index)
       self.writeContent(promise_file, promse_content)
       os.chmod(promise_file, 0755)
-
-  def configReports(self, amount):
-    promise_dir = os.path.join(self.base_dir, 'monitor-report')
-    promse_content = "/bin/bash echo something"
     for index in range(1, amount+1):
-      promise_file = os.path.join(promise_dir, 'monitor_report-%s' % index)
+      promise_file = os.path.join(plugin_dir, 'monitor_promise-%s.py' % index)
       self.writeContent(promise_file, promse_content)
-      os.chmod(promise_file, 0755)
+      os.chmod(promise_file, 0644)
 
   def checkOPML(self, url_list):
     opml_title = "<title>%(root_title)s</title>" % self.monitor_config_dict
@@ -116,54 +121,6 @@ randomsleep = /bin/echo sleep
           base_url=url)
       self.assertTrue(opml_outline in opml_content)
 
-  def check_promises(self, sequential=False):
-    promise_cron = os.path.join(self.base_dir, 'cron.d', 'monitor-promises')
-    self.assertTrue(os.path.exists(promise_cron))
-    with open(promise_cron) as cronf:
-      promise_command_list = cronf.read()
-
-    if not sequential:
-      promise_entry = '* * * * * /bin/echo sleep 60 && %(promise_run_script)s --pid_path "%(promise_runner_pid)s" --output "%(public_folder)s" --promise_folder "%(promise_folder)s" --timeout_file "None" --monitor_promise_folder "%(monitor_promise_folder)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(base_dir)s/public" --instance_name "%(title)s" --hosting_name "%(root_title)s"'
-      entry_line = promise_entry % self.monitor_config_dict
-      self.assertTrue(entry_line in promise_command_list,
-            "%s not in %s" %(entry_line, promise_command_list))
-    else:
-      promise_entry = '* * * * * /bin/echo sleep 60 &&%(promise_run_script)s --pid_path "%(promise_pid)s" --output "%(promise_output)s" --promise_script "%(promise_executable)s" --promise_name "%(promise_name)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(base_dir)s/public" --instance_name "%(title)s" --hosting_name "%(root_title)s"'
-    
-      promise_dir = os.path.join(self.base_dir, 'promise')
-      for filename in os.listdir(promise_dir):
-        promise_dict = dict(
-          promise_pid=os.path.join(self.base_dir, 'run', '%s.pid' % filename),
-          promise_output=os.path.join(self.base_dir, 'public', '%s.status.json' % filename),
-          promise_executable=os.path.join(promise_dir, filename),
-          promise_name=filename
-        )
-        promise_dict.update(self.monitor_config_dict)
-        entry_line = promise_entry % promise_dict
-        self.assertTrue(entry_line in promise_command_list)
-
-  def check_report(self):
-    promise_entry = '* * * * * %(promise_run_script)s --pid_path "%(promise_pid)s" --output "%(promise_output)s" --promise_script "%(promise_executable)s" --promise_name "%(promise_name)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(data_dir)s" --instance_name "%(title)s" --hosting_name "%(root_title)s" --promise_type "report"'
-    promise_dir = os.path.join(self.base_dir, 'monitor-report')
-    data_dir = os.path.join(self.base_dir, 'private', 'documents')
-
-    promise_cron = os.path.join(self.base_dir, 'cron.d', 'monitor-reports')
-    self.assertTrue(os.path.exists(promise_cron))
-    with open(promise_cron) as cronf:
-      promise_command_list = cronf.read()
-
-    for filename in os.listdir(promise_dir):
-      promise_dict = dict(
-        promise_pid=os.path.join(self.base_dir, 'run', '%s.pid' % filename),
-        promise_output=os.path.join(data_dir, '%s.report.json' % filename),
-        promise_executable=os.path.join(promise_dir, filename),
-        promise_name=filename,
-        data_dir=data_dir
-      )
-      promise_dict.update(self.monitor_config_dict)
-      entry_line = promise_entry % promise_dict
-      self.assertTrue(entry_line in promise_command_list)
-
   def check_folder_equals(self, source, destination):
     self.assertTrue(os.path.isdir(source))
 
@@ -261,7 +218,7 @@ randomsleep = /bin/echo sleep
     self.assertTrue(os.path.exists(os.path.join(self.base_dir, 'private/documents')))
 
   def test_monitor_bootstrap_promises(self):
-    self.configPromises(3)
+    self.configPromiseList(3)
 
     config_content = self.monitor_conf % self.monitor_config_dict
     self.writeContent(self.monitor_config_file, config_content)
@@ -273,38 +230,12 @@ randomsleep = /bin/echo sleep
 
     self.checkOPML([self.monitor_config_dict['base_url']])
 
-    self.check_promises()
-
     # Check update promises_list
-    self.configPromises(5) # Add two promises
+    self.configPromiseList(5) # Add two promises
     os.unlink(os.path.join(self.base_dir, 'promise', 'monitor_promise-2')) # drop promise number 2
     instance2 = Monitoring(self.monitor_config_file)
     instance2.bootstrapMonitor()
     self.assertTrue(os.path.exists(promise_file))
-    self.check_promises()
-
-  def test_monitor_bootstrap_report(self):
-    self.configReports(3)
-
-    config_content = self.monitor_conf % self.monitor_config_dict
-    self.writeContent(self.monitor_config_file, config_content)
-
-    instance = Monitoring(self.monitor_config_file)
-    instance.bootstrapMonitor()
-    promise_file = os.path.join(self.base_dir, 'monitor-bootstrap-status')
-    self.assertTrue(os.path.exists(promise_file))
-
-    self.checkOPML([self.monitor_config_dict['base_url']])
-
-    self.check_report()
-
-    # Check update promises_list
-    self.configReports(5) # Add two promises
-    os.unlink(os.path.join(self.base_dir, 'monitor-report', 'monitor_report-1')) # drop promise number 2
-    instance2 = Monitoring(self.monitor_config_file)
-    instance2.bootstrapMonitor()
-    self.assertTrue(os.path.exists(promise_file))
-    self.check_report()
 
   def test_monitor_bootstrap_genconfig(self):
     config_content = self.monitor_conf % self.monitor_config_dict

slapos/test/monitor/testglobalstate.py

@@ -5,50 +5,56 @@ import shutil
 import tempfile
 import unittest
 import json
+import pkg_resources
 
 from slapos.monitor import globalstate
-from slapos.monitor.runpromise import RunPromise, parseArguments
+from slapos.monitor.runpromise import MonitorPromiseLauncher, getArgumentParser
 from slapos.monitor.monitor import Monitoring
+from jsonschema import validate
 
 class MonitorGlobalTest(unittest.TestCase):
 
   def setUp(self):
     self.base_dir = tempfile.mkdtemp()
-    os.mkdir(os.path.join(self.base_dir, 'promise'))
-    os.mkdir(os.path.join(self.base_dir, 'monitor-promise'))
+    self.etc_dir = os.path.join(self.base_dir, 'etc')
+    self.output_dir = os.path.join(self.base_dir, '.slapgrid/promise/result')
+    os.mkdir(self.etc_dir)
+    os.mkdir(os.path.join(self.etc_dir, 'plugin'))
+    os.mkdir(os.path.join(self.etc_dir, 'promise'))
     os.mkdir(os.path.join(self.base_dir, 'public'))
     os.mkdir(os.path.join(self.base_dir, 'private'))
     os.mkdir(os.path.join(self.base_dir, 'cron.d'))
+    os.mkdir(os.path.join(self.base_dir, 'log'))
+    os.mkdir(os.path.join(self.base_dir, 'log-backup'))
     os.mkdir(os.path.join(self.base_dir, 'logrotate.d'))
     os.mkdir(os.path.join(self.base_dir, 'monitor-report'))
     os.mkdir(os.path.join(self.base_dir, 'webdav'))
     os.mkdir(os.path.join(self.base_dir, 'run'))
+    os.mkdir(os.path.join(self.base_dir, 'private/documents'))
     self.writeContent(os.path.join(self.base_dir, 'param'), '12345')
     self.writeContent(os.path.join(self.base_dir, '.monitor_pwd'), 'bcuandjy')
     self.writeContent(os.path.join(self.base_dir, 'test-httpd-cors.cfg'), '')
     self.writeContent(os.path.join(self.base_dir, 'monitor-htpasswd'), '12345')
-    self.writeContent(os.path.join(self.base_dir, 'instance.cfg'), """[instance]
-name = Monitor
-root-name = Monitor ROOT
-computer = COMP-1234
-ipv4 = 10.0.151.118
-ipv6 = 2001:34c:1254:df3:89::5df3
-software-release = http://some.url.com/software.cfg
-software-type = default
-partition = slappart10""")
 
     self.monitor_config_file = os.path.join(self.base_dir, 'monitor.conf')
     self.public_dir = os.path.join(self.base_dir, 'public')
     self.private_dir = os.path.join(self.base_dir, 'private')
 
+    monitor_schema_string = \
+      pkg_resources.resource_string(
+        'slapos.monitor',
+        'doc/monitor_instance.schema.json')
+    self.monitor_instance_schema = json.loads(monitor_schema_string)
+
+
     self.monitor_config_dict = dict(
+      monitor_conf=self.monitor_config_file,
       base_dir=self.base_dir,
       root_title="Monitor ROOT",
       title="Monitor",
-      url_list="",
+      url_list="https://sub.monitor.test.com https://sub2.monitor.test.com",
       base_url="https://monitor.test.com",
-      monitor_promise_folder=os.path.join(self.base_dir, 'monitor-promise'),
-      promise_folder=os.path.join(self.base_dir, 'promise'),
+      etc_dir=self.etc_dir,
       promise_runner_pid=os.path.join(self.base_dir, 'run', 'monitor-promises.pid'),
       public_folder=os.path.join(self.base_dir, 'public'),
       public_path_list="",
@@ -59,18 +65,18 @@ partition = slappart10""")
     )
     self.monitor_conf = """[monitor]
 parameter-file-path = %(base_dir)s/knowledge0.cfg
-promise-folder = %(base_dir)s/promise
 service-pid-folder = %(base_dir)s/run
-monitor-promise-folder = %(base_dir)s/monitor-promise
 private-folder = %(base_dir)s/private
 public-folder = %(base_dir)s/public
 public-path-list = %(public_path_list)s
 private-path-list = %(private_path_list)s
 crond-folder = %(base_dir)s/cron.d
 logrotate-folder = %(base_dir)s/logrotate.d
-report-folder = %(base_dir)s/monitor-report
 root-title = %(root_title)s
 pid-file =  %(base_dir)s/monitor.pid
+log-folder = %(base_dir)s/log
+log-backup-folder = %(base_dir)s/log-backup
+document-folder = %(base_dir)s/private/documents
 parameter-list = 
   raw monitor-user admin
   file sample %(base_dir)s/param
@@ -89,6 +95,21 @@ service-pid-folder = %(base_dir)s/run
 promise-output-file = %(base_dir)s/monitor-bootstrap-status
 promise-runner = %(promise_run_script)s
 randomsleep = /bin/echo sleep
+
+[promises]
+output-folder = %(base_dir)s/public
+legacy-promise-folder = %(etc_dir)s/promise
+promise-folder = %(etc_dir)s/plugin
+partition-folder = %(base_dir)s
+computer-id = COMP-1234
+partition-cert = 
+partition-key = 
+partition-id = slappart0
+ipv6 = 2001:34c:1254:df3:89::5df3
+software-release = http://some.url.com/software.cfg
+master-url = http://10.0.151.118:50000
+software-type = default
+ipv4 = 10.0.151.118
 """
 
   def tearDown(self):
@@ -109,7 +130,7 @@ randomsleep = /bin/echo sleep
 echo "%(output)s"
 exit %(code)s
 """ % result_dict
-    promise_path = os.path.join(self.base_dir, 'promise', name)
+    promise_path = os.path.join(self.etc_dir, 'promise', name)
     self.writeContent(promise_path, content)
     os.chmod(promise_path, 0755)
     return promise_path
@@ -118,18 +139,13 @@ exit %(code)s
     pid_path = os.path.join(self.base_dir, 'run', 'monitor-promise.pid')
 
     promise_cmd = [
-      '--pid_path',
-      '%s' % pid_path, '--output', os.path.join(self.base_dir, 'public'),
-      '--promise_folder', os.path.join(self.base_dir, 'promise'),
-      '--monitor_promise_folder', os.path.join(self.base_dir, 'monitor-promise'),
-      '--promise_type', 'status',
-      '--monitor_url', 'https://monitor.test.com/share/private/',
-      '--history_folder', os.path.join(self.base_dir, 'public'),
-      '--instance_name', 'Monitor', '--hosting_name', 'Monitor ROOT']
-    arg_parser = parseArguments()
+      '--pid-path',
+      '%s' % pid_path, '-c', self.monitor_config_file]
+    arg_parser = getArgumentParser()
     return arg_parser.parse_args(promise_cmd)
 
   def test_monitor_instance_state(self):
+    self.maxDiff = None
     config_content = self.monitor_conf % self.monitor_config_dict
     self.writeContent(self.monitor_config_file, config_content)
 
@@ -141,31 +157,27 @@ exit %(code)s
     self.writePromise('promise_3', success=False)
     self.writePromise('promise_4')
     parser = self.getPromiseParser()
-    promise_runner = RunPromise(parser)
-    promise_runner.runPromise()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_1.status.json')))
-    self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_2.status.json')))
-    self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_3.status.json')))
-    self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_4.status.json')))
+    self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_1.status.json')))
+    self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_2.status.json')))
+    self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_3.status.json')))
+    self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_4.status.json')))
 
+    os.symlink(self.output_dir, '%s/public/promise' % self.base_dir)
     # generate instance state files
-    globalstate.run([self.monitor_config_file, os.path.join(self.base_dir, 'instance.cfg')])
+    globalstate.run(self.monitor_config_file)
     self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'feed')))
     self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'monitor.global.json')))
     self.assertTrue(os.path.exists(os.path.join(self.private_dir, 'monitor.global.json')))
     expected_result = """{
 	"status": "ERROR",
-	"_embedded": {
-		"instance": {
-			"partition": "slappart10",
+	"partition_id": "slappart0",
 	"ipv6": "2001:34c:1254:df3:89::5df3",
-			"computer": "COMP-1234",
 	"ipv4": "10.0.151.118",
-			"software-release": "http://some.url.com/software.cfg",
-			"software-type": "default"
-		}
-	},
+	"software_release": "http://some.url.com/software.cfg",
+	"software_type": "default",
 	"parameters": [{
 		"key": "",
 		"value": "admin",
@@ -206,7 +218,11 @@ exit %(code)s
 		},
 		"private_url": {
 			"href": "https://monitor.test.com/share/private/"
-		}
+		},
+		"related_monitor": [
+		  {"href": "https://sub.monitor.test.com/share/public"},
+      {"href": "https://sub2.monitor.test.com/share/public"}
+    ]
 	},
 	"aggregate_reference": "COMP-1234",
 	"type": "global",
@@ -222,15 +238,22 @@ exit %(code)s
     # all promises are OK now
     self.writePromise('promise_2', success=True)
     self.writePromise('promise_3', success=True)
-    promise_runner.runPromise()
-    globalstate.run([self.monitor_config_file, os.path.join(self.base_dir, 'instance.cfg')])
+    # rerun right now
+    promise_runner.config.force = True
+    promise_runner.start()
+    globalstate.run(self.monitor_config_file)
 
     expected_result_dict = json.loads(expected_result)
     expected_result_dict["status"] = "OK"
     expected_result_dict["state"] = {'error': 0, 'success': 4}
+    instance_result_dict = None
     with open(os.path.join(self.private_dir, 'monitor.global.json')) as r:
-      result = json.loads(r.read().decode("utf-8"))
+      instance_result_dict = json.loads(r.read().decode("utf-8"))
+      result = instance_result_dict.copy()
       result.pop("date")
       self.assertEquals(result,
         expected_result_dict)
 
+    # validate returned json result
+    validate(instance_result_dict, self.monitor_instance_schema)
+

slapos/test/monitor/testrunpromise.py

@@ -6,39 +6,68 @@ import tempfile
 import unittest
 import json
 from datetime import datetime
-from slapos.monitor.runpromise import RunPromise, parseArguments
+from slapos.monitor.runpromise import MonitorPromiseLauncher, getArgumentParser
+from slapos.grid.promise.generic import PROMISE_LOG_FOLDER_NAME, PROMISE_RESULT_FOLDER_NAME
 
 class MonitorPromiseTest(unittest.TestCase):
 
   def setUp(self):
     self.base_dir = tempfile.mkdtemp()
-    self.promise_dir = os.path.join(self.base_dir, 'promise')
-    self.monitor_promise_dir = os.path.join(self.base_dir, 'monitor-promise')
-    self.report_dir = os.path.join(self.base_dir, 'report')
+    self.etc_dir = os.path.join(self.base_dir, 'etc')
+    self.output_dir = os.path.join(self.base_dir, PROMISE_RESULT_FOLDER_NAME)
+    self.log_output_dir = os.path.join(self.base_dir, PROMISE_LOG_FOLDER_NAME)
+    self.promise_dir = os.path.join(self.etc_dir, 'plugin')
+    self.old_promise_dir = os.path.join(self.etc_dir, 'promise')
     self.public_dir = os.path.join(self.base_dir, 'public')
     self.private_dir = os.path.join(self.base_dir, 'private')
     self.run_dir = os.path.join(self.base_dir, 'run')
+    self.log_dir = os.path.join(self.base_dir, 'log')
+    os.mkdir(self.etc_dir)
     os.mkdir(self.promise_dir)
     os.mkdir(self.public_dir)
     os.mkdir(self.private_dir)
-    os.mkdir(self.report_dir)
     os.mkdir(self.run_dir)
-    os.mkdir(self.monitor_promise_dir)
-
+    os.mkdir(self.old_promise_dir)
+    os.mkdir(self.log_dir)
+
+    self.monitor_config = """[monitor]
+private-folder = %(base_dir)s/private
+public-folder = %(base_dir)s/public
+root-title = %(root_title)s
+log-folder = %(base_dir)s/log
+base-url = %(base_url)s
+title = %(title)s
+
+[promises]
+pid-path = %(base_dir)s/run/promises.pid
+partition-folder = %(base_dir)s
+computer-id = COMP-1234
+partition-cert = 
+partition-key = 
+partition-id = slappart0
+ipv6 = 2001:34c:1254:df3:89::5df3
+software-release = http://some.url.com/software.cfg
+master-url = http://10.0.151.118:50000
+software-type = default
+ipv4 = 10.0.151.118
+""" % {'base_dir': self.base_dir, 'title': 'Monitor', 'root_title': 'Monitor ROOT',
+       'base_url': 'https://monitor.test.com', }
+
+    self.monitor_config_file = os.path.join(self.base_dir, 'monitor.conf')
+    with open(self.monitor_config_file, 'w') as f:
+      f.write(self.monitor_config)
 
   def tearDown(self):
     if os.path.exists(self.base_dir):
       shutil.rmtree(self.base_dir)
 
-  def writePromiseOK(self, name, monitor_folder=False):
+  def writePromiseOK(self, name):
     content = """#!/bin/sh
 
 echo "success"
 exit 0
 """
-    promise_path = os.path.join(self.promise_dir, name)
-    if monitor_folder:
-      promise_path = os.path.join(self.monitor_promise_dir, name)
+    promise_path = os.path.join(self.old_promise_dir, name)
     self.writeContent(promise_path, content)
     os.chmod(promise_path, 0755)
     return promise_path
@@ -49,348 +78,333 @@ exit 0
 echo "failed"
 exit 2
 """
-    promise_path = os.path.join(self.promise_dir, name)
+    promise_path = os.path.join(self.old_promise_dir, name)
     self.writeContent(promise_path, content)
     os.chmod(promise_path, 0755)
     return promise_path
 
+  def generatePromiseScript(self, name, success=True, failure_count=1, content="",
+      periodicity=0.03):
+    promise_content = """from zope import interface as zope_interface
+from slapos.grid.promise import interface
+from slapos.grid.promise import GenericPromise
+
+class RunPromise(GenericPromise):
+
+ zope_interface.implements(interface.IPromise)
+
+ def __init__(self, config):
+   GenericPromise.__init__(self, config)
+   self.setPeriodicity(minute=%(periodicity)s)
+
+ def sense(self):
+   %(content)s
+
+   if not %(success)s:
+     self.logger.error("failed")
+   else:
+     self.logger.info("success")
+
+ def anomaly(self):
+   return self._anomaly(result_count=2, failure_amount=%(failure_amount)s)
+
+ def test(self):
+   return self._test(result_count=1, failure_amount=%(failure_amount)s)
+""" % {'success': success, 'content': content, 'failure_amount': failure_count,
+      'periodicity': periodicity}
+ 
+    with open(os.path.join(self.promise_dir, name), 'w') as f:
+      f.write(promise_content)
+    return os.path.join(self.promise_dir, name)
+
   def writeContent(self, file_path, config):
     with open(file_path, 'w') as cfg:
       cfg.write(config)
 
-  def getUniquePromiseParser(self, name, promise_path, promise_type):
-    pid_path = os.path.join(self.run_dir, '%s.pid' % name)
-    if promise_type == "report":
-      output_path = os.path.join(self.private_dir, '%s.report.json' % name)
-    else:
-      output_path = os.path.join(self.public_dir, '%s.status.json' % name)
-    promise_cmd = [
-      '--pid_path',
-      '%s' % pid_path, '--output', output_path,
-      '--promise_script', promise_path,
-      '--promise_name', name, '--promise_type', promise_type,
-      '--monitor_url', 'https://monitor.test.com/share/private/',
-      '--history_folder', self.public_dir,
-      '--instance_name', 'Monitor', '--hosting_name', 'Monitor ROOT']
-    arg_parser = parseArguments()
-    return arg_parser.parse_args(promise_cmd)
+  def getPromiseParser(self, use_config=True, check_anomaly=False, force=False):
 
-  def getPromiseParser(self):
-    pid_path = os.path.join(self.run_dir, 'monitor-promise.pid')
+    arg_parser = getArgumentParser()
+    base_list = ['-c', self.monitor_config_file]
+    if use_config:
+      if check_anomaly:
+        base_list.append('-a')
+      if force:
+        base_list.append('--force')
+      return arg_parser.parse_args(base_list)
 
+    pid_path = os.path.join(self.run_dir, 'monitor-promise.pid')
     promise_cmd = [
-      '--pid_path',
-      '%s' % pid_path, '--output', self.public_dir,
-      '--promise_folder', self.promise_dir,
-      '--monitor_promise_folder', self.monitor_promise_dir,
-      '--promise_type', 'status',
-      '--monitor_url', 'https://monitor.test.com/share/private/',
-      '--history_folder', self.public_dir,
-      '--instance_name', 'Monitor', '--hosting_name', 'Monitor ROOT']
-    arg_parser = parseArguments()
+      '--pid-path',
+      '%s' % pid_path, '-o', self.public_dir,
+      '--master-url', 'http://10.0.151.118:50000',
+      '--partition-id', 'slappart0',
+      '--computer-id', 'COMP-1234']
     return arg_parser.parse_args(promise_cmd)
 
-  def test_promise_OK(self):
-
-    promise = self.writePromiseOK('promise_1')
+  def test_promise_generic(self):
+    promise_file = self.generatePromiseScript('my_promise.py', success=True)
+    promise_file2 = self.generatePromiseScript('my_second_promise.py', success=True)
     parser = self.getPromiseParser()
-    promise_runner = RunPromise(parser)
-    promise_runner.runPromise()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    result_file = os.path.join(self.public_dir, 'promise_1.status.json')
+    result_file = os.path.join(self.output_dir, 'my_promise.status.json')
+    os.system('cat %s' % result_file)
     self.assertTrue(os.path.exists(result_file))
-    result1 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time = result1.pop('change-time', 0)
-    change_date = datetime.fromtimestamp(change_time)
-    execution_time = result1.pop('execution-time')
-    start_date = result1.pop('start-date')
-
-    expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
-      u'title': u'promise_1', u'instance': u'Monitor',
-      u'_links': 
-      {u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
-      u'message': u'success\n', u'type': u'status',
-      u'portal-type': u'promise', u'returncode': 0}
-    self.assertEquals(expected_result, result1)
-
-    # second run
-    time.sleep(1)
-    promise_runner.runPromise()
-    result2 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time2 = result2.pop('change-time', 0)
-    start_date2 = result2.pop('start-date')
-    change_date2 = datetime.fromtimestamp(change_time2)
-    execution_time2 = result2.pop('execution-time')
-    self.assertEquals(expected_result, result2)
-    self.assertEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
-      change_date2.strftime('%Y-%m-%d %H:%M:%S'))
-
-    history_file = os.path.join(self.public_dir, 'promise_1.history.json')
-    self.assertTrue(os.path.exists(history_file))
-    history = json.load(open(history_file))
-    self.assertTrue(history['date'] > change_time)
-    self.assertTrue(len(history['data']) == 2)
-
-    result1['change-time'] = change_time
-    result1['start-date'] = start_date
-    result1.pop('_links')
-    result1['execution-time'] = execution_time
-    result2['start-date'] = start_date2
-    result2['change-time'] = change_time2
-    result2['execution-time'] = execution_time2
-    # not in history
-    result2.pop('_links')
-    result2.pop('hosting_subscription')
-    result2.pop('title')
-    result2.pop('instance')
-    result2.pop('type')
-    result2.pop('portal-type')
-    self.assertEquals(history['data'][0], result1)
-    self.assertEquals(history['data'][1], result2)
-
-  def test_promise_two_folder(self):
-
-    promise = self.writePromiseOK('promise_1')
-    promise2 = self.writePromiseOK('promise_2', monitor_folder=True)
+    my_result = json.loads(open(result_file).read().decode("utf-8"))
+    my_result['result'].pop('date')
+    expected_result = {
+      u'title': u'my_promise', u'name': u'my_promise.py',
+      u'result': {
+        u'failed': False, u'message': u'success', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s/my_promise.py' % self.promise_dir,
+    }
+    self.assertEquals(expected_result, my_result)
+
+    result_file = os.path.join(self.output_dir, 'my_second_promise.status.json')
+    self.assertTrue(os.path.exists(result_file))
+    second_result = json.loads(open(result_file).read().decode("utf-8"))
+    second_result['result'].pop('date')
+
+    expected_result = {
+      u'title': u'my_second_promise', u'name': u'my_second_promise.py',
+      u'result': {
+        u'failed': False, u'message': u'success', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s/my_second_promise.py' % self.promise_dir,
+    }
+    self.assertEquals(expected_result, second_result)
+
+  def test_promise_generic_failed(self):
+    promise_file = self.generatePromiseScript('my_promise.py', success=False)
     parser = self.getPromiseParser()
-    promise_runner = RunPromise(parser)
-    promise_runner.runPromise()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    result_file = os.path.join(self.public_dir, 'promise_1.status.json')
-    result2_file = os.path.join(self.public_dir, 'promise_2.status.json')
+    result_file = os.path.join(self.output_dir, 'my_promise.status.json')
     self.assertTrue(os.path.exists(result_file))
-    self.assertTrue(os.path.exists(result2_file))
-    result1 = json.loads(open(result_file).read().decode("utf-8"))
-    result1.pop('change-time')
-    result1.pop('start-date')
-    self.assertTrue(result1.pop('execution-time', None) is not None)
-
-    expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
-      u'title': u'promise_1', u'instance': u'Monitor',
-      u'_links': 
-      {u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
-      u'message': u'success\n', u'type': u'status',
-      u'portal-type': u'promise', u'returncode': 0}
-    self.assertEquals(expected_result, result1)
+    my_result = json.loads(open(result_file).read().decode("utf-8"))
+    my_result['result'].pop('date')
+
+    expected_result = {
+      u'title': u'my_promise', u'name': u'my_promise.py',
+      u'result': {
+        u'failed': True, u'message': u'failed', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s/my_promise.py' % self.promise_dir,
+    }
+    self.assertEquals(expected_result, my_result)
+
+  def test_promise_generic_status_change(self):
+    promise_file = self.generatePromiseScript('my_promise.py', success=False)
+    parser = self.getPromiseParser()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    result2 = json.loads(open(result2_file).read())
-    result2.pop('change-time')
-    result2.pop('start-date')
-    self.assertTrue(result2.pop('execution-time', None) is not None)
-
-    expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
-      u'title': u'promise_2', u'instance': u'Monitor',
-      u'_links': 
-      {u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
-      u'message': u'success\n', u'type': u'status',
-      u'portal-type': u'promise', u'returncode': 0}
-    self.assertEquals(expected_result, result2)
+    result_file = os.path.join(self.output_dir, 'my_promise.status.json')
+    self.assertTrue(os.path.exists(result_file))
+    my_result = json.loads(open(result_file).read().decode("utf-8"))
+    my_result['result'].pop('date')
+
+    expected_result = {
+      u'title': u'my_promise', u'name': u'my_promise.py',
+      u'result': {
+        u'failed': True, u'message': u'failed', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s/my_promise.py' % self.promise_dir,
+    }
+    self.assertEquals(expected_result, my_result)
+
+    if "my_promise" in sys.modules:
+      # force to reload the module without rerun python
+      os.system('rm %s/*.pyc' % self.promise_dir)
+      del sys.modules["my_promise"]
+    time.sleep(1.5)
+    self.generatePromiseScript('my_promise.py', success=True, periodicity=0.017)
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
+
+    my_result = json.loads(open(result_file).read().decode("utf-8"))
+    my_result['result'].pop('date')
+
+    expected_result = {
+      u'title': u'my_promise', u'name': u'my_promise.py',
+      u'result': {
+        u'failed': False, u'message': u'success', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s/my_promise.py' % self.promise_dir,
+    }
+    self.assertEquals(expected_result, my_result)
+
+  def test_promise_generic_periodicity(self):
+    promise_file = self.generatePromiseScript('my_promise.py', success=False, periodicity=0.03)
+    parser = self.getPromiseParser()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-  def test_promise_One_By_One(self):
+    result_file = os.path.join(self.output_dir, 'my_promise.status.json')
+    self.assertTrue(os.path.exists(result_file))
 
-    promise = self.writePromiseOK('promise_1')
-    parser = self.getUniquePromiseParser('promise_1', promise, 'status')
-    promise_runner = RunPromise(parser)
-    promise_runner.runPromise()
+    # run again
+    os.unlink(result_file)
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
+    self.assertFalse(os.path.exists(result_file))
 
-    result_file = os.path.join(self.public_dir, 'promise_1.status.json')
+    time.sleep(2)
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
     self.assertTrue(os.path.exists(result_file))
-    result1 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time = result1.pop('change-time', 0)
-    change_date = datetime.fromtimestamp(change_time)
-    execution_time = result1.pop('execution-time', None)
-    start_date = result1.pop('start-date')
-
-    expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
-      u'title': u'promise_1', u'instance': u'Monitor',
-      u'_links': 
-        {u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
-        u'message': u'success\n', u'type': u'status',
-        u'portal-type': u'promise', u'returncode': 0}
-    self.assertEquals(expected_result, result1)
-
-    # second run
-    promise_runner.runPromise()
-    result2 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time2 = result2.pop('change-time', 2)
-    result2.pop('start-date')
-    change_date2 = datetime.fromtimestamp(change_time2)
-    self.assertTrue(result2.pop('execution-time', None) is not None)
-    self.assertEquals(expected_result, result2)
-    self.assertEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
-      change_date2.strftime('%Y-%m-%d %H:%M:%S'))
 
-    history_file = os.path.join(self.public_dir, 'promise_1.history.json')
-    self.assertTrue(os.path.exists(history_file))
-    history = json.load(open(history_file))
-    self.assertTrue(history['date'] > change_time)
-    self.assertTrue(len(history['data']) == 1)
+  def test_promise_generic_run_only(self):
+    promise_file = self.generatePromiseScript('my_promise.py', success=False)
+    second_promise = self.generatePromiseScript('my_second_promise.py', success=False)
+    parser = self.getPromiseParser()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    result1['change-time'] = change_time
-    result1['start-date'] = start_date
-    result1['execution-time'] = execution_time
-    result1.pop('_links')
-    self.assertEquals(history['data'][0], result1)
+    result_file = os.path.join(self.output_dir, 'my_promise.status.json')
+    second_result = os.path.join(self.output_dir, 'my_second_promise.status.json')
+    self.assertTrue(os.path.exists(result_file))
+    self.assertTrue(os.path.exists(second_result))
 
-  def test_promise_NOK(self):
+    config = getArgumentParser().parse_args(['-c', self.monitor_config_file, '-a', '-f',
+                                      '--run-only', 'my_second_promise.py'])
+    os.unlink(result_file)
+    promise_runner = MonitorPromiseLauncher(config)
+    promise_runner.start()
+    self.assertFalse(os.path.exists(result_file))
+    self.assertTrue(os.path.exists(second_result))
 
-    promise = self.writePromiseNOK('promise_1')
+  def test_promise_OK(self):
+    promise = self.writePromiseOK('promise_1')
     parser = self.getPromiseParser()
-    promise_runner = RunPromise(parser)
-    promise_runner.runPromise()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    result_file = os.path.join(self.public_dir, 'promise_1.status.json')
+    result_file = os.path.join(self.output_dir, 'promise_1.status.json')
     self.assertTrue(os.path.exists(result_file))
     result1 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time = result1.pop('change-time', 0)
-    change_date = datetime.fromtimestamp(change_time)
-    start_date = result1.pop('start-date')
-
-    self.assertTrue(result1.pop('execution-time', None) is not None)
-    expected_result = {u'status': u'ERROR', u'hosting_subscription': u'Monitor ROOT',
-      u'title': u'promise_1', u'instance': u'Monitor',
-      u'_links': 
-        {u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
-        u'message': u'failed\n', u'type': u'status',
-        u'portal-type': u'promise', u'returncode': 2}
-    self.assertEquals(expected_result, result1)
+    start_date = datetime.strptime(result1['result'].pop('date'), '%Y-%m-%dT%H:%M:%S')
+
+    expected_result = {
+      u'title': u'promise_1', u'name': u'promise_1',
+      u'result': {
+        u'failed': False, u'message': u'success', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s' % promise,
+    }
 
     # second run
-    promise_runner.runPromise()
+    parser = self.getPromiseParser(force=True)
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
     result2 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time2 = result2.pop('change-time', 1)
-    result2.pop('start-date')
-    self.assertTrue(result2.pop('execution-time', None) is not None)
-    change_date2 = datetime.fromtimestamp(change_time2)
+    start_date2 = datetime.strptime(result2['result'].pop('date'), '%Y-%m-%dT%H:%M:%S')
     self.assertEquals(expected_result, result2)
-    self.assertEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
-      change_date2.strftime('%Y-%m-%d %H:%M:%S'))
 
-  def test_promise_mixed(self):
+  def test_promise_two_folder(self):
 
     promise = self.writePromiseOK('promise_1')
+    promise2 = self.writePromiseOK('promise_2')
     parser = self.getPromiseParser()
-    promise_runner = RunPromise(parser)
-    promise_runner.runPromise()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    result_file = os.path.join(self.public_dir, 'promise_1.status.json')
+    result_file = os.path.join(self.output_dir, 'promise_1.status.json')
+    result2_file = os.path.join(self.output_dir, 'promise_2.status.json')
     self.assertTrue(os.path.exists(result_file))
+    self.assertTrue(os.path.exists(result2_file))
     result1 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time = result1.pop('change-time')
-    change_date = datetime.fromtimestamp(change_time)
-    start_date = result1.pop('start-date')
-
-    self.assertTrue(result1.pop('execution-time', None) is not None)
-    expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
-      u'title': u'promise_1', u'instance': u'Monitor',
-      u'_links': 
-        {u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
-        u'message': u'success\n', u'type': u'status',
-        u'portal-type': u'promise', u'returncode': 0}
+    start_date = datetime.strptime(result1['result'].pop('date'), '%Y-%m-%dT%H:%M:%S')
+
+    expected_result = {
+      u'title': u'promise_1', u'name': u'promise_1',
+      u'result': {
+        u'failed': False, u'message': u'success', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s' % promise,
+    }
     self.assertEquals(expected_result, result1)
 
-    # second run with failure
-    time.sleep(2)
-    promise = self.writePromiseNOK('promise_1')
-    parser = self.getPromiseParser()
-    expected_result['message'] = 'failed\n'
-    expected_result['status'] = 'ERROR'
-    expected_result['returncode'] = 2
-    promise_runner.runPromise()
-
-    result2 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time2 = result2.pop('change-time')
-    result2.pop('start-date')
-    change_date2 = datetime.fromtimestamp(change_time2)
-    self.assertTrue(result2.pop('execution-time', None) is not None)
+    result2 = json.loads(open(result2_file).read())
+    start_date2 = datetime.strptime(result2['result'].pop('date'), '%Y-%m-%dT%H:%M:%S')
+
+    expected_result = {
+      u'title': u'promise_2', u'name': u'promise_2',
+      u'result': {
+        u'failed': False, u'message': u'success', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s' % promise2,
+    }
     self.assertEquals(expected_result, result2)
-    self.assertNotEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
-      change_date2.strftime('%Y-%m-%d %H:%M:%S'))
 
-  def test_report_OK(self):
-    
-    promise = self.writePromiseOK('sample_report')
-    parser = self.getUniquePromiseParser('sample_report', promise, 'report')
-    promise_runner = RunPromise(parser)
-    promise_runner.runPromise()
+  def test_promise_NOK(self):
+    promise = self.writePromiseNOK('promise_1')
+    parser = self.getPromiseParser()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    result_file = os.path.join(self.private_dir, 'sample_report.report.json')
+    result_file = os.path.join(self.output_dir, 'promise_1.status.json')
     self.assertTrue(os.path.exists(result_file))
     result1 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time = result1.pop('change-time', 0)
-    change_date = datetime.fromtimestamp(change_time)
-    start_date = result1.pop('start-date')
-    execution_time = result1.pop('execution-time')
-    
-    expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
-      u'title': u'sample_report', u'instance': u'Monitor',
-      u'_links': 
-      {u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
-      u'message': u'success\n', u'type': u'report',
-      u'portal-type': u'report', u'returncode': 0}
+    result1['result'].pop('date')
+    expected_result = {
+      u'title': u'promise_1', u'name': u'promise_1',
+      u'result': {
+        u'failed': True, u'message': u'failed', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s' % promise,
+    }
     self.assertEquals(expected_result, result1)
 
     # second run
-    promise_runner.runPromise()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
     result2 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time2 = result2.pop('change-time', 2)
-    result2.pop('start-date')
-    result2.pop('execution-time')
-    change_date2 = datetime.fromtimestamp(change_time2)
+    result2['result'].pop('date')
     self.assertEquals(expected_result, result2)
-    self.assertEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
-      change_date2.strftime('%Y-%m-%d %H:%M:%S'))
-
-    history_file = os.path.join(self.public_dir, 'sample_report.history.json')
-    self.assertTrue(os.path.exists(history_file))
-    history = json.load(open(history_file))
 
+  def test_promise_mixed(self):
+    promise = self.writePromiseOK('promise_1')
+    parser = self.getPromiseParser()
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
-    result1['change-time'] = change_time
-    result1['start-date'] = start_date
-    result1['execution-time'] = execution_time
-    #result1.pop('_links')
-    self.assertEquals(history, result1)
-
-  def test_report_mixed(self):
-    
-    promise = self.writePromiseOK('sample_report')
-    parser = self.getUniquePromiseParser('sample_report', promise, 'report')
-    promise_runner = RunPromise(parser)
-    promise_runner.runPromise()
-
-    result_file = os.path.join(self.private_dir, 'sample_report.report.json')
+    result_file = os.path.join(self.output_dir, 'promise_1.status.json')
     self.assertTrue(os.path.exists(result_file))
     result1 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time = result1.pop('change-time', 0)
-    change_date = datetime.fromtimestamp(change_time)
-    start_date = result1.pop('start-date')
-    execution_time = result1.pop('execution-time')
-    
-    expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
-      u'title': u'sample_report', u'instance': u'Monitor',
-      u'_links': 
-      {u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
-      u'message': u'success\n', u'type': u'report',
-      u'portal-type': u'report', u'returncode': 0}
+    result1['result'].pop('date')
+    expected_result = {
+      u'title': u'promise_1', u'name': u'promise_1',
+      u'result': {
+        u'failed': False, u'message': u'success', u'type': u'Test Result'
+        },
+      u'execution-time': 0.1,
+      u'path': u'%s' % promise,
+    }
     self.assertEquals(expected_result, result1)
 
     # second run with failure
-    time.sleep(2)
-    promise = self.writePromiseNOK('sample_report')
-    parser = self.getUniquePromiseParser('sample_report', promise, 'report')
-    expected_result['message'] = 'failed\n'
-    expected_result['status'] = 'ERROR'
-    expected_result['returncode'] = 2
-    promise_runner.runPromise()
+    time.sleep(1)
+    promise = self.writePromiseNOK('promise_1')
+    parser = self.getPromiseParser(force=True)
+    expected_result['result']['message'] = u'failed'
+    expected_result['result']['failed'] = True
+    promise_runner = MonitorPromiseLauncher(parser)
+    promise_runner.start()
 
     result2 = json.loads(open(result_file).read().decode("utf-8"))
-    change_time2 = result2.pop('change-time')
-    result2.pop('start-date')
-    result2.pop('execution-time')
-    change_date2 = datetime.fromtimestamp(change_time2)
+    result2['result'].pop('date')
     self.assertEquals(expected_result, result2)
-    self.assertNotEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
-      change_date2.strftime('%Y-%m-%d %H:%M:%S'))
-