monitor stack: reduce amount of history files

stack/monitor/scripts/globalstate.py

@@ -104,6 +104,7 @@ def main(args_list):
         'success': success,
         'warning': warning,
       },
+      type='global',
       date=latest_date,
       _links={"rss_url": {"href": "%s/public/feed" % base_url},
               "public_url": {"href": "%s/share/jio_public/" % base_url},

stack/monitor/scripts/monitor.py

@@ -97,7 +97,6 @@ class Monitoring(object):
     self.report_script_folder = config.get("monitor", "report-folder")
     self.webdav_url = '%s/share' % config.get("monitor", "base-url")
     self.public_url = '%s/public' % config.get("monitor", "base-url")
-    self.status_history_folder = os.path.join(self.public_folder, 'history')
     self.python = config.get("monitor", "python") or "python"
     self.public_path_list = config.get("monitor", "public-path-list").split()
     self.private_path_list = config.get("monitor", "private-path-list").split()
@@ -107,7 +106,7 @@ class Monitoring(object):
     self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
 
     self.config_folder = os.path.join(self.private_folder, 'config')
-    self.report_folder = os.path.join(self.private_folder, 'report')
+    self.report_folder = self.private_folder
 
     self.promise_dict = {}
     for promise_folder in self.promise_folder_list:
@@ -237,14 +236,14 @@ class Monitoring(object):
       else:
         response = urllib2.urlopen(url)
     except urllib2.HTTPError:
-      return 'Unknow Instance'
+      return 'Unknown Instance'
     else:
       try:
         monitor_dict = json.loads(response.read())
-        return monitor_dict.get('title', 'Unknow Instance')
+        return monitor_dict.get('title', 'Unknown Instance')
       except ValueError, e:
         print "Bad Json file at %s" % url
-    return 'Unknow Instance'
+    return 'Unknown Instance'
 
   def configureFolders(self):
     # configure public and private folder
@@ -256,7 +255,6 @@ class Monitoring(object):
     jio_private = os.path.join(self.webdav_folder, 'jio_private')
     mkdirAll(jio_public)
     mkdirAll(jio_private)
-    mkdirAll(self.status_history_folder)
     try:
       os.symlink(self.public_folder, os.path.join(jio_public, '.jio_documents'))
     except OSError, e:
@@ -269,11 +267,9 @@ class Monitoring(object):
         raise
 
     self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
-    self.report_folder = os.path.join(self.report_folder, '.jio_documents')
     config_folder = os.path.join(self.config_folder, '.jio_documents')
     mkdirAll(self.data_folder)
     mkdirAll(config_folder)
-    mkdirAll(self.report_folder)
     try:
       os.symlink(os.path.join(self.private_folder, 'data'),
                   os.path.join(jio_private, 'data'))
@@ -285,11 +281,6 @@ class Monitoring(object):
     except OSError, e:
       if e.errno != os.errno.EEXIST:
         raise
-    try:
-      os.symlink(self.report_folder, os.path.join(jio_private, 'report'))
-    except OSError, e:
-      if e.errno != os.errno.EEXIST:
-        raise
 
   def makeConfigurationFiles(self):
     config_folder = os.path.join(self.config_folder, '.jio_documents')
@@ -397,7 +388,7 @@ class Monitoring(object):
   def generateReportCronEntries(self):
     cron_line_list = []
     # We should add the possibility to modify this parameter later from monitor interface
-    report_frequency = "*/30 * * * *"
+    report_frequency = "*/20 * * * *"
 
     report_name_list = [name.replace('.report.json', '')
       for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
@@ -451,7 +442,7 @@ class Monitoring(object):
 
     for service_name, promise in self.promise_items:
       service_config = promise["configuration"]
-      service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)  # hardcoded
+      service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)
       mkdirAll(os.path.dirname(service_status_path))
 
       promise_cmd_line = [
@@ -463,7 +454,7 @@ class Monitoring(object):
         '--promise_script "%s"' % promise["path"],
         '--promise_name "%s"' % service_name,
         '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
-        '--history_folder "%s"' % self.status_history_folder,
+        '--history_folder "%s"' % self.public_folder,
         '--instance_name "%s"' % self.title,
         '--hosting_name "%s"' % self.root_title]
 
@@ -539,13 +530,21 @@ class Monitoring(object):
 
     # Rotate monitor data files
     option_list = [
-      'daily', 'nocreate', 'noolddir', 'rotate 30',
+      'daily', 'nocreate', 'noolddir', 'rotate 5',
       'nocompress', 'extension .json', 'dateext',
       'dateformat -%Y-%m-%d', 'notifempty'
     ]
     file_list = ["%s/*.data.json" % self.data_folder]
     self.generateLogrotateEntry('monitor.data', file_list, option_list)
 
+    # Rotate public history status file, delete data of previous days
+    option_list = [
+      'daily', 'nocreate', 'rotate 0',
+      'nocompress', 'notifempty'
+    ]
+    file_list = ["%s/*.history.json" % self.public_folder]
+    self.generateLogrotateEntry('monitor.service.status', file_list, option_list)
+
     # Add cron entry for SlapOS Collect
     command = "%s %s --output_folder %s --collector_db %s" % (self.python,
       self.collect_script, self.data_folder, self.collector_db)

stack/monitor/scripts/run-promise.py

@@ -65,6 +65,7 @@ def main():
   status_json['title'] = parser.promise_name
   status_json['instance'] = parser.instance_name
   status_json['hosting_subscription'] = parser.hosting_name
+  status_json['type'] = parser.promise_type
 
   # Save the lastest status change date (needed for rss)
   status_json['change-time'] = ps_process.create_time()
@@ -85,9 +86,7 @@ def main():
   os.remove(parser.pid_path)
 
 def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
-  old_history_list = []
-  keep_item_amount = 25
-  history_path = os.path.join(history_folder, name, '.jio_documents')
+  history_path = os.path.join(history_folder)
   if not os.path.exists(status_file):
     return
   if not os.path.exists(history_folder):
@@ -101,25 +100,46 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
       else: raise
   with open(status_file, 'r') as sf:
     status_dict = json.loads(sf.read())
-    filename = '%s.%s.json' % (
-      status_dict['start-date'].replace(' ', '_').replace(':', ''),
-      promise_type)
+  
+  if promise_type == 'status':
+    filename = '%s.history.json' % name
+    history_file = os.path.join(history_path, filename)
+    # Remove links from history (not needed)
+    status_dict.pop('_links', None)
+    if not os.path.exists(history_file):
+      with open(history_file, 'w') as f_history:
+        data_dict = {
+          "date": time.time(),
+          "data": [status_dict]
+        }
+        f_history.write(json.dumps(data_dict))
+    else:
+      with open (history_file, mode="r+") as f_history:
+        f_history.seek(0,2)
+        position = f_history.tell() -2
+        f_history.seek(position)
+        #f_history.write(',%s]}' % str(status_dict))
+        f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
+  elif promise_type == 'report':
+    # keep_item_amount = 3
+    filename = '%s.history.json' % (
+      name)
 
-  copyfile(status_file, os.path.join(history_path, filename))
-  # Don't let history foler grow too much, keep xx files
-  file_list = filter(os.path.isfile,
-      glob.glob("%s/*.%s.json" % (history_path, promise_type))
-    )
-  file_count = len(file_list)
-  if file_count > keep_item_amount:
-    file_list.sort(key=lambda x: os.path.getmtime(x))
-    while file_count > keep_item_amount:
-      to_delete = file_list.pop(0)
-      try:
-        os.unlink(to_delete)
-        file_count -= 1
-      except OSError:
-        raise
+    copyfile(status_file, os.path.join(history_path, filename))
+    """# Don't let history foler grow too much, keep xx files
+    file_list = filter(os.path.isfile,
+        glob.glob("%s/*.%s.history.json" % (history_path, promise_type))
+      )
+    file_count = len(file_list)
+    if file_count > keep_item_amount:
+      file_list.sort(key=lambda x: os.path.getmtime(x))
+      while file_count > keep_item_amount:
+        to_delete = file_list.pop(0)
+        try:
+          os.unlink(to_delete)
+          file_count -= 1
+        except OSError:
+          raise"""
 
 def generateStatusJsonFromProcess(process, start_date=None, title=None):
   stdout, stderr = process.communicate()