#!/usr/bin/env python import sys import os import stat import json import ConfigParser import traceback import argparse import time import glob import urllib2 import ssl from datetime import datetime OPML_START = """<?xml version="1.0" encoding="UTF-8"?> <!-- OPML generated by SlapOS --> <opml version="1.1"> <head> <title>%(root_title)s</title> <dateCreated>%(creation_date)s</dateCreated> <dateModified>%(modification_date)s</dateModified> </head> <body> <outline text="%(outline_title)s">""" OPML_END = """ </outline> </body> </opml>""" OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" url="%(global_url)s" />' def parseArguments(): """ Parse arguments for monitor instance. """ parser = argparse.ArgumentParser() parser.add_argument('--config_file', default='monitor.cfg', help='Monitor Configuration file') parser.add_argument('--promise-folder', action='append', dest='promise_folder_list', default=[], help='The path to get promise executable files') parser.add_argument('--public-folder', action='append', dest='public_folder', help='The path of public folder. All files in this folders will have public acess') parser.add_argument('--private-folder', action='append', dest='private_folder', help='The path of private folder. All files in this folders will be accessible with password') parser.add_argument('--promise-runner', help='The path of promise runner, use to run promise files') parser.add_argument('--wrapper-path', help='Path of monitor generated promise scripts files.') return parser.parse_args() def mkdirAll(path): try: os.makedirs(path) except OSError, e: if e.errno == os.errno.EEXIST and os.path.isdir(path): pass else: raise def softConfigGet(config, *args, **kwargs): try: return config.get(*args, **kwargs) except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): return None class Monitoring(object): def __init__(self, configuration_file): config = self.loadConfig([configuration_file]) # Set Monitor variables self.monitor_hal_json = config.get("monitor", "monitor-hal-json") self.title = config.get("monitor", "title") self.root_title = config.get("monitor", "root-title") self.service_pid_folder = config.get("monitor", "service-pid-folder") self.crond_folder = config.get("monitor", "crond-folder") self.logrotate_d = config.get("monitor", "logrotate-folder") self.promise_runner = config.get("monitor", "promise-runner") self.promise_folder_list = config.get("monitor", "promise-folder-list").split() self.public_folder = config.get("monitor", "public-folder") self.private_folder = config.get("monitor", "private-folder") self.collector_db = config.get("monitor", "collector-db") self.collect_script = config.get("monitor", "collect-script") self.webdav_folder = config.get("monitor", "webdav-folder") self.webdav_url = '%s/share' % config.get("monitor", "base-url") self.public_url = '%s/public' % config.get("monitor", "base-url") self.status_history_folder = os.path.join(self.public_folder, 'history') self.python = config.get("monitor", "python") or "python" self.public_path_list = config.get("monitor", "public-path-list").split() self.private_path_list = config.get("monitor", "private-path-list").split() self.monitor_url_list = config.get("monitor", "monitor-url-list").split() self.parameter_list = [param.strip() for param in config.get("monitor", "parameter-list").split('\n') if param] # Use this file to write knowledge0_cfg required by webrunner self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip() self.config_folder = os.path.join(self.private_folder, 'config') self.promise_dict = {} for promise_folder in self.promise_folder_list: self.setupPromiseDictFromFolder(promise_folder) def loadConfig(self, pathes, config=None): if config is None: config = ConfigParser.ConfigParser() try: config.read(pathes) except ConfigParser.MissingSectionHeaderError: traceback.print_exc() return config def readInstanceConfiguration(self): type_list = ['raw', 'file', 'htpasswd'] configuration_list = [] if not self.parameter_list: return [] for config in self.parameter_list: config_list = config.strip().split(' ') # type: config_list[0] if len(config_list) >= 3 and config_list[0] in type_list: if config_list[0] == 'raw': configuration_list.append(dict( key='', title=config_list[1], value=' '.join(config_list[2:]) )) elif (config_list[0] == 'file' or config_list[0] == 'htpasswd') and \ os.path.exists(config_list[2]) and os.path.isfile(config_list[2]): try: with open(config_list[2]) as cfile: parameter = dict( key=config_list[1], title=config_list[1], value=cfile.read(), description={ "type": config_list[0], "file": config_list[2] } ) if config_list[0] == 'htpasswd': if len(config_list) != 5 or not os.path.exists(config_list[4]): print 'htpasswd file is not specified: %s' % str(config_list) return parameter['description']['user'] = config_list[3] parameter['description']['htpasswd'] = config_list[4] configuration_list.append(parameter) except OSError, e: print 'Cannot read file %s, Error is: %s' % (config_list[2], str(e)) pass return configuration_list def setupPromiseDictFromFolder(self, folder): for filename in os.listdir(folder): path = os.path.join(folder, filename) if os.path.isfile(path) and os.access(path, os.X_OK): self.promise_dict[filename] = {"path": path, "configuration": ConfigParser.ConfigParser()} # get promises configurations #for filename in os.listdir(monitor_promise_folder): # path = os.path.join(monitor_promise_folder, filename) # if os.path.isfile(path) and filename[-4:] == ".cfg": # promise_name = filename[:-4] # if promise_name in promise_dict: # loadConfig([path], promise_dict[promise_name]["configuration"]) def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""): if destination_folder: if source_path_list: for path in source_path_list: path = path.rstrip('/') dirname = os.path.join(destination_folder, name) try: mkdirAll(dirname) # could also raise OSError os.symlink(path, os.path.join(dirname, os.path.basename(path))) except OSError, e: if e.errno != os.errno.EEXIST: raise def getMonitorTitleFromUrl(self, monitor_url): # This file should be generated if not monitor_url.startswith('https://') or not monitor_url.startswith('http://'): return 'Unknow Instance' if not monitor_url.endswith('/'): monitor_url = monitor_url + '/' context = ssl._create_unverified_context() url = monitor_url + '/.jio_documents/monitor.global.json' # XXX Hard Coded path try: response = urllib2.urlopen(url, context=context) except urllib2.HTTPError: return 'Unknow Instance' else: try: monitor_dict = json.loads(response.read()) return monitor_dict.get('title', 'Unknow Instance') except ValueError, e: print "Bad Json file at %s" % url return 'Unknow Instance' def configureFolders(self): # configure public and private folder self.createSymlinksFromConfig(self.webdav_folder, [self.public_folder]) self.createSymlinksFromConfig(self.webdav_folder, [self.private_folder]) #configure jio_documents folder jio_public = os.path.join(self.webdav_folder, 'jio_public') jio_private = os.path.join(self.webdav_folder, 'jio_private') mkdirAll(jio_public) mkdirAll(jio_private) mkdirAll(self.status_history_folder) try: os.symlink(self.public_folder, os.path.join(jio_public, '.jio_documents')) except OSError, e: if e.errno != os.errno.EEXIST: raise try: os.symlink(self.private_folder, os.path.join(jio_private, '.jio_documents')) except OSError, e: if e.errno != os.errno.EEXIST: raise self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents') config_folder = os.path.join(self.config_folder, '.jio_documents') mkdirAll(self.data_folder) mkdirAll(config_folder) try: os.symlink(os.path.join(self.private_folder, 'data'), os.path.join(jio_private, 'data')) except OSError, e: if e.errno != os.errno.EEXIST: raise try: os.symlink(self.config_folder, os.path.join(jio_private, 'config')) except OSError, e: if e.errno != os.errno.EEXIST: raise def makeConfigurationFiles(self): config_folder = os.path.join(self.config_folder, '.jio_documents') parameter_config_file = os.path.join(config_folder, 'config.parameters.json') parameter_file = os.path.join(config_folder, 'config.json') #mkdirAll(config_folder) parameter_list = self.readInstanceConfiguration() description_dict = {} if parameter_list: for i in range(0, len(parameter_list)): key = parameter_list[i]['key'] if key: description_dict[key] = parameter_list[i].pop('description') with open(parameter_config_file, 'w') as config_file: config_file.write(json.dumps(description_dict)) with open(parameter_file, 'w') as config_file: config_file.write(json.dumps(parameter_list)) try: with open(self.parameter_cfg_file, 'w') as pfile: pfile.write('[public]\n') for parameter in parameter_list: if parameter['key']: pfile.write('%s = %s\n' % (parameter['key'], parameter['value'])) except OSError, e: print "Error failed to create file %s" % self.parameter_cfg_file pass def generateOpmlFile(self, feed_url_list, output_file): if os.path.exists(output_file): creation_date = datetime.fromtimestamp(os.path.getctime(output_file)).utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") else: creation_date = modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") opml_content = OPML_START % {'creation_date': creation_date, 'modification_date': modification_date, 'outline_title': 'Monitoring RSS Feed list', 'root_title': self.root_title} opml_content += OPML_OUTLINE_FEED % {'title': self.title, 'html_url': self.public_url + '/feed', 'xml_url': self.public_url + '/feed', 'global_url': "%s/jio_public/" % self.webdav_url} for feed_url in feed_url_list: opml_content += OPML_OUTLINE_FEED % {'title': self.getMonitorTitleFromUrl(feed_url + "/share/jio_public/"), 'html_url': feed_url + '/public/feed', 'xml_url': feed_url + '/public/feed', 'global_url': "%s/share/jio_public/" % feed_url} opml_content += OPML_END with open(output_file, 'w') as wfile: wfile.write(opml_content) def generateLogrotateEntry(self, name, file_list, option_list): """ Will add a new entry in logrotate.d folder. This can help to rotate data file daily """ content = "%(logfiles)s {\n%(options)s\n}\n" % { 'logfiles': ' '.join(file_list), 'options': '\n'.join(option_list) } file_path = os.path.join(self.logrotate_d, name) with open(file_path, 'w') as flog: flog.write(content) def generateMonitorHalJson(self): monitor_link_dict = {"webdav": {"href": self.webdav_url}, "public": {"href": "%s/public" % self.webdav_url}, "private": {"href": "%s/private" % self.webdav_url}, "rss": {"href": "%s/feed" % self.public_url}, "jio_public": {"href": "%s/jio_public/" % self.webdav_url}, "jio_private": {"href": "%s/jio_private/" % self.webdav_url} } if self.title: self.monitor_dict["title"] = self.title if self.monitor_url_list: monitor_link_dict["related_monitor"] = [{"href": url} for url in self.monitor_url_list] self.monitor_dict["_links"] = monitor_link_dict if self.promise_items: service_list = [] for service_name, promise in self.promise_items: service_config = promise["configuration"] tmp = softConfigGet(service_config, "service", "title") service_dict = {} service_dict["id"] = service_name service_dict["_links"] = {"status": {"href": "%s/public/%s.status.json" % (self.webdav_url, service_name)}} # hardcoded if tmp: service_dict["title"] = tmp service_list.append(service_dict) self.monitor_dict["_embedded"] = {"service": service_list} with open(self.monitor_hal_json, "w") as fp: json.dump(self.monitor_dict, fp) def generateServiceCronEntries(self): # XXX only if at least one configuration file is modified, then write in the cron #cron_line_list = ['PATH=%s\n' % os.environ['PATH']] cron_line_list = [] service_name_list = [name.replace('.status.json', '') for name in os.listdir(self.public_folder) if name.endswith('.status.json')] for service_name, promise in self.promise_items: service_config = promise["configuration"] service_status_path = "%s/%s.status.json" % (self.public_folder, service_name) # hardcoded mkdirAll(os.path.dirname(service_status_path)) promise_cmd_line = [ softConfigGet(service_config, "service", "frequency") or "* * * * *", self.promise_runner, '--pid_path %s' % os.path.join(self.service_pid_folder, "%s.pid" % service_name), '--output %s' % service_status_path, '--promise_script %s' % promise["path"], '--promise_name "%s"' % service_name, '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded, '--history_folder %s' % self.status_history_folder, '--instance_name "%s"' % self.title, '--hosting_name "%s"' % self.root_title] cron_line_list.append(' '.join(promise_cmd_line)) if service_name in service_name_list: service_name_list.pop(service_name_list.index(service_name)) """wrapper_path = os.path.join(self.wraper_folder, service_name) with open(wrapper_path, "w") as fp: fp.write("#!/bin/sh\n%s" % command) # XXX hardcoded, use dash, sh or bash binary! os.chmod(wrapper_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH )""" if service_name_list != []: # XXX Some service was removed, delete his status file so monitor will not consider his status anymore for service_name in service_name_list: status_path = os.path.join(self.public_folder, '%s.status.json' % service_name) if os.path.exists(status_path): try: os.unlink(status_path) except OSError, e: print "Error: Failed to delete %s" % status_path, str(e) pass with open(self.crond_folder + "/monitor-promises", "w") as fp: fp.write("\n".join(cron_line_list)) def addCronEntry(self, name, frequency, command): entry_line = '%s %s' % (frequency, command) cron_entry_file = os.path.join(self.crond_folder, name) with open(cron_entry_file, "w") as cronf: cronf.write(entry_line) def bootstrapMonitor(self): # create symlinks from service configurations self.promise_items = self.promise_dict.items() for service_name, promise in self.promise_items: service_config = promise["configuration"] public_path_list = softConfigGet(service_config, "service", "public-path-list") private_path_list = softConfigGet(service_config, "service", "private-path-list") if public_path_list: self.createSymlinksFromConfig(self.public_folder, public_path_list.split(), service_name) if private_path_list: self.createSymlinksFromConfig(self.private_folder, private_path_list.split(), service_name) # create symlinks from monitor.conf self.createSymlinksFromConfig(self.public_folder, self.public_path_list) self.createSymlinksFromConfig(self.private_folder, self.private_path_list) self.configureFolders() # generate monitor.json self.monitor_dict = {} self.generateMonitorHalJson() # Generate OPML file self.generateOpmlFile(self.monitor_url_list, os.path.join(self.public_folder, 'feeds')) # put promises to a cron file self.generateServiceCronEntries() # Generate parameters files and scripts self.makeConfigurationFiles() # Rotate monitor data files option_list = [ 'daily', 'nocreate', 'noolddir', 'rotate 30', 'nocompress', 'extension .json', 'dateext', 'dateformat -%Y-%m-%d', 'notifempty' ] file_list = ["%s/*.data.json" % self.data_folder] self.generateLogrotateEntry('monitor.data', file_list, option_list) # Add cron entry for SlapOS Collect command = "%s %s --output_folder %s --collector_db %s" % (self.python, self.collect_script, self.data_folder, self.collector_db) self.addCronEntry('monitor_collect', '* * * * *', command) return 0 if __name__ == "__main__": parser = parseArguments() monitor = Monitoring(parser.config_file) sys.exit(monitor.bootstrapMonitor())