Skip to content

S
slapos.toolbox
Commit 0177ace5 authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

monitor: Escape title when generating OPML

parent 1c2a9658
Hide whitespace changes
Inline Side-by-side
Showing with 379 additions and 379 deletions
slapos/monitor/monitor.py
View file @ 0177ace5
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import sys import sys
import os import os
import stat import stat
import json import json
import ConfigParser import ConfigParser
import traceback import traceback
import argparse import argparse
import urllib2 import urllib2
import ssl import ssl
import glob import glob
import socket import socket
from datetime import datetime from datetime import datetime
from xml.sax.saxutils import escape from xml.sax.saxutils import escape
OPML_START = """<?xml version="1.0" encoding="UTF-8"?> OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
<!-- OPML generated by SlapOS --> <!-- OPML generated by SlapOS -->
<opml version="1.1"> <opml version="1.1">
<head> <head>
<title>%(root_title)s</title> <title>%(root_title)s</title>
<dateCreated>%(creation_date)s</dateCreated> <dateCreated>%(creation_date)s</dateCreated>
<dateModified>%(modification_date)s</dateModified> <dateModified>%(modification_date)s</dateModified>
</head> </head>
<body> <body>
<outline text="%(outline_title)s">""" <outline text="%(outline_title)s">"""
OPML_END = """ </outline> OPML_END = """ </outline>
</body> </body>
</opml>""" </opml>"""
OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" url="%(global_url)s" />' OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" url="%(global_url)s" />'
def parseArguments(): def parseArguments():
""" """
Parse arguments for monitor instance. Parse arguments for monitor instance.
""" """
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config-file', required=True, parser.add_argument('-c', '--config-file', required=True,
default='monitor.cfg', default='monitor.cfg',
help='Monitor Configuration file') help='Monitor Configuration file')
return parser.parse_args() return parser.parse_args()
def mkdirAll(path): def mkdirAll(path):
try: try:
os.makedirs(path) os.makedirs(path)
except OSError, e: except OSError, e:
if e.errno == os.errno.EEXIST and os.path.isdir(path): if e.errno == os.errno.EEXIST and os.path.isdir(path):
pass pass
else: raise else: raise
def softConfigGet(config, *args, **kwargs): def softConfigGet(config, *args, **kwargs):
try: try:
return config.get(*args, **kwargs) return config.get(*args, **kwargs)
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
return None return None
def createSymlink(source, destination): def createSymlink(source, destination):
try: try:
os.symlink(source, destination) os.symlink(source, destination)
except OSError, e: except OSError, e:
if e.errno != os.errno.EEXIST: if e.errno != os.errno.EEXIST:
raise raise
class Monitoring(object): class Monitoring(object):
def __init__(self, configuration_file): def __init__(self, configuration_file):
self._config_file = configuration_file self._config_file = configuration_file
config = self.loadConfig([configuration_file]) config = self.loadConfig([configuration_file])
# Set Monitor variables # Set Monitor variables
self.title = config.get("monitor", "title") self.title = config.get("monitor", "title")
self.root_title = config.get("monitor", "root-title") self.root_title = config.get("monitor", "root-title")
self.service_pid_folder = config.get("monitor", "service-pid-folder") self.service_pid_folder = config.get("monitor", "service-pid-folder")
self.crond_folder = config.get("monitor", "crond-folder") self.crond_folder = config.get("monitor", "crond-folder")
self.public_folder = config.get("monitor", "public-folder") self.public_folder = config.get("monitor", "public-folder")
self.private_folder = config.get("monitor", "private-folder") self.private_folder = config.get("monitor", "private-folder")
self.webdav_folder = config.get("monitor", "webdav-folder") self.webdav_folder = config.get("monitor", "webdav-folder")
self.webdav_url = '%s/share' % config.get("monitor", "base-url") self.webdav_url = '%s/share' % config.get("monitor", "base-url")
self.public_url = '%s/public' % config.get("monitor", "base-url") self.public_url = '%s/public' % config.get("monitor", "base-url")
self.public_path_list = config.get("monitor", "public-path-list").split() self.public_path_list = config.get("monitor", "public-path-list").split()
self.private_path_list = config.get("monitor", "private-path-list").split() self.private_path_list = config.get("monitor", "private-path-list").split()
self.monitor_url_list = config.get("monitor", "monitor-url-list").split() self.monitor_url_list = config.get("monitor", "monitor-url-list").split()
self.parameter_list = [param.strip() for param in config.get("monitor", "parameter-list").split('\n') if param] self.parameter_list = [param.strip() for param in config.get("monitor", "parameter-list").split('\n') if param]
# Use this file to write knowledge0_cfg required by webrunner # Use this file to write knowledge0_cfg required by webrunner
self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip() self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
self.pid_file = config.get("monitor", "pid-file") self.pid_file = config.get("monitor", "pid-file")
self.promise_output_file = config.get("monitor", "promise-output-file") self.promise_output_file = config.get("monitor", "promise-output-file")
self.promise_folder = config.get("promises", 'promise-folder') self.promise_folder = config.get("promises", 'promise-folder')
self.legacy_promise_folder = config.get("promises", 'legacy-promise-folder') self.legacy_promise_folder = config.get("promises", 'legacy-promise-folder')
self.promise_output = config.get("promises", 'output-folder') self.promise_output = config.get("promises", 'output-folder')
self.config_folder = os.path.join(self.private_folder, 'config') self.config_folder = os.path.join(self.private_folder, 'config')
self.data_folder = config.get("monitor", "document-folder") self.data_folder = config.get("monitor", "document-folder")
self.bootstrap_is_ok = True self.bootstrap_is_ok = True
def loadConfig(self, pathes, config=None): def loadConfig(self, pathes, config=None):
if config is None: if config is None:
config = ConfigParser.ConfigParser() config = ConfigParser.ConfigParser()
try: try:
config.read(pathes) config.read(pathes)
except ConfigParser.MissingSectionHeaderError: except ConfigParser.MissingSectionHeaderError:
traceback.print_exc() traceback.print_exc()
return config return config
def readInstanceConfiguration(self): def readInstanceConfiguration(self):
type_list = ['raw', 'file', 'htpasswd', 'httpdcors'] type_list = ['raw', 'file', 'htpasswd', 'httpdcors']
configuration_list = [] configuration_list = []
if not self.parameter_list: if not self.parameter_list:
return [] return []
for config in self.parameter_list: for config in self.parameter_list:
config_list = config.strip().split(' ') config_list = config.strip().split(' ')
# type: config_list[0] # type: config_list[0]
if len(config_list) >= 3 and config_list[0] in type_list: if len(config_list) >= 3 and config_list[0] in type_list:
if config_list[0] == 'raw': if config_list[0] == 'raw':
configuration_list.append(dict( configuration_list.append(dict(
key='', key='',
title=config_list[1], title=config_list[1],
value=' '.join(config_list[2:]) value=' '.join(config_list[2:])
)) ))
elif (config_list[0] == 'file' or config_list[0] == 'htpasswd'): elif (config_list[0] == 'file' or config_list[0] == 'htpasswd'):
directory = os.path.dirname(config_list[2]) directory = os.path.dirname(config_list[2])
if not os.path.exists(directory) or not os.access(directory, os.W_OK): if not os.path.exists(directory) or not os.access(directory, os.W_OK):
raise OSError("Directory does not exists or does not have write acess") raise OSError("Directory does not exists or does not have write acess")
if os.path.exists(config_list[2]) and os.path.isfile(config_list[2]): if os.path.exists(config_list[2]) and os.path.isfile(config_list[2]):
try: try:
with open(config_list[2]) as cfile: with open(config_list[2]) as cfile:
param_value = cfile.read() param_value = cfile.read()
except OSError, e: except OSError, e:
print 'Cannot read file %s, Error is: %s' % (config_list[2], str(e)) print 'Cannot read file %s, Error is: %s' % (config_list[2], str(e))
pass pass
else: else:
param_value = "" param_value = ""
parameter = dict( parameter = dict(
key=config_list[1], key=config_list[1],
title=config_list[1], title=config_list[1],
value=param_value, value=param_value,
description={ description={
"type": config_list[0], "type": config_list[0],
"file": config_list[2] "file": config_list[2]
} }
) )
if config_list[0] == 'htpasswd': if config_list[0] == 'htpasswd':
if len(config_list) != 5 or not os.path.exists(config_list[4]): if len(config_list) != 5 or not os.path.exists(config_list[4]):
print 'htpasswd file is not specified: %s' % str(config_list) print 'htpasswd file is not specified: %s' % str(config_list)
continue continue
parameter['description']['user'] = config_list[3] parameter['description']['user'] = config_list[3]
parameter['description']['htpasswd'] = config_list[4] parameter['description']['htpasswd'] = config_list[4]
configuration_list.append(parameter) configuration_list.append(parameter)
elif config_list[0] == 'httpdcors' and os.path.exists(config_list[2]) and \ elif config_list[0] == 'httpdcors' and os.path.exists(config_list[2]) and \
os.path.exists(config_list[3]): os.path.exists(config_list[3]):
old_cors_file = os.path.join( old_cors_file = os.path.join(
os.path.dirname(config_list[2]), os.path.dirname(config_list[2]),
'prev_%s' % os.path.basename(config_list[2]) 'prev_%s' % os.path.basename(config_list[2])
) )
try: try:
cors_content = "" cors_content = ""
if os.path.exists(old_cors_file): if os.path.exists(old_cors_file):
with open(old_cors_file) as cfile: with open(old_cors_file) as cfile:
cors_content = cfile.read() cors_content = cfile.read()
else: else:
# Create empty file # Create empty file
with open(old_cors_file, 'w') as cfile: with open(old_cors_file, 'w') as cfile:
cfile.write("") cfile.write("")
parameter = dict( parameter = dict(
key=config_list[1], key=config_list[1],
title=config_list[1], title=config_list[1],
value=cors_content, value=cors_content,
description={ description={
"type": config_list[0], "type": config_list[0],
"cors_file": config_list[2], "cors_file": config_list[2],
"gracefull_bin": config_list[3] "gracefull_bin": config_list[3]
} }
) )
configuration_list.append(parameter) configuration_list.append(parameter)
except OSError, e: except OSError, e:
print 'Cannot read file at %s, Error is: %s' % (old_cors_file, str(e)) print 'Cannot read file at %s, Error is: %s' % (old_cors_file, str(e))
pass pass
return configuration_list return configuration_list
def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""): def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""):
if destination_folder: if destination_folder:
if source_path_list: if source_path_list:
for path in source_path_list: for path in source_path_list:
path = path.rstrip('/') path = path.rstrip('/')
dirname = os.path.join(destination_folder, name) dirname = os.path.join(destination_folder, name)
try: try:
mkdirAll(dirname) # could also raise OSError mkdirAll(dirname) # could also raise OSError
os.symlink(path, os.path.join(dirname, os.path.basename(path))) os.symlink(path, os.path.join(dirname, os.path.basename(path)))
except OSError, e: except OSError, e:
if e.errno != os.errno.EEXIST: if e.errno != os.errno.EEXIST:
raise raise
def getMonitorTitleFromUrl(self, monitor_url): def getMonitorTitleFromUrl(self, monitor_url):
# This file should be generated # This file should be generated
if not monitor_url.startswith('https://') and not monitor_url.startswith('http://'): if not monitor_url.startswith('https://') and not monitor_url.startswith('http://'):
return 'Unknown Instance' return 'Unknown Instance'
if not monitor_url.endswith('/'): if not monitor_url.endswith('/'):
monitor_url = monitor_url + '/' monitor_url = monitor_url + '/'
url = monitor_url + '/monitor.global.json' url = monitor_url + '/monitor.global.json'
success = False success = False
monitor_title = 'Unknown Instance' monitor_title = 'Unknown Instance'
try: try:
# Timeout after 20 seconds # Timeout after 20 seconds
timeout = 20 timeout = 20
# XXX - working here with public url # XXX - working here with public url
if hasattr(ssl, '_create_unverified_context'): if hasattr(ssl, '_create_unverified_context'):
context = ssl._create_unverified_context() context = ssl._create_unverified_context()
response = urllib2.urlopen(url, context=context, timeout=timeout) response = urllib2.urlopen(url, context=context, timeout=timeout)
else: else:
response = urllib2.urlopen(url, timeout=timeout) response = urllib2.urlopen(url, timeout=timeout)
except urllib2.HTTPError: except urllib2.HTTPError:
print "ERROR: Failed to get Monitor configuration file at %s " % url print "ERROR: Failed to get Monitor configuration file at %s " % url
except socket.timeout, e: except socket.timeout, e:
print "ERROR: Timeout while downloading monitor config at %s " % url print "ERROR: Timeout while downloading monitor config at %s " % url
else: else:
try: try:
monitor_dict = json.loads(response.read()) monitor_dict = json.loads(response.read())
monitor_title = monitor_dict.get('title', 'Unknown Instance') monitor_title = monitor_dict.get('title', 'Unknown Instance')
success = True success = True
except ValueError, e: except ValueError, e:
print "ERROR: Json file at %s is not valid" % url print "ERROR: Json file at %s is not valid" % url
self.bootstrap_is_ok = success self.bootstrap_is_ok = success
return monitor_title return monitor_title
def configureFolders(self): def configureFolders(self):
# create symlinks from monitor.conf # create symlinks from monitor.conf
self.createSymlinksFromConfig(self.public_folder, self.public_path_list) self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
self.createSymlinksFromConfig(self.private_folder, self.private_path_list) self.createSymlinksFromConfig(self.private_folder, self.private_path_list)
# configure public and private folder # configure public and private folder
self.createSymlinksFromConfig(self.webdav_folder, [self.public_folder]) self.createSymlinksFromConfig(self.webdav_folder, [self.public_folder])
self.createSymlinksFromConfig(self.webdav_folder, [self.private_folder]) self.createSymlinksFromConfig(self.webdav_folder, [self.private_folder])
config_jio_folder = os.path.join(self.config_folder, '.jio_documents') config_jio_folder = os.path.join(self.config_folder, '.jio_documents')
mkdirAll(config_jio_folder) mkdirAll(config_jio_folder)
def makeConfigurationFiles(self): def makeConfigurationFiles(self):
config_folder = os.path.join(self.config_folder, '.jio_documents') config_folder = os.path.join(self.config_folder, '.jio_documents')
parameter_config_file = os.path.join(config_folder, 'config.parameters.json') parameter_config_file = os.path.join(config_folder, 'config.parameters.json')
parameter_file = os.path.join(config_folder, 'config.json') parameter_file = os.path.join(config_folder, 'config.json')
parameter_list = self.readInstanceConfiguration() parameter_list = self.readInstanceConfiguration()
description_dict = {} description_dict = {}
if parameter_list: if parameter_list:
for i in range(0, len(parameter_list)): for i in range(0, len(parameter_list)):
key = parameter_list[i]['key'] key = parameter_list[i]['key']
if key: if key:
description_dict[key] = parameter_list[i].pop('description') description_dict[key] = parameter_list[i].pop('description')
with open(parameter_config_file, 'w') as config_file: with open(parameter_config_file, 'w') as config_file:
config_file.write(json.dumps(description_dict)) config_file.write(json.dumps(description_dict))
with open(parameter_file, 'w') as config_file: with open(parameter_file, 'w') as config_file:
config_file.write(json.dumps(parameter_list)) config_file.write(json.dumps(parameter_list))
try: try:
with open(self.parameter_cfg_file, 'w') as pfile: with open(self.parameter_cfg_file, 'w') as pfile:
pfile.write('[public]\n') pfile.write('[public]\n')
for parameter in parameter_list: for parameter in parameter_list:
if parameter['key']: if parameter['key']:
pfile.write('%s = %s\n' % (parameter['key'], parameter['value'])) pfile.write('%s = %s\n' % (parameter['key'], parameter['value']))
except OSError, e: except OSError, e:
print "Error failed to create file %s" % self.parameter_cfg_file print "Error failed to create file %s" % self.parameter_cfg_file
pass pass
def generateOpmlFile(self, feed_url_list, output_file): def generateOpmlFile(self, feed_url_list, output_file):
if os.path.exists(output_file): if os.path.exists(output_file):
creation_date = datetime.utcfromtimestamp(os.path.getctime(output_file))\ creation_date = datetime.utcfromtimestamp(os.path.getctime(output_file))\
.strftime("%a, %d %b %Y %H:%M:%S +0000") .strftime("%a, %d %b %Y %H:%M:%S +0000")
modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
else: else:
creation_date = modification_date = datetime.utcnow()\ creation_date = modification_date = datetime.utcnow()\
.strftime("%a, %d %b %Y %H:%M:%S +0000") .strftime("%a, %d %b %Y %H:%M:%S +0000")
opml_content = OPML_START % {'creation_date': creation_date, opml_content = OPML_START % {'creation_date': creation_date,
'modification_date': modification_date, 'modification_date': modification_date,
'outline_title': 'Monitoring RSS Feed list', 'outline_title': 'Monitoring RSS Feed list',
'root_title': escape(self.root_title)} 'root_title': escape(self.root_title)}
opml_content += OPML_OUTLINE_FEED % {'title': self.title, opml_content += OPML_OUTLINE_FEED % {'title': escape(self.title),
'html_url': self.public_url + '/feed', 'html_url': self.public_url + '/feed',
'xml_url': self.public_url + '/feed', 'xml_url': self.public_url + '/feed',
'global_url': "%s/private/" % self.webdav_url} 'global_url': "%s/private/" % self.webdav_url}
for feed_url in feed_url_list: for feed_url in feed_url_list:
opml_content += OPML_OUTLINE_FEED % { opml_content += OPML_OUTLINE_FEED % {
'title': self.getMonitorTitleFromUrl(feed_url + "/share/public/"), 'title': self.getMonitorTitleFromUrl(feed_url + "/share/public/"),
'html_url': feed_url + '/public/feed', 'html_url': feed_url + '/public/feed',
'xml_url': feed_url + '/public/feed', 'xml_url': feed_url + '/public/feed',
'global_url': "%s/share/private/" % feed_url} 'global_url': "%s/share/private/" % feed_url}
opml_content += OPML_END opml_content += OPML_END
with open(output_file, 'w') as wfile: with open(output_file, 'w') as wfile:
wfile.write(opml_content) wfile.write(opml_content)
def cleanupMonitorDeprecated(self): def cleanupMonitorDeprecated(self):
# Monitor report feature is removed # Monitor report feature is removed
cleanup_file_list = glob.glob("%s/*.history.json" % self.private_folder) cleanup_file_list = glob.glob("%s/*.history.json" % self.private_folder)
cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.private_folder)) cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.private_folder))
cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.data_folder)) cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.data_folder))
cleanup_file_list.extend(glob.glob("%s/*.history.json" % self.data_folder)) cleanup_file_list.extend(glob.glob("%s/*.history.json" % self.data_folder))
cleanup_file_list.extend(glob.glob("%s/*.status.json" % self.public_folder)) cleanup_file_list.extend(glob.glob("%s/*.status.json" % self.public_folder))
cleanup_file_list.append(self.crond_folder + "/monitor-reports") cleanup_file_list.append(self.crond_folder + "/monitor-reports")
cleanup_file_list.append(self.crond_folder + "/monitor-promises") cleanup_file_list.append(self.crond_folder + "/monitor-promises")
for file in cleanup_file_list: for file in cleanup_file_list:
try: try:
if os.path.exists(file): if os.path.exists(file):
os.unlink(file) os.unlink(file)
except OSError, e: except OSError, e:
print "failed to remove file %s." % file, str(e) print "failed to remove file %s." % file, str(e)
# cleanup result of promises that was removed # cleanup result of promises that was removed
promise_list = os.listdir(self.legacy_promise_folder) promise_list = os.listdir(self.legacy_promise_folder)
for name in os.listdir(self.promise_folder): for name in os.listdir(self.promise_folder):
if name.startswith('__init__'): if name.startswith('__init__'):
continue continue
promise_list.append(os.path.splitext(name)[0]) promise_list.append(os.path.splitext(name)[0])
for name in os.listdir(self.promise_output): for name in os.listdir(self.promise_output):
if not name.endswith('.status.json'): if not name.endswith('.status.json'):
continue continue
try: try:
position = promise_list.index(name.replace('.status.json', '')) position = promise_list.index(name.replace('.status.json', ''))
except ValueError: except ValueError:
status_path = os.path.join(self.promise_output, name) status_path = os.path.join(self.promise_output, name)
if os.path.exists(status_path): if os.path.exists(status_path):
try: try:
os.unlink(status_path) os.unlink(status_path)
except OSError, e: except OSError, e:
print "Error: Failed to delete %s" % status_path, str(e) print "Error: Failed to delete %s" % status_path, str(e)
else: else:
promise_list.pop(position) promise_list.pop(position)
def bootstrapMonitor(self): def bootstrapMonitor(self):
if os.path.exists(self.promise_output_file): if os.path.exists(self.promise_output_file):
os.unlink(self.promise_output_file) os.unlink(self.promise_output_file)
# save pid of current process into file # save pid of current process into file
with open(self.pid_file, 'w') as pid_file: with open(self.pid_file, 'w') as pid_file:
pid_file.write(str(os.getpid())) pid_file.write(str(os.getpid()))
self.configureFolders() self.configureFolders()
# Generate OPML file # Generate OPML file
self.generateOpmlFile(self.monitor_url_list, self.generateOpmlFile(self.monitor_url_list,
os.path.join(self.public_folder, 'feeds')) os.path.join(self.public_folder, 'feeds'))
# cleanup deprecated entries # cleanup deprecated entries
self.cleanupMonitorDeprecated() self.cleanupMonitorDeprecated()
# Generate parameters files and scripts # Generate parameters files and scripts
self.makeConfigurationFiles() self.makeConfigurationFiles()
# Write an empty file when monitor bootstrap went until the end # Write an empty file when monitor bootstrap went until the end
if self.bootstrap_is_ok: if self.bootstrap_is_ok:
with open(self.promise_output_file, 'w') as promise_file: with open(self.promise_output_file, 'w') as promise_file:
promise_file.write("") promise_file.write("")
print "SUCCESS: bootstrap is OK" print "SUCCESS: bootstrap is OK"
return 0 return 0
def main(): def main():
parser = parseArguments() parser = parseArguments()
monitor = Monitoring(parser.config_file) monitor = Monitoring(parser.config_file)
sys.exit(monitor.bootstrapMonitor()) sys.exit(monitor.bootstrapMonitor())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7