diff --git a/stack/monitor/buildout.cfg b/stack/monitor/buildout.cfg index c69b63f1b5fc24bfcbbd5e5bc26f639dc6579618..e4eabe87aad6157f919683fe541177672bbd44b9 100644 --- a/stack/monitor/buildout.cfg +++ b/stack/monitor/buildout.cfg @@ -30,6 +30,7 @@ eggs = PyRSS2Gen Jinja2 APacheDEX + pyparsing [make-rss-script] recipe = slapos.recipe.template @@ -43,7 +44,7 @@ recipe = slapos.recipe.template url = ${:_profile_base_location_}/monitor.cfg.in output = ${buildout:directory}/monitor.cfg filename = monitor.cfg -md5sum = af5c10c05b03b59a49187fd3e151eb43 +md5sum = 377112bd0e7e5f17f2d69950fac872ed mode = 0644 [monitor-bin] @@ -134,6 +135,22 @@ md5sum = 00b230e1dee6e7f25d2050f6f0ae39e2 filename = run-apachedex.py.in mode = 0644 +[log-tools] +recipe = hexagonit.recipe.download +url = ${:_profile_base_location_}/${:filename} +download-only = true +md5sum = 8add29b36060967209e54732967e8d80 +filename = logTools.py +mode = 0644 + +[errorlog-2rss] +recipe = hexagonit.recipe.download +url = ${:_profile_base_location_}/${:filename} +download-only = true +md5sum = 9617c1a06ef81c5141c8b8d08a985939 +filename = errorlog2rss.py.in +mode = 0644 + [dcron-service] recipe = slapos.recipe.template url = ${template-dcron-service:output} diff --git a/stack/monitor/errorlog2rss.py.in b/stack/monitor/errorlog2rss.py.in new file mode 100644 index 0000000000000000000000000000000000000000..b6ed66c07ce41f35544401ad1a24aad4e5dfbf87 --- /dev/null +++ b/stack/monitor/errorlog2rss.py.in @@ -0,0 +1,69 @@ +#!{{ python_executable }} +# BEWARE: This file is operated by slapgrid +# BEWARE: It will be overwritten automatically + +import sys +import os +import subprocess +from datetime import datetime, timedelta + +type = "{{ type }}".strip() +log_file_list = {{ log_file_list }} +module_folder = "{{ log_tool_folder }}".strip() +output_folder = "{{ log_as_rss_ouptut }}".strip() +db_path = "{{ db_location }}".strip() +base_link = "{{ base_link }}".strip() +interval = {{ hour_interval }} +rotate_time = {{ rotate_time }} +limit = {{ item_limit }} +PARSE_TYPE_LIST = ['zope'] + +if not os.path.exists(module_folder) or not os.path.isdir(module_folder): + print "ERROR: Could not load log parsing Module. %s is not a directory. Exiting..." % module_folder + exit(1) + +if not os.path.exists(output_folder) or not os.path.isdir(output_folder): + print "ERROR: %s is not a directory. Exiting..." % output_folder + exit(1) + +if not type in PARSE_TYPE_LIST: + print "ERROR: Could not parse specified log file type. Exiting..." % output_folder + exit(1) + +sys.path.append(module_folder) +import logTools + +for name, log_path in log_file_list.items(): + if not os.path.exists(log_path): + print "WARNING: File %s not found..." % log_path + continue + rss_path = os.path.join(output_folder, name) + if not rss_path.endswith('.html'): + rss_path += '.html' + start_date = (datetime.now() - + timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S') + to_date = (datetime.now() - + timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S') + item_list = [] + if type == 'zope': + method = logTools.isZopeLogBeginLine + zope_parser = logTools.getZopeParser() + item_list = logTools.parseLog(log_path, zope_parser, method, + filter_with="ERROR", start_date=start_date) + if item_list: + start = 0 + size = len(item_list) + if limit and size > limit: + start = size - limit - 1 + print "Found %s log entries matching date upper than %s..." % (size, start_date) + + logTools.insertRssDb(db_path, item_list[start:-1], name) + print "%s items inserted into database..." % limit + + logTools.truncateRssDb(db_path, to_date) + + print "Generating RSS entries with %s items..." % limit + logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit) + + + diff --git a/stack/monitor/logTools.py b/stack/monitor/logTools.py new file mode 100644 index 0000000000000000000000000000000000000000..3cdfbe85d8e3b9c658be008ef1592efb1d6b6b09 --- /dev/null +++ b/stack/monitor/logTools.py @@ -0,0 +1,147 @@ +from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex +import os, re +import datetime +import uuid +import base64 +import sqlite3 +import PyRSS2Gen + +def init_db(db_path): + db = sqlite3.connect(db_path) + c = db.cursor() + c.executescript(""" +CREATE TABLE IF NOT EXISTS rss_entry ( + name VARCHAR(25), + datetime VARCHAR(15), + status VARCHAR(20), + method VARCHAR(25), + title VARCHAR(255), + content VARCHAR(255)); +""") + db.commit() + db.close() + +def getZopeParser(): + integer = Word(nums) + serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + integer + ":" + integer + ":" + integer + "," + integer) + status = Word(string.uppercase, max=7, min=3) + word = Word( alphas+nums+"@._-" ) + message = Regex(".*") + bnf = serverDateTime.setResultsName("timestamp") + status.setResultsName("statusCode") + \ + word.setResultsName("method") + message.setResultsName("content") + return bnf + +def isZopeLogBeginLine(line): + # This expression will check if line start with a date string + # XXX - if line match expression, then regex.group() return the date + if not line or line.strip() == "------": + return None + regex = re.match(r"(^\d{2,4}-\d{2}-\d{1,2}\s+\d{2}:\d{2}:\d{2}?[,\d]+)", + line) + return regex + + +def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_format=""): + if not os.path.exists(path): + print "ERROR: cannot get file: %s" % path + return [] + log_result = [] + if not date_format: + date_format = "%Y-%m-%d %H:%M:%S,%f" + with open(path, 'r') as logfile: + index = 0 + for line in logfile: + regex = method(line) + if not regex: + if index == 0 or line.strip() == "------": + continue + # Add this line to log content + log_result[index - 1]['content'] += ("\n" + line) + else: + try: + fields = parserbnf.parseString(line) + if filter_with and not fields.statusCode == filter_with: + continue + if start_date and regex.group() < start_date: + continue + log_result.append(dict(datetime=datetime.datetime.strptime( + fields.timestamp , date_format), + status=fields.get('statusCode', ''), + method=fields.get('method', ''), + title=fields.content, + content=fields.content)) + index += 1 + except Exception: + raise + # print "WARNING: Could not parse log line. %s \n << %s >>" % (str(e), line) + return log_result + +def insertRssDb(db_path, entry_list, rss_name): + init_db(db_path) + db = sqlite3.connect(db_path) + for entry in entry_list: + date = entry['datetime'].strftime('%Y-%m-%d %H:%M:%S') + db.execute("insert into rss_entry(name, datetime, status, method, title, content) values (?, ?, ?, ?, ?, ?)", + (rss_name, date, entry['status'], entry['method'], entry['title'], entry['content'])) + db.commit() + db.close() + +def truncateRssDb(db_path, to_date): + db = sqlite3.connect(db_path) + db.execute("delete from rss_entry where datetime<?", (to_date,)) + db.commit() + db.close() + +def selectRssDb(db_path, rss_name, start_date, limit=0): + db = sqlite3.connect(db_path) + query = "select name, datetime, status, method, title, content from rss_entry " + query += "where name=? and datetime>=? order by datetime DESC" + if limit: + query += " limit ?" + rows = db.execute(query, (rss_name, start_date, limit)) + else: + rows = db.execute(query, (rss_name, start_date)) + #db.close() + if rows: + return rows + return [] + +def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0): + items = [] + + db = sqlite3.connect(db_path) + query = "select name, datetime, status, method, title, content from rss_entry " + query += "where name=? and datetime>=? order by datetime DESC" + if limit: + query += " limit ?" + entry_list = db.execute(query, (name, start_date, limit)) + else: + entry_list = db.execute(query, (name, start_date)) + + for entry in entry_list: + name, rss_date, status, method, title, content = entry + if method: + title = "[%s] %s" % (method, title) + title = "[%s] %s" % (status, title) + rss_item = PyRSS2Gen.RSSItem( + title = title, + link = "", + description = content.replace('\n', '<br/>'), + pubDate = rss_date, + guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (rss_date, url_link))) + ) + items.append(rss_item) + db.close() + + ### Build the rss feed + items.reverse() + rss_feed = PyRSS2Gen.RSS2 ( + title = name, + link = url_link, + description = name, + lastBuildDate = datetime.datetime.utcnow(), + items = items + ) + + with open(rss_path, 'w') as rss_ouput: + rss_ouput.write(rss_feed.to_xml()) diff --git a/stack/monitor/monitor.cfg.in b/stack/monitor/monitor.cfg.in index e41ebee03d36987f9e56c72ea3235eb729a28751..e39b4ddc00272b27eb96aedc4b193c573f6b9b2e 100644 --- a/stack/monitor/monitor.cfg.in +++ b/stack/monitor/monitor.cfg.in @@ -19,6 +19,7 @@ index-filename = index.cgi index-path = $${monitor-directory:www}/$${:index-filename} db-path = $${monitor-directory:etc}/monitor.db monitor-password-path = $${monitor-directory:etc}/.monitor.shadow +log-rss-directory = $${monitor-directory:log-rss-directory} [monitor-directory] recipe = slapos.cookbook:mkdirectory @@ -53,6 +54,7 @@ monitor-result = $${:var}/monitor apachedex-result = $${:srv}/apachedex private-directory = $${:srv}/monitor-private +log-rss-directory = $${:srv}/ERRORLogAsRSS [public-symlink] recipe = cns.recipe.symlink @@ -189,7 +191,51 @@ context = recipe = plone.recipe.command command = ln -s $${:source} $${monitor-directory:private-directory} source = - + +# ErrorLog URL As RSS +[error-log-rss-base] +recipe = slapos.recipe.template:jinja2 +template = ${errorlog-2rss:location}/${errorlog-2rss:filename} +rendered = $${monitor-directory:bin}/$${:script-name} +mode = 0700 +extensions = jinja2.ext.do +base-link = $${monitor-parameters:url}/private/ +extra-context = +context = + raw python_executable ${buildout:directory}/bin/${extra-eggs:interpreter} + raw log_tool_folder ${log-tools:location} + raw db_location $${monitor-directory:etc}/$${:script-name}.db + key log_as_rss_ouptut monitor-directory:log-rss-directory + key base_link :base-link + $${:extra-context} + +[log-as-rss-symlink] +< = monitor-directory-access +source = $${monitor-directory:log-rss-directory} + +[error-log-rss] +<= error-log-rss-base +script-name = fileLog_AsRSS +# rotate_time: Max time (in hour) before remove an rss entry from database +# item_limit: Max number of rss entries to generate per cycle +extra-context = + section log_file_list error-log-list + raw item_limit 10 + raw hour_interval 1 + raw rotate_time 24 + raw type zope + +[error-log-list] +# XXX - error-log-name = FILE_PATH + +[cron-entry-logAsRss] +<= cron +recipe = slapos.cookbook:cron.d +name = $${error-log-rss:script-name} +frequency = 30 * * * * +command = $${error-log-rss:rendered} + +# Apache Dex [apachedex-entries-base] recipe = slapos.recipe.template:jinja2 template = ${run-apachedex:location}/${run-apachedex:filename} diff --git a/stack/monitor/monitor.py.in b/stack/monitor/monitor.py.in index ac007c798a84ea25edbaf27de69a1ddafbf8173a..81bb657df2bce5cfbc8cb33aeaba0a4ccf6c141b 100644 --- a/stack/monitor/monitor.py.in +++ b/stack/monitor/monitor.py.in @@ -64,6 +64,13 @@ def init_db(db): CREATE TABLE IF NOT EXISTS status ( timestamp INTEGER UNIQUE, status VARCHAR(255)); +CREATE TABLE IF NOT EXISTS rss_entry ( + name VARCHAR(255), + timestamp INTEGER, + status VARCHAR(255), + method VARCHAR(255), + title VARCHAR(255), + content VARCHAR(255)); CREATE TABLE IF NOT EXISTS individual_status ( timestamp INTEGER, status VARCHAR(255),