Commit e4ee60fb authored by Alain Takoudjou's avatar Alain Takoudjou Committed by Tristan Cavelier

monitor: allow to generate rss from zope longrequest log

parent 2deabf02
......@@ -16,7 +16,7 @@ base_link = "{{ base_link }}".strip()
interval = {{ hour_interval }}
rotate_time = {{ rotate_time }}
limit = {{ item_limit }}
PARSE_TYPE_LIST = ['zope']
PARSE_TYPE_LIST = ['zope', 'zope-longrequest']
if not os.path.exists(module_folder) or not os.path.isdir(module_folder):
print "ERROR: Could not load log parsing Module. %s is not a directory. Exiting..." % module_folder
......@@ -33,6 +33,11 @@ if not type in PARSE_TYPE_LIST:
sys.path.append(module_folder)
import logTools
start_date = (datetime.now() -
timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
to_date = (datetime.now() -
timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
for name, log_path in log_file_list.items():
if not os.path.exists(log_path):
print "WARNING: File %s not found..." % log_path
......@@ -40,16 +45,21 @@ for name, log_path in log_file_list.items():
rss_path = os.path.join(output_folder, name)
if not rss_path.endswith('.html'):
rss_path += '.html'
start_date = (datetime.now() -
timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
to_date = (datetime.now() -
timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
item_list = []
date_format = "%Y-%m-%d %H:%M:%S,%f"
log_filter = "ERROR"
if type == 'zope':
method = logTools.isZopeLogBeginLine
zope_parser = logTools.getZopeParser()
item_list = logTools.parseLog(log_path, zope_parser, method,
filter_with="ERROR", start_date=start_date)
log_parser = logTools.getZopeParser()
elif type == 'zope-longrequest':
log_filter = ""
method = logTools.isZopeLogBeginLine
log_parser = logTools.getZopeLogRequestParser()
item_list = logTools.parseLog(log_path, log_parser, method,
filter_with=log_filter, start_date=start_date,
date_format=date_format)
if item_list:
start = 0
size = len(item_list)
......@@ -60,10 +70,16 @@ for name, log_path in log_file_list.items():
logTools.insertRssDb(db_path, item_list[start:-1], name)
print "%s items inserted into database..." % limit
logTools.truncateRssDb(db_path, to_date)
print "Generating RSS entries with %s items..." % limit
logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit)
else:
print "No log entries with date upper than %s..." % start_date
try:
# Remove olders entries from database
logTools.truncateRssDb(db_path, to_date)
except Exception,e:
print "Database not exist!! ", str(e)
from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex
from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex, Literal
import os, re
import datetime
import uuid
......@@ -16,19 +16,35 @@ CREATE TABLE IF NOT EXISTS rss_entry (
status VARCHAR(20),
method VARCHAR(25),
title VARCHAR(255),
content VARCHAR(255));
url VARCHAR(255),
content TEXT);
""")
db.commit()
db.close()
def getZopeLogRequestParser():
integer = Word(nums)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " +
integer + ":" + integer + ":" + integer + "," + integer)
word = Word( alphas+nums+"@._-" )
info = Regex("([\d\w\s:\.]+;){2}")#Combine(word + ";" + Literal(" ") + word + ";")
request = Combine(Suppress("request: ") + Suppress(word+" ") + Regex(".*"))
no_request = Combine(Suppress("[No request]") + Regex(".*"))
bnf = serverDateTime.setResultsName("timestamp") + Suppress("-") + \
info.setResultsName("title") + \
(no_request | request).setResultsName("link")
return bnf
def getZopeParser():
integer = Word(nums)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + integer + ":" + integer + ":" + integer + "," + integer)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " +
integer + ":" + integer + ":" + integer + "," + integer)
status = Word(string.uppercase, max=7, min=3)
word = Word( alphas+nums+"@._-" )
message = Regex(".*")
bnf = serverDateTime.setResultsName("timestamp") + status.setResultsName("statusCode") + \
word.setResultsName("method") + message.setResultsName("content")
bnf = serverDateTime.setResultsName("timestamp") + \
status.setResultsName("statusCode") + \
word.setResultsName("method") + message.setResultsName("title")
return bnf
def isZopeLogBeginLine(line):
......@@ -68,8 +84,9 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f
fields.timestamp , date_format),
status=fields.get('statusCode', ''),
method=fields.get('method', ''),
title=fields.content,
content=fields.content))
url=fields.get('link', ''),
title=fields.title,
content=fields.get('content', fields.title)))
index += 1
except Exception:
raise
......@@ -81,8 +98,8 @@ def insertRssDb(db_path, entry_list, rss_name):
db = sqlite3.connect(db_path)
for entry in entry_list:
date = entry['datetime'].strftime('%Y-%m-%d %H:%M:%S')
db.execute("insert into rss_entry(name, datetime, status, method, title, content) values (?, ?, ?, ?, ?, ?)",
(rss_name, date, entry['status'], entry['method'], entry['title'], entry['content']))
db.execute("insert into rss_entry(name, datetime, status, method, title, url, content) values (?, ?, ?, ?, ?, ?, ?)",
(rss_name, date, entry['status'], entry['method'], entry['title'], entry['url'], entry['content']))
db.commit()
db.close()
......@@ -94,7 +111,7 @@ def truncateRssDb(db_path, to_date):
def selectRssDb(db_path, rss_name, start_date, limit=0):
db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, content from rss_entry "
query = "select name, datetime, status, method, title, url, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC"
if limit:
query += " limit ?"
......@@ -110,7 +127,7 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
items = []
db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, content from rss_entry "
query = "select name, datetime, status, method, title, url, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC"
if limit:
query += " limit ?"
......@@ -119,13 +136,14 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
entry_list = db.execute(query, (name, start_date))
for entry in entry_list:
name, rss_date, status, method, title, content = entry
name, rss_date, status, method, title, url, content = entry
if method:
title = "[%s] %s" % (method, title)
title = "[%s] %s" % (status, title)
if status:
title = "[%s] %s" % (status, title)
rss_item = PyRSS2Gen.RSSItem(
title = title,
link = "",
link = url,
description = content.replace('\n', '<br/>'),
pubDate = rss_date,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (rss_date, url_link)))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment