monitor: improve changes for log rss

monitor: do not check list of rss item with start_date, only consider limit

stack/monitor/buildout.cfg

@@ -139,7 +139,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = f880212b392a937a637f6eb244f0e6b5
+md5sum = 91a1a46bcdb7024035a734482c8c7bc9
 filename = logTools.py
 mode = 0644
 
@@ -147,7 +147,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = 49eb40b5515db17bf9c63d87162ad4ad
+md5sum = e723f929b3afd44924a55910954a1a06
 filename = errorlog2rss.py.in
 mode = 0644
 

stack/monitor/errorlog2rss.py.in

@@ -61,25 +61,25 @@ for name, log_path in log_file_list.items():
                         filter_with=log_filter, start_date=start_date,
                                       date_format=date_format)
   if item_list:
-    start = 0
+    items = item_list
     size = len(item_list)
     if limit and size > limit:
-      start = size - limit - 1
-    print "Found %s log entries matching date upper than %s..." % (size, start_date)
+      items = item_list[size-limit-1:-1]
+    print "%s: Found %s log entries matching date upper than %s..." % (name,
+                            size, start_date)
     
-    logTools.insertRssDb(db_path, item_list[start:-1], name)
-    print "%s items inserted into database..." % limit
+    logTools.insertRssDb(db_path, items, name)
     
-    print "Generating RSS entries with %s items..." % limit
-    logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit)
+    print "Generating RSS entries with maximum %s items..." % limit
+    logTools.generateRSS(db_path, name, rss_path, base_link+name, limit=limit)
   else:
-    print "No log entries with date upper than %s..." % start_date
+    print "%s: No log entries with date upper than %s..." % (name, start_date)
   
 try:
   # Remove olders entries from database
   logTools.truncateRssDb(db_path, to_date)
 except Exception,e:
-  print "Database not exist!! ", str(e)
+  print "Database not exist!! \n", str(e)
 
 
 

stack/monitor/logTools.py

@@ -40,7 +40,7 @@ def getZopeParser():
   serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + 
                       integer + ":" + integer + ":" + integer + "," + integer)
   status = Word(string.uppercase, max=7, min=3)
-  word = Word( alphas+nums+"@._-" )
+  word = Word( alphas+nums+"@._-:/#" )
   message = Regex(".*")
   bnf = serverDateTime.setResultsName("timestamp") + \
           status.setResultsName("statusCode") + \
@@ -64,21 +64,24 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f
   log_result = []
   if not date_format:
     date_format = "%Y-%m-%d %H:%M:%S,%f"
+  skip_entry = False
   with open(path, 'r') as logfile:
     index = 0
     for line in logfile:
       regex = method(line)
       if not regex:
-        if index == 0 or line.strip() == "------":
+        if index == 0 or line.strip() == "------" or skip_entry:
           continue
-        # Add this line to log content
+        # Add this line to log content, if entry is not skipped
         log_result[index - 1]['content'] += ("\n" + line)
       else:
         try:
           fields = parserbnf.parseString(line)
-          if filter_with and not fields.statusCode == filter_with:
+          skip_entry = filter_with and not fields.statusCode == filter_with
+          if skip_entry:
             continue
-          if start_date and regex.group() < start_date:
+          skip_entry = start_date and regex.group() < start_date
+          if skip_entry:
             continue
           log_result.append(dict(datetime=datetime.datetime.strptime(
                             fields.timestamp , date_format),
@@ -89,7 +92,8 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f
                             content=fields.get('content', fields.title)))
           index += 1
         except Exception:
-          raise
+          continue
+          #raise
           # print "WARNING: Could not parse log line. %s \n << %s >>" % (str(e), line)
   return log_result
 
@@ -123,17 +127,17 @@ def selectRssDb(db_path, rss_name, start_date, limit=0):
     return rows
   return []
 
-def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
+def generateRSS(db_path, name, rss_path, url_link, limit=10):
   items = []
   
   db = sqlite3.connect(db_path)
   query = "select name, datetime, status, method, title, url, content from rss_entry "
-  query += "where name=? and datetime>=? order by datetime DESC"
+  query += "where name=? order by datetime DESC"
   if limit:
     query += " limit ?"
-    entry_list = db.execute(query, (name, start_date, limit))
+    entry_list = db.execute(query, (name, limit))
   else:
-    entry_list = db.execute(query, (name, start_date))
+    entry_list = db.execute(query, (name,))
   
   for entry in entry_list:
     name, rss_date, status, method, title, url, content = entry