new logging subsystem using the python-asn1 codec.

coro/__init__.py

@@ -44,6 +44,7 @@ coro.dns.cache.install()
 # ============================================================================
 
 compact_traceback = tb.traceback_string
+traceback_data = tb.traceback_data
 
 # This overrides the one in <_coro>
 def default_exception_notifier():

coro/log/README.md

+
+Logging Subsystem
+=================
+
+Goals:
+
+1. performance (string formatting is avoided)
+2. simplicity
+3. machine-readable (log processing becomes trivial).
+
+Binary Logging
+--------------
+
+I have worked on many projects over the years that involve processing
+logs, sometimes *HUGE* amounts of logs.  It's very frustrating parsing
+free-formatted, whatever-the-developer-felt-like-writing log files
+into useful data.  Binary logging steps around the problem by making
+the logs machine readable from the start.
+
+Reading the Logs
+----------------
+
+In the ``scripts`` directory you will find ``catlog``, which will process
+either stdin, or a path arg.  It can synchronize with the output of
+``tail``, or ``tail -f``.
+
+Log Levels
+----------
+
+I intend to add a layer that supports log levels, backward compatible
+with this code, in that the first item of '*data' will be an integer
+log level.

coro/log/__init__.py

+# -*- Mode: Python; indent-tabs-mode: nil -*-
+
+__all__ = ['asn1']
+
+import asn1
+import coro
+import time
+
+def is_binary (ob):
+    if type(ob) is not bytes:
+        return False
+    else:
+        return '\x00' in ob
+
+def frob(ob):
+    if type(ob) is bytes:
+        if is_binary (ob):
+            if len(ob) < 500 or args.big:
+                return ob.encode ('hex')
+            else:
+                return '<large>'
+        return ob
+    else:
+        return ob
+
+class StderrLogger:
+
+    # ISO 8601
+    time_format = '%Y-%m-%dT%H:%M:%S'    
+
+    def log (self, *data):
+        data = [frob(x) for x in data]
+        coro.write_stderr ("%s %r\n" % (time.strftime (self.time_format), data))
+
+class ComboLogger:
+
+    def __init__ (self, *loggers):
+        self.loggers = loggers
+
+    def log (self, *data):
+        for logger in self.loggers:
+            logger.log (*data)
+

coro/log/asn1.py

+
+from coro.asn1.python import encode
+import coro
+import struct
+
+class Logger:
+
+    magic = '%\xf1\xbfB'
+
+    def __init__ (self, file):
+        self.encode = encode
+        self.file = file
+
+    def log (self, *data):
+        data = self.encode ((coro.now_usec, data))
+        self.file.write (
+            '%s%s%s' % (
+                self.magic,
+                struct.pack ('>I', len(data)),
+                data
+                )
+            )
+        self.file.flush()

coro/log/catlog

+#!/usr/bin/env python
+# -*- Mode: Python -*-
+
+import argparse
+import struct
+import sys
+import datetime
+from coro.asn1.python import decode
+
+class Sync:
+    
+    magic = '%\xf1\xbfB'
+    def __init__ (self):
+        self.state = 0
+        self.last = None
+    def feed (self, ch):
+        if ch == self.magic[self.state]:
+            self.state += 1
+            if self.state == 4:
+                return True
+            else:
+                return False
+        else:
+            self.state = 0
+            self.last = ch
+            return False
+    def resync (self, fdin):
+        self.state = 0
+        give_up = 10000
+        i = 0
+        while i < give_up:
+            ch = fdin.read (1)
+            i += 1
+            if ch == '':
+                raise EOFError
+            else:
+                if self.feed (ch):
+                    return
+        raise ValueError ("unable to sync: is this an asn1 log file?")
+
+def is_binary (ob):
+    if type(ob) is not bytes:
+        return False
+    else:
+        return '\x00' in ob
+
+def frob(ob):
+    if type(ob) is bytes:
+        if is_binary (ob):
+            if len(ob) < 500 or args.big:
+                return ob.encode ('hex')
+            else:
+                return '<large>'
+        return ob
+    else:
+        return ob
+
+def main():
+    if args.file is None:
+        file = sys.stdin
+    else:
+        file = open (args.file, 'rb')
+
+    fromtimestamp = datetime.datetime.fromtimestamp
+    write = sys.stdout.write
+
+    s = Sync()
+
+    s.resync (file)
+
+    while 1:
+        size, = struct.unpack ('>I', file.read (4))
+        block = file.read (size)
+        if len(block) != size:
+            break
+        try:
+            (timestamp, info), size = decode (block)
+        except:
+            write ("[hiccup]\n")
+            s.resync (file)
+            continue
+        timestamp /= 1000000.0
+        info = [frob(x) for x in info]
+        iso = fromtimestamp (timestamp).isoformat()
+        write ('%s %r\n' % (iso, info))
+        magic = file.read (4)
+        if not magic:
+            break
+        elif magic != Sync.magic:
+            s.resync (file)
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser (description='asn1 log decoder')
+    p.add_argument ('-b', '--big', action='store_true', help="show large strings", default=False)
+    p.add_argument ('file', help="input file", nargs="?", metavar="FILE")
+    args = p.parse_args()
+    main()

coro/tb.py

@@ -100,3 +100,31 @@ def traceback_string(t=None, v=None, tb=None):
     first = tbinfo[-1]
     info = '[' + ('] ['.join (tbinfo)) + ']'
     return repr(((first), str(t), str(v), info))
+
+def traceback_data (t=None, v=None, tb=None):
+    """Returns compact traceback data representing the current exception.
+
+    If an exception is not provided as an argument, then it will get the
+    current exception from `sys.exc_info`.
+
+    :Parameters:
+        - `t`: Exception type.
+        - `v`: Exception value.
+        - `tb`: Traceback object.
+
+    :Return:
+        Returns [(file_name, fun_name, line_num), ...] for the current exception and stack trace.
+    """
+    if t is None:
+        t, v, tb = sys.exc_info()
+    tbinfo = []
+    while tb is not None:
+        tbinfo.append ((
+            _get_module_name (tb.tb_frame.f_code.co_filename),
+            tb.tb_frame.f_code.co_name,
+            tb.tb_lineno
+        ))
+        tb = tb.tb_next
+    # just to be safe
+    del tb
+    return (str(t), str(v), tbinfo)