Commit the zeo-1_0-debug-branch to the trunk.

I expect this code will become ZEO 1.1.

src/ZEO/ClientStorage.py

@@ -13,22 +13,27 @@
 ##############################################################################
 """Network ZODB storage client
 """
-__version__='$Revision: 1.39 $'[11:-2]
+__version__='$Revision: 1.40 $'[11:-2]
 
-import struct, time, os, socket, string, Sync, zrpc, ClientCache
-import tempfile, Invalidator, ExtensionClass, thread
-import ThreadedAsync
-
-now=time.time
+import struct, time, os, socket, string
+import tempfile, thread
 from struct import pack, unpack
+from types import TupleType
+
+import Invalidator, ExtensionClass
+import ThreadedAsync, Sync, zrpc, ClientCache
+
 from ZODB import POSException, BaseStorage
 from ZODB.TimeStamp import TimeStamp
-from zLOG import LOG, PROBLEM, INFO
 
-try: from ZODB.ConflictResolution import ResolvedSerial
-except: ResolvedSerial='rs'
+from ZEO.logger import zLogger
 
-TupleType=type(())
+log = zLogger("ZEO Client")
+
+try:
+    from ZODB.ConflictResolution import ResolvedSerial
+except:
+    ResolvedSerial='rs'
 
 class ClientStorageError(POSException.StorageError):
     """An error occured in the ZEO Client Storage"""
@@ -63,7 +68,11 @@ class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage):
                     'supportsUndo':0, 'supportsVersions': 0,
                     }
 
-        self._call=zrpc.asyncRPC(connection, debug=debug,
+        if debug:
+            debug_log = log
+        else:
+            debug_log = None
+        self._call=zrpc.asyncRPC(connection, debug=debug_log,
                                  tmin=min_disconnect_poll,
                                  tmax=max_disconnect_poll)
 
@@ -132,7 +141,7 @@ class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage):
             # If we can't connect right away, go ahead and open the cache
             # and start a separate thread to try and reconnect.
 
-            LOG("ClientStorage", PROBLEM, "Failed to connect to storage")
+            log.problem("Failed to connect to storage")
             self._cache.open()
             thread.start_new_thread(self._call.connect,(0,))
 
@@ -140,7 +149,7 @@ class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage):
             # notifyConnected
 
     def notifyConnected(self, s):
-        LOG("ClientStorage", INFO, "Connected to storage")
+        log.info("Connected to storage")
         self._lock_acquire()
         try:
             
@@ -197,7 +206,7 @@ class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage):
     ### responsible for starting the thread that makes the connection.
 
     def notifyDisconnected(self, ignored):
-        LOG("ClientStorage", PROBLEM, "Disconnected from storage")
+        log.problem("Disconnected from storage")
         self._connected=0
         self._transaction=None
         thread.start_new_thread(self._call.connect,(0,))
@@ -233,7 +242,7 @@ class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage):
     def close(self):
         self._lock_acquire()
         try:
-            LOG("ClientStorage", INFO, "close")
+            log.info("close")
             self._call.closeIntensionally()
             try:
                 self._tfile.close()
@@ -550,6 +559,9 @@ class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage):
 
     def sync(self): self._call.sync()
 
+    def status(self):
+        self._call.sendMessage('status')
+
 def getWakeup(_w=[]):
     if _w: return _w[0]
     import trigger

src/ZEO/logger.py

+import string
+from types import StringType
+from zLOG import *
+
+__all__ = ["zLogger", "format_msg"]
+
+_MAX_MSG_SIZE = 120
+
+def format_msg(*args):
+    accum = []
+    total_len = 0
+    for arg in args:
+        if not isinstance(arg, StringType):
+            arg = str(arg)
+        accum.append(arg)
+        total_len = total_len + len(arg)
+        if total_len >= _MAX_MSG_SIZE:
+            break
+    m = string.join(accum)
+    if len(m) > _MAX_MSG_SIZE:
+        m = m[:_MAX_MSG_SIZE] + ' ...'
+    return m
+
+class zLogger:
+
+    def __init__(self, channel):
+        self.channel = channel
+
+    def __str__(self):
+        raise RuntimeError, "don't print me"
+
+    def trace(self, msg):
+        LOG(self.channel, TRACE, msg)
+
+    def debug(self, msg):
+        LOG(self.channel, DEBUG, msg)
+
+    def blather(self, msg):
+        LOG(self.channel, BLATHER, msg)
+
+    def info(self, msg):
+        LOG(self.channel, INFO, msg)
+
+    def problem(self, msg):
+        LOG(self.channel, PROBLEM, msg)
+
+    def warning(self, msg):
+        LOG(self.channel, WARNING, msg)
+
+    def error(self, msg, error=None):
+        LOG(self.channel, ERROR, msg, error=error)
+
+    def panic(self, msg):
+        LOG(self.channel, PANIC, msg)

src/ZEO/smac.py

@@ -14,11 +14,11 @@
 """Sized message async connections
 """
 
-__version__ = "$Revision: 1.15 $"[11:-2]
+__version__ = "$Revision: 1.16 $"[11:-2]
 
 import asyncore, string, struct, zLOG, sys, Acquisition
 import socket, errno
-from zLOG import LOG, TRACE, ERROR, INFO
+from logger import zLogger
 
 # Use the dictionary to make sure we get the minimum number of errno
 # entries.   We expect that EWOULDBLOCK == EAGAIN on most systems --
@@ -48,10 +48,10 @@ class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
         SizedMessageAsyncConnection.inheritedAttribute(
             '__init__')(self, sock, map)
         self.addr=addr
-        if debug is not None:
-            self._debug=debug
-        elif not hasattr(self, '_debug'):
-            self._debug=__debug__ and 'smac'
+        if debug is None and __debug__:
+            self._debug = zLogger("smac")
+        else:
+            self._debug = debug
         self.__state=None
         self.__inp=None
         self.__inpl=0
@@ -132,10 +132,12 @@ class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
 
     def message_output(self, message,
                        pack=struct.pack, len=len):
-        if self._debug:
-            if len(message) > 40: m=message[:40]+' ...'
-            else: m=message
-            LOG(self._debug, TRACE, 'message_output %s' % `m`)
+        if self._debug is not None:
+            if len(message) > 40:
+                m = message[:40]+' ...'
+            else:
+                m = message
+            self._debug.trace('message_output %s' % `m`)
 
         append=self.__append
         if append is None:
@@ -143,13 +145,6 @@ class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
         
         append(pack(">i",len(message))+message)
 
-    def log_info(self, message, type='info'):
-        if type=='error': type=ERROR
-        else: type=INFO
-        LOG('ZEO', type, message)
-
-    log=log_info
-
     def close(self):
         if self.__append is not None:
             self.__append=None

src/ZEO/start.py

@@ -15,7 +15,7 @@
 """Start the server storage.
 """
 
-__version__ = "$Revision: 1.30 $"[11:-2]
+__version__ = "$Revision: 1.31 $"[11:-2]
 
 import sys, os, getopt, string
 
@@ -75,11 +75,9 @@ def main(argv):
                            os.path.join(var, 'ZEO_SERVER.pid')
                            )
 
-    opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:')
-
-    fs=os.path.join(var, 'Data.fs')
+    fs = os.path.join(var, 'Data.fs')
 
-    usage="""%s [options] [filename]
+    usage = """%s [options] [filename]
 
     where options are:
 
@@ -121,6 +119,13 @@ def main(argv):
     if no file name is specified, then %s is used.
     """ % (me, fs)
 
+    try:
+        opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:')
+    except getopt.error, err:
+        print err
+        print usage
+        sys.exit(1)
+
     port=None
     debug=detailed=0
     host=''
@@ -217,15 +222,15 @@ def main(argv):
             import signal
 
             signal.signal(signal.SIGTERM,
-                          lambda sig, frame, s=storages: shutdown(s)
-                          )
+                          lambda sig, frame, s=storages: shutdown(s))
             signal.signal(signal.SIGINT,
-                          lambda sig, frame, s=storages: shutdown(s, 0)
-                          )
-            try: signal.signal(signal.SIGHUP, rotate_logs_handler)
-            except: pass
-
-        except: pass
+                          lambda sig, frame, s=storages: shutdown(s, 0))
+            try:
+                signal.signal(signal.SIGHUP, rotate_logs_handler)
+            except:
+                pass
+        except:
+            pass
 
         items=storages.items()
         items.sort()
@@ -236,13 +241,16 @@ def main(argv):
 
         ZEO.StorageServer.StorageServer(unix, storages)
 
-        try: ppid, pid = os.getppid(), os.getpid()
-        except: pass # getpid not supported
-        else: open(zeo_pid,'w').write("%s %s" % (ppid, pid))
+        try:
+            ppid, pid = os.getppid(), os.getpid()
+        except:
+            pass # getpid not supported
+        else:
+            open(zeo_pid,'w').write("%s %s" % (ppid, pid))
 
     except:
         # Log startup exception and tell zdaemon not to restart us.
-        info=sys.exc_info()
+        info = sys.exc_info()
         try:
             import zLOG
             zLOG.LOG("z2", zLOG.PANIC, "Startup exception",
@@ -280,21 +288,29 @@ def shutdown(storages, die=1):
     # unnecessary, since we now use so_reuseaddr.
     for ignored in 1,2:
         for socket in asyncore.socket_map.values():
-            try: socket.close()
-            except: pass
+            try:
+                socket.close()
+            except:
+                pass
 
     for storage in storages.values():
-        try: storage.close()
-        except: pass
+        try:
+            storage.close()
+        except:
+            pass
 
     try:
         from zLOG import LOG, INFO
         LOG('ZEO Server', INFO,
             "Shutting down (%s)" % (die and "shutdown" or "restart")
             )
-    except: pass
+    except:
+        pass
     
-    if die: sys.exit(0)
-    else: sys.exit(1)
+    if die:
+        sys.exit(0)
+    else:
+        sys.exit(1)
 
-if __name__=='__main__': main(sys.argv)
+if __name__ == '__main__':
+    main(sys.argv)

src/ZEO/tests/testZEO.py

@@ -172,6 +172,7 @@ class GenericTests(ZEOTestBase,
     def tearDown(self):
         """Try to cause the tests to halt"""
         self.running = 0
+        self._storage.status()
         self._storage.close()
         self._server.close()
         os.waitpid(self._pid, 0)
@@ -218,6 +219,7 @@ class GenericTests(ZEOTestBase,
         oid = self._storage.new_oid()
         self._storage.store(oid, None, '', '', t)
         self._storage.tpc_vote(t)
+        self._storage.status()
         self._storage.tpc_finish(t)
 
         for store, trans in self._storages:
@@ -237,7 +239,7 @@ class GenericTests(ZEOTestBase,
     def _get_timestamp(self):
         t = time.time()
         t = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
-        return 't'
+        return `t`
 
 class ZEOFileStorageTests(GenericTests):
     __super_setUp = GenericTests.setUp

src/ZEO/trigger.py

@@ -15,10 +15,8 @@
 # This module is a simplified version of the select_trigger module
 # from Sam Rushing's Medusa server.
 
-
 import asyncore
-#import asynchat
-
+import errno
 import os
 import socket
 import string
@@ -26,7 +24,7 @@ import thread
     
 if os.name == 'posix':
 
-    class trigger (asyncore.file_dispatcher):
+    class trigger(asyncore.file_dispatcher):
 
         "Wake up a call to select() running in the main thread"
 
@@ -58,10 +56,10 @@ if os.name == 'posix':
         # new data onto a channel's outgoing data queue at the same time that
         # the main thread is trying to remove some]
 
-        def __init__ (self):
+        def __init__(self):
             r, w = self._fds = os.pipe()
             self.trigger = w
-            asyncore.file_dispatcher.__init__ (self, r)
+            asyncore.file_dispatcher.__init__(self, r)
             self.lock = thread.allocate_lock()
             self.thunks = []
 
@@ -69,30 +67,35 @@ if os.name == 'posix':
             os.close(self._fds[0])
             os.close(self._fds[1])
 
-        def __repr__ (self):
-            return '<select-trigger (pipe) at %x>' % id(self)
+        def __repr__(self):
+            return '<select-trigger(pipe) at %x>' % id(self)
 
-        def readable (self):
+        def readable(self):
             return 1
 
-        def writable (self):
+        def writable(self):
             return 0
 
-        def handle_connect (self):
+        def handle_connect(self):
             pass
 
-        def pull_trigger (self, thunk=None):
+        def pull_trigger(self, thunk=None):
             # print 'PULL_TRIGGER: ', len(self.thunks)
             if thunk:
                 try:
                     self.lock.acquire()
-                    self.thunks.append (thunk)
+                    self.thunks.append(thunk)
                 finally:
                     self.lock.release()
-            os.write (self.trigger, 'x')
+            os.write(self.trigger, 'x')
 
-        def handle_read (self):
-            self.recv (8192)
+        def handle_read(self):
+            try:
+                self.recv(8192)
+            except os.error, err:
+                if err[0] == errno.EAGAIN: # resource temporarily unavailable
+                    return
+                raise
             try:
                 self.lock.acquire()
                 for thunk in self.thunks:
@@ -101,7 +104,7 @@ if os.name == 'posix':
                     except:
                         nil, t, v, tbinfo = asyncore.compact_traceback()
                         print ('exception in trigger thunk:'
-                               ' (%s:%s %s)' % (t, v, tbinfo))
+                               '(%s:%s %s)' % (t, v, tbinfo))
                 self.thunks = []
             finally:
                 self.lock.release()
@@ -113,13 +116,13 @@ else:
 
     # win32-safe version
 
-    class trigger (asyncore.dispatcher):
+    class trigger(asyncore.dispatcher):
 
         address = ('127.9.9.9', 19999)
 
-        def __init__ (self):
-            a = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
-            w = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+        def __init__(self):
+            a = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            w = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 
             # set TCP_NODELAY to true to avoid buffering
             w.setsockopt(socket.IPPROTO_TCP, 1, 1)
@@ -137,45 +140,50 @@ else:
                         raise 'Bind Error', 'Cannot bind trigger!'
                     port=port - 1
             
-            a.listen (1)
-            w.setblocking (0)
+            a.listen(1)
+            w.setblocking(0)
             try:
-                w.connect (self.address)
+                w.connect(self.address)
             except:
                 pass
             r, addr = a.accept()
             a.close()
-            w.setblocking (1)
+            w.setblocking(1)
             self.trigger = w
 
-            asyncore.dispatcher.__init__ (self, r)
+            asyncore.dispatcher.__init__(self, r)
             self.lock = thread.allocate_lock()
             self.thunks = []
             self._trigger_connected = 0
 
-        def __repr__ (self):
+        def __repr__(self):
             return '<select-trigger (loopback) at %x>' % id(self)
 
-        def readable (self):
+        def readable(self):
             return 1
 
-        def writable (self):
+        def writable(self):
             return 0
 
-        def handle_connect (self):
+        def handle_connect(self):
             pass
 
-        def pull_trigger (self, thunk=None):
+        def pull_trigger(self, thunk=None):
             if thunk:
                 try:
                     self.lock.acquire()
-                    self.thunks.append (thunk)
+                    self.thunks.append(thunk)
                 finally:
                     self.lock.release()
-            self.trigger.send ('x')
+            self.trigger.send('x')
 
-        def handle_read (self):
-            self.recv (8192)
+        def handle_read(self):
+            try:
+                self.recv(8192)
+            except os.error, err:
+                if err[0] == errno.EAGAIN: # resource temporarily unavailable
+                    return
+                raise
             try:
                 self.lock.acquire()
                 for thunk in self.thunks:

src/ZEO/zrpc.py

@@ -14,7 +14,7 @@
 """Simple rpc mechanisms
 """
 
-__version__ = "$Revision: 1.22 $"[11:-2]
+__version__ = "$Revision: 1.23 $"[11:-2]
 
 from cPickle import loads
 import cPickle
@@ -61,12 +61,12 @@ class asyncRPC(SizedMessageAsyncConnection):
         self.__call_la=l.acquire
         self.__call_lr=l.release
 
-    def connect(self, tryonce=1, log_type='client'):
+    def connect(self, tryonce=1):
         t=self._tmin
         connection = self._connection
         debug=self._debug
         while self.__closed == 0:
-            if log_type: LOG(log_type, INFO,
+            LOG("client", INFO,
                 'Trying to connect to server: %s' % `connection`)
             try:
                 if type(connection) is type(''):
@@ -75,15 +75,15 @@ class asyncRPC(SizedMessageAsyncConnection):
                     s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                 s.connect(connection)    
             except Exception, err:
-                if debug:
-                    LOG(debug, DEBUG, "Failed to connect to server: %s" % err)
+                if debug is not None:
+                    debug.blather("Failed to connect to server: %s" % err)
                 if tryonce: return 0
                 time.sleep(t)
                 t=t*2
                 if t > self._tmax: t=self._tmax
             else:
-                if debug:
-                    LOG(debug, DEBUG, "Connected to server")
+                if debug is not None:
+                    debug.blather("Connected to server")
                     
                 # Make sure the result lock is set, se we don't
                 # get an old result (e.g. the exception that
@@ -199,12 +199,12 @@ class asyncRPC(SizedMessageAsyncConnection):
         self._outOfBand=f
 
     def message_input(self, m):
-        if self._debug:
+        if self._debug is not None:
             if len(m) > 60:
                 md = repr(m[:60]) + ' ...'
             else:
                 md = repr(m)
-            LOG(self._debug, TRACE, 'message_input %s' % md)
+            self._debug.trace('message_input %s' % md)
 
         c=m[:1]
         if c in 'RE':

src/ZEO/zrpc/smac.py

@@ -14,11 +14,11 @@
 """Sized message async connections
 """
 
-__version__ = "$Revision: 1.15 $"[11:-2]
+__version__ = "$Revision: 1.16 $"[11:-2]
 
 import asyncore, string, struct, zLOG, sys, Acquisition
 import socket, errno
-from zLOG import LOG, TRACE, ERROR, INFO
+from logger import zLogger
 
 # Use the dictionary to make sure we get the minimum number of errno
 # entries.   We expect that EWOULDBLOCK == EAGAIN on most systems --
@@ -48,10 +48,10 @@ class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
         SizedMessageAsyncConnection.inheritedAttribute(
             '__init__')(self, sock, map)
         self.addr=addr
-        if debug is not None:
-            self._debug=debug
-        elif not hasattr(self, '_debug'):
-            self._debug=__debug__ and 'smac'
+        if debug is None and __debug__:
+            self._debug = zLogger("smac")
+        else:
+            self._debug = debug
         self.__state=None
         self.__inp=None
         self.__inpl=0
@@ -132,10 +132,12 @@ class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
 
     def message_output(self, message,
                        pack=struct.pack, len=len):
-        if self._debug:
-            if len(message) > 40: m=message[:40]+' ...'
-            else: m=message
-            LOG(self._debug, TRACE, 'message_output %s' % `m`)
+        if self._debug is not None:
+            if len(message) > 40:
+                m = message[:40]+' ...'
+            else:
+                m = message
+            self._debug.trace('message_output %s' % `m`)
 
         append=self.__append
         if append is None:
@@ -143,13 +145,6 @@ class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
         
         append(pack(">i",len(message))+message)
 
-    def log_info(self, message, type='info'):
-        if type=='error': type=ERROR
-        else: type=INFO
-        LOG('ZEO', type, message)
-
-    log=log_info
-
     def close(self):
         if self.__append is not None:
             self.__append=None