Use tagged version of src/ZEO

lib/python/ZEO/ClientStorage.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""The ClientStorage class and the exceptions that it may raise.
+
+Public contents of this module:
+
+ClientStorage -- the main class, implementing the Storage API
+"""
+
+import cPickle
+import os
+import socket
+import tempfile
+import threading
+import time
+import types
+import logging
+
+from ZEO import ServerStub
+from ZEO.cache import ClientCache
+from ZEO.TransactionBuffer import TransactionBuffer
+from ZEO.Exceptions import ClientStorageError, ClientDisconnected, AuthError
+from ZEO.auth import get_module
+from ZEO.zrpc.client import ConnectionManager
+
+from ZODB import POSException
+from ZODB.loglevels import BLATHER
+from persistent.TimeStamp import TimeStamp
+
+logger = logging.getLogger('ZEO.ClientStorage')
+_pid = str(os.getpid())
+
+def log2(msg, level=logging.INFO, subsys=_pid, exc_info=False):
+    message = "(%s) %s" % (subsys, msg)
+    logger.log(level, message, exc_info=exc_info)
+
+try:
+    from ZODB.ConflictResolution import ResolvedSerial
+except ImportError:
+    ResolvedSerial = 'rs'
+
+def tid2time(tid):
+    return str(TimeStamp(tid))
+
+def get_timestamp(prev_ts=None):
+    """Internal helper to return a unique TimeStamp instance.
+
+    If the optional argument is not None, it must be a TimeStamp; the
+    return value is then guaranteed to be at least 1 microsecond later
+    the argument.
+    """
+    t = time.time()
+    t = TimeStamp(*time.gmtime(t)[:5] + (t % 60,))
+    if prev_ts is not None:
+        t = t.laterThan(prev_ts)
+    return t
+
+class DisconnectedServerStub:
+    """Internal helper class used as a faux RPC stub when disconnected.
+
+    This raises ClientDisconnected on all attribute accesses.
+
+    This is a singleton class -- there should be only one instance,
+    the global disconnected_stub, os it can be tested by identity.
+    """
+
+    def __getattr__(self, attr):
+        raise ClientDisconnected()
+
+# Singleton instance of DisconnectedServerStub
+disconnected_stub = DisconnectedServerStub()
+
+MB = 1024**2
+
+class ClientStorage(object):
+
+    """A Storage class that is a network client to a remote storage.
+
+    This is a faithful implementation of the Storage API.
+
+    This class is thread-safe; transactions are serialized in
+    tpc_begin().
+    """
+
+    # Classes we instantiate.  A subclass might override.
+
+    TransactionBufferClass = TransactionBuffer
+    ClientCacheClass = ClientCache
+    ConnectionManagerClass = ConnectionManager
+    StorageServerStubClass = ServerStub.StorageServer
+
+    def __init__(self, addr, storage='1', cache_size=20 * MB,
+                 name='', client=None, debug=0, var=None,
+                 min_disconnect_poll=5, max_disconnect_poll=300,
+                 wait_for_server_on_startup=None, # deprecated alias for wait
+                 wait=None, wait_timeout=None,
+                 read_only=0, read_only_fallback=0,
+                 username='', password='', realm=None):
+        """ClientStorage constructor.
+
+        This is typically invoked from a custom_zodb.py file.
+
+        All arguments except addr should be keyword arguments.
+        Arguments:
+
+        addr -- The server address(es).  This is either a list of
+            addresses or a single address.  Each address can be a
+            (hostname, port) tuple to signify a TCP/IP connection or
+            a pathname string to signify a Unix domain socket
+            connection.  A hostname may be a DNS name or a dotted IP
+            address.  Required.
+
+        storage -- The storage name, defaulting to '1'.  The name must
+            match one of the storage names supported by the server(s)
+            specified by the addr argument.  The storage name is
+            displayed in the Zope control panel.
+
+        cache_size -- The disk cache size, defaulting to 20 megabytes.
+            This is passed to the ClientCache constructor.
+
+        name -- The storage name, defaulting to ''.  If this is false,
+            str(addr) is used as the storage name.
+
+        client -- A name used to construct persistent cache filenames.
+            Defaults to None, in which case the cache is not persistent.
+            See ClientCache for more info.
+
+        debug -- Ignored.  This is present only for backwards
+            compatibility with ZEO 1.
+
+        var -- When client is not None, this specifies the directory
+            where the persistent cache files are created.  It defaults
+            to None, in whichcase the current directory is used.
+
+        min_disconnect_poll -- The minimum delay in seconds between
+            attempts to connect to the server, in seconds.  Defaults
+            to 5 seconds.
+
+        max_disconnect_poll -- The maximum delay in seconds between
+            attempts to connect to the server, in seconds.  Defaults
+            to 300 seconds.
+
+        wait_for_server_on_startup -- A backwards compatible alias for
+            the wait argument.
+
+        wait -- A flag indicating whether to wait until a connection
+            with a server is made, defaulting to true.
+
+        wait_timeout -- Maximum time to wait for a connection before
+            giving up.  Only meaningful if wait is True.
+
+        read_only -- A flag indicating whether this should be a
+            read-only storage, defaulting to false (i.e. writing is
+            allowed by default).
+
+        read_only_fallback -- A flag indicating whether a read-only
+            remote storage should be acceptable as a fallback when no
+            writable storages are available.  Defaults to false.  At
+            most one of read_only and read_only_fallback should be
+            true.
+
+        username -- string with username to be used when authenticating.
+            These only need to be provided if you are connecting to an
+            authenticated server storage.
+
+        password -- string with plaintext password to be used
+            when authenticated.
+
+        Note that the authentication protocol is defined by the server
+        and is detected by the ClientStorage upon connecting (see
+        testConnection() and doAuth() for details).
+        """
+
+        log2("%s (pid=%d) created %s/%s for storage: %r" %
+             (self.__class__.__name__,
+              os.getpid(),
+              read_only and "RO" or "RW",
+              read_only_fallback and "fallback" or "normal",
+              storage))
+
+        if debug:
+            log2("ClientStorage(): debug argument is no longer used")
+
+        # wait defaults to True, but wait_for_server_on_startup overrides
+        # if not None
+        if wait_for_server_on_startup is not None:
+            if wait is not None and wait != wait_for_server_on_startup:
+                log2("ClientStorage(): conflicting values for wait and "
+                     "wait_for_server_on_startup; wait prevails",
+                     level=logging.WARNING)
+            else:
+                log2("ClientStorage(): wait_for_server_on_startup "
+                     "is deprecated; please use wait instead")
+                wait = wait_for_server_on_startup
+        elif wait is None:
+            wait = 1
+
+        self._addr = addr # For tests
+
+        # A ZEO client can run in disconnected mode, using data from
+        # its cache, or in connected mode.  Several instance variables
+        # are related to whether the client is connected.
+
+        # _server: All method calls are invoked through the server
+        #    stub.  When not connect, set to disconnected_stub an
+        #    object that raises ClientDisconnected errors.
+
+        # _ready: A threading Event that is set only if _server
+        #    is set to a real stub.
+
+        # _connection: The current zrpc connection or None.
+
+        # _connection is set as soon as a connection is established,
+        # but _server is set only after cache verification has finished
+        # and clients can safely use the server.  _pending_server holds
+        # a server stub while it is being verified.
+
+        self._server = disconnected_stub
+        self._connection = None
+        self._pending_server = None
+        self._ready = threading.Event()
+
+        # _is_read_only stores the constructor argument
+        self._is_read_only = read_only
+        # _conn_is_read_only stores the status of the current connection
+        self._conn_is_read_only = 0
+        self._storage = storage
+        self._read_only_fallback = read_only_fallback
+        self._username = username
+        self._password = password
+        self._realm = realm
+
+        # Flag tracking disconnections in the middle of a transaction.  This
+        # is reset in tpc_begin() and set in notifyDisconnected().
+        self._midtxn_disconnect = 0
+
+        # _server_addr is used by sortKey()
+        self._server_addr = None
+        self._tfile = None
+        self._pickler = None
+
+        self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client',
+                      'supportsUndo':0, 'supportsVersions': 0,
+                      'supportsTransactionalUndo': 0}
+
+        self._tbuf = self.TransactionBufferClass()
+        self._db = None
+        self._ltid = None # the last committed transaction
+
+        # _serials: stores (oid, serialno) as returned by server
+        # _seriald: _check_serials() moves from _serials to _seriald,
+        #           which maps oid to serialno
+
+        # XXX If serial number matches transaction id, then there is
+        # no need to have all this extra infrastructure for handling
+        # serial numbers.  The vote call can just return the tid.
+        # If there is a conflict error, we can't have a special method
+        # called just to propagate the error.
+        self._serials = []
+        self._seriald = {}
+
+        self.__name__ = name or str(addr) # Standard convention for storages
+
+        # A ClientStorage only allows one thread to commit at a time.
+        # Mutual exclusion is achieved using _tpc_cond, which
+        # protects _transaction.  A thread that wants to assign to
+        # self._transaction must acquire _tpc_cond first.  A thread
+        # that decides it's done with a transaction (whether via success
+        # or failure) must set _transaction to None and do
+        # _tpc_cond.notify() before releasing _tpc_cond.
+        self._tpc_cond = threading.Condition()
+        self._transaction = None
+
+        # Prevent multiple new_oid calls from going out.  The _oids
+        # variable should only be modified while holding the
+        # _oid_lock.
+        self._oid_lock = threading.Lock()
+        self._oids = [] # Object ids retrieved from new_oids()
+
+        # load() and tpc_finish() must be serialized to guarantee
+        # that cache modifications from each occur atomically.
+        # It also prevents multiple load calls occuring simultaneously,
+        # which simplifies the cache logic.
+        self._load_lock = threading.Lock()
+        # _load_oid and _load_status are protected by _lock
+        self._load_oid = None
+        self._load_status = None
+
+        # Can't read data in one thread while writing data
+        # (tpc_finish) in another thread.  In general, the lock
+        # must prevent access to the cache while _update_cache
+        # is executing.
+        self._lock = threading.Lock()
+
+        # Decide whether to use non-temporary files
+        if client is not None:
+            dir = var or os.getcwd()
+            cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
+        else:
+            cache_path = None
+        self._cache = self.ClientCacheClass(cache_path)
+        # XXX When should it be opened?
+        self._cache.open()
+
+        self._rpc_mgr = self.ConnectionManagerClass(addr, self,
+                                                    tmin=min_disconnect_poll,
+                                                    tmax=max_disconnect_poll)
+
+        if wait:
+            self._wait(wait_timeout)
+        else:
+            # attempt_connect() will make an attempt that doesn't block
+            # "too long," for a very vague notion of too long.  If that
+            # doesn't succeed, call connect() to start a thread.
+            if not self._rpc_mgr.attempt_connect():
+                self._rpc_mgr.connect()
+
+    def _wait(self, timeout=None):
+        if timeout is not None:
+            deadline = time.time() + timeout
+            log2("Setting deadline to %f" % deadline, level=BLATHER)
+        else:
+            deadline = None
+        # Wait for a connection to be established.
+        self._rpc_mgr.connect(sync=1)
+        # When a synchronous connect() call returns, there is
+        # a valid _connection object but cache validation may
+        # still be going on.  This code must wait until validation
+        # finishes, but if the connection isn't a zrpc async
+        # connection it also needs to poll for input.
+        if self._connection.is_async():
+            while 1:
+                self._ready.wait(30)
+                if self._ready.isSet():
+                    break
+                if timeout and time.time() > deadline:
+                    log2("Timed out waiting for connection",
+                         level=logging.WARNING)
+                    break
+                log2("Waiting for cache verification to finish")
+        else:
+            self._wait_sync(deadline)
+
+    def _wait_sync(self, deadline=None):
+        # If there is no mainloop running, this code needs
+        # to call poll() to cause asyncore to handle events.
+        while 1:
+            if self._ready.isSet():
+                break
+            if deadline and time.time() > deadline:
+                log2("Timed out waiting for connection", level=logging.WARNING)
+                break
+            log2("Waiting for cache verification to finish")
+            if self._connection is None:
+                # If the connection was closed while we were
+                # waiting for it to become ready, start over.
+                return self._wait(deadline - time.time())
+            else:
+                self._connection.pending(30)
+
+    def close(self):
+        """Storage API: finalize the storage, releasing external resources."""
+        self._tbuf.close()
+        if self._cache is not None:
+            self._cache.close()
+            self._cache = None
+        if self._rpc_mgr is not None:
+            self._rpc_mgr.close()
+            self._rpc_mgr = None
+
+    def registerDB(self, db, limit):
+        """Storage API: register a database for invalidation messages.
+
+        This is called by ZODB.DB (and by some tests).
+
+        The storage isn't really ready to use until after this call.
+        """
+        self._db = db
+
+    def is_connected(self):
+        """Return whether the storage is currently connected to a server."""
+        # This function is used by clients, so we only report that a
+        # connection exists when the connection is ready to use.
+        return self._ready.isSet()
+
+    def sync(self):
+        """Handle any pending invalidation messages.
+
+        This is called by the sync method in ZODB.Connection.
+        """
+        # If there is no connection, return immediately.  Technically,
+        # there are no pending invalidations so they are all handled.
+        # There doesn't seem to be much benefit to raising an exception.
+
+        cn = self._connection
+        if cn is not None:
+            cn.pending()
+
+    def doAuth(self, protocol, stub):
+        if not (self._username and self._password):
+            raise AuthError, "empty username or password"
+
+        module = get_module(protocol)
+        if not module:
+            log2("%s: no such an auth protocol: %s" %
+                 (self.__class__.__name__, protocol), level=logging.WARNING)
+            return
+
+        storage_class, client, db_class = module
+
+        if not client:
+            log2("%s: %s isn't a valid protocol, must have a Client class" %
+                 (self.__class__.__name__, protocol), level=logging.WARNING)
+            raise AuthError, "invalid protocol"
+
+        c = client(stub)
+
+        # Initiate authentication, returns boolean specifying whether OK
+        return c.start(self._username, self._realm, self._password)
+
+    def testConnection(self, conn):
+        """Internal: test the given connection.
+
+        This returns:   1 if the connection is an optimal match,
+                        0 if it is a suboptimal but acceptable match.
+        It can also raise DisconnectedError or ReadOnlyError.
+
+        This is called by ZEO.zrpc.ConnectionManager to decide which
+        connection to use in case there are multiple, and some are
+        read-only and others are read-write.
+
+        This works by calling register() on the server.  In read-only
+        mode, register() is called with the read_only flag set.  In
+        writable mode and in read-only fallback mode, register() is
+        called with the read_only flag cleared.  In read-only fallback
+        mode only, if the register() call raises ReadOnlyError, it is
+        retried with the read-only flag set, and if this succeeds,
+        this is deemed a suboptimal match.  In all other cases, a
+        succeeding register() call is deemed an optimal match, and any
+        exception raised by register() is passed through.
+        """
+        log2("Testing connection %r" % conn)
+        # XXX Check the protocol version here?
+        self._conn_is_read_only = 0
+        stub = self.StorageServerStubClass(conn)
+
+        auth = stub.getAuthProtocol()
+        log2("Server authentication protocol %r" % auth)
+        if auth:
+            skey = self.doAuth(auth, stub)
+            if skey:
+                log2("Client authentication successful")
+                conn.setSessionKey(skey)
+            else:
+                log2("Authentication failed")
+                raise AuthError, "Authentication failed"
+
+        try:
+            stub.register(str(self._storage), self._is_read_only)
+            return 1
+        except POSException.ReadOnlyError:
+            if not self._read_only_fallback:
+                raise
+            log2("Got ReadOnlyError; trying again with read_only=1")
+            stub.register(str(self._storage), read_only=1)
+            self._conn_is_read_only = 1
+            return 0
+
+    def notifyConnected(self, conn):
+        """Internal: start using the given connection.
+
+        This is called by ConnectionManager after it has decided which
+        connection should be used.
+        """
+        if self._cache is None:
+            # the storage was closed, but the connect thread called
+            # this method before it was stopped.
+            return
+
+        # XXX would like to report whether we get a read-only connection
+        if self._connection is not None:
+            reconnect = 1
+        else:
+            reconnect = 0
+        self.set_server_addr(conn.get_addr())
+
+        # If we are upgrading from a read-only fallback connection,
+        # we must close the old connection to prevent it from being
+        # used while the cache is verified against the new connection.
+        if self._connection is not None:
+            self._connection.close()
+        self._connection = conn
+
+        if reconnect:
+            log2("Reconnected to storage: %s" % self._server_addr)
+        else:
+            log2("Connected to storage: %s" % self._server_addr)
+
+        stub = self.StorageServerStubClass(conn)
+        self._oids = []
+        self._info.update(stub.get_info())
+        self.verify_cache(stub)
+        if not conn.is_async():
+            log2("Waiting for cache verification to finish")
+            self._wait_sync()
+        self._handle_extensions()
+
+    def _handle_extensions(self):
+        for name in self.getExtensionMethods().keys():
+            if not hasattr(self, name):
+                setattr(self, name, self._server.extensionMethod(name))
+
+    def set_server_addr(self, addr):
+        # Normalize server address and convert to string
+        if isinstance(addr, types.StringType):
+            self._server_addr = addr
+        else:
+            assert isinstance(addr, types.TupleType)
+            # If the server is on a remote host, we need to guarantee
+            # that all clients used the same name for the server.  If
+            # they don't, the sortKey() may be different for each client.
+            # The best solution seems to be the official name reported
+            # by gethostbyaddr().
+            host = addr[0]
+            try:
+                canonical, aliases, addrs = socket.gethostbyaddr(host)
+            except socket.error, err:
+                log2("Error resolving host: %s (%s)" % (host, err),
+                     level=BLATHER)
+                canonical = host
+            self._server_addr = str((canonical, addr[1]))
+
+    def sortKey(self):
+        # If the client isn't connected to anything, it can't have a
+        # valid sortKey().  Raise an error to stop the transaction early.
+        if self._server_addr is None:
+            raise ClientDisconnected
+        else:
+            return '%s:%s' % (self._storage, self._server_addr)
+
+    def verify_cache(self, server):
+        """Internal routine called to verify the cache.
+
+        The return value (indicating which path we took) is used by
+        the test suite.
+        """
+
+        # If verify_cache() finishes the cache verification process,
+        # it should set self._server.  If it goes through full cache
+        # verification, then endVerify() should self._server.
+
+        last_inval_tid = self._cache.getLastTid()
+        if last_inval_tid is not None:
+            ltid = server.lastTransaction()
+            if ltid == last_inval_tid:
+                log2("No verification necessary (last_inval_tid up-to-date)")
+                self._server = server
+                self._ready.set()
+                return "no verification"
+
+            # log some hints about last transaction
+            log2("last inval tid: %r %s\n"
+                 % (last_inval_tid, tid2time(last_inval_tid)))
+            log2("last transaction: %r %s" %
+                 (ltid, ltid and tid2time(ltid)))
+
+            pair = server.getInvalidations(last_inval_tid)
+            if pair is not None:
+                log2("Recovering %d invalidations" % len(pair[1]))
+                self.invalidateTransaction(*pair)
+                self._server = server
+                self._ready.set()
+                return "quick verification"
+
+        log2("Verifying cache")
+        # setup tempfile to hold zeoVerify results
+        self._tfile = tempfile.TemporaryFile(suffix=".inv")
+        self._pickler = cPickle.Pickler(self._tfile, 1)
+        self._pickler.fast = 1 # Don't use the memo
+
+        # XXX should batch these operations for efficiency
+        # XXX need to acquire lock...
+        for oid, tid, version in self._cache.contents():
+            server.verify(oid, version, tid)
+        self._pending_server = server
+        server.endZeoVerify()
+        return "full verification"
+
+    ### Is there a race condition between notifyConnected and
+    ### notifyDisconnected? In Particular, what if we get
+    ### notifyDisconnected in the middle of notifyConnected?
+    ### The danger is that we'll proceed as if we were connected
+    ### without worrying if we were, but this would happen any way if
+    ### notifyDisconnected had to get the instance lock.  There's
+    ### nothing to gain by getting the instance lock.
+
+    def notifyDisconnected(self):
+        """Internal: notify that the server connection was terminated.
+
+        This is called by ConnectionManager when the connection is
+        closed or when certain problems with the connection occur.
+        """
+        log2("Disconnected from storage: %s" % repr(self._server_addr))
+        self._connection = None
+        self._ready.clear()
+        self._server = disconnected_stub
+        self._midtxn_disconnect = 1
+
+    def __len__(self):
+        """Return the size of the storage."""
+        # XXX Where is this used?
+        return self._info['length']
+
+    def getName(self):
+        """Storage API: return the storage name as a string.
+
+        The return value consists of two parts: the name as determined
+        by the name and addr argments to the ClientStorage
+        constructor, and the string 'connected' or 'disconnected' in
+        parentheses indicating whether the storage is (currently)
+        connected.
+        """
+        return "%s (%s)" % (
+            self.__name__,
+            self.is_connected() and "connected" or "disconnected")
+
+    def getSize(self):
+        """Storage API: an approximate size of the database, in bytes."""
+        return self._info['size']
+
+    def getExtensionMethods(self):
+        """getExtensionMethods
+
+        This returns a dictionary whose keys are names of extra methods
+        provided by this storage. Storage proxies (such as ZEO) should
+        call this method to determine the extra methods that they need
+        to proxy in addition to the standard storage methods.
+        Dictionary values should be None; this will be a handy place
+        for extra marshalling information, should we need it
+        """
+        return self._info.get('extensionMethods', {})
+
+    def supportsUndo(self):
+        """Storage API: return whether we support undo."""
+        return self._info['supportsUndo']
+
+    def supportsVersions(self):
+        """Storage API: return whether we support versions."""
+        return self._info['supportsVersions']
+
+    def supportsTransactionalUndo(self):
+        """Storage API: return whether we support transactional undo."""
+        return self._info['supportsTransactionalUndo']
+
+    def isReadOnly(self):
+        """Storage API: return whether we are in read-only mode."""
+        if self._is_read_only:
+            return 1
+        else:
+            # If the client is configured for a read-write connection
+            # but has a read-only fallback connection, _conn_is_read_only
+            # will be True.
+            return self._conn_is_read_only
+
+    def _check_trans(self, trans):
+        """Internal helper to check a transaction argument for sanity."""
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
+        if self._transaction is not trans:
+            raise POSException.StorageTransactionError(self._transaction,
+                                                       trans)
+
+    def abortVersion(self, version, txn):
+        """Storage API: clear any changes made by the given version."""
+        self._check_trans(txn)
+        tid, oids = self._server.abortVersion(version, id(txn))
+        # When a version aborts, invalidate the version and
+        # non-version data.  The non-version data should still be
+        # valid, but older versions of ZODB will change the
+        # non-version serialno on an abort version.  With those
+        # versions of ZODB, you'd get a conflict error if you tried to
+        # commit a transaction with the cached data.
+
+        # XXX If we could guarantee that ZODB gave the right answer,
+        # we could just invalidate the version data.
+        for oid in oids:
+            self._tbuf.invalidate(oid, '')
+        return tid, oids
+
+    def commitVersion(self, source, destination, txn):
+        """Storage API: commit the source version in the destination."""
+        self._check_trans(txn)
+        tid, oids = self._server.commitVersion(source, destination, id(txn))
+        if destination:
+            # just invalidate our version data
+            for oid in oids:
+                self._tbuf.invalidate(oid, source)
+        else:
+            # destination is "", so invalidate version and non-version
+            for oid in oids:
+                self._tbuf.invalidate(oid, "")
+        return tid, oids
+
+    def history(self, oid, version, length=1):
+        """Storage API: return a sequence of HistoryEntry objects.
+
+        This does not support the optional filter argument defined by
+        the Storage API.
+        """
+        return self._server.history(oid, version, length)
+
+    def getSerial(self, oid):
+        """Storage API: return current serial number for oid."""
+        return self._server.getSerial(oid)
+
+    def loadSerial(self, oid, serial):
+        """Storage API: load a historical revision of an object."""
+        return self._server.loadSerial(oid, serial)
+
+    def load(self, oid, version):
+        """Storage API: return the data for a given object.
+
+        This returns the pickle data and serial number for the object
+        specified by the given object id and version, if they exist;
+        otherwise a KeyError is raised.
+        """
+        return self.loadEx(oid, version)[:2]
+
+    def loadEx(self, oid, version):
+        self._lock.acquire()    # for atomic processing of invalidations
+        try:
+            t = self._cache.load(oid, version)
+            if t:
+                return t
+        finally:
+            self._lock.release()
+
+        if self._server is None:
+            raise ClientDisconnected()
+
+        self._load_lock.acquire()
+        try:
+            self._lock.acquire()
+            try:
+                self._load_oid = oid
+                self._load_status = 1
+            finally:
+                self._lock.release()
+
+            data, tid, ver = self._server.loadEx(oid, version)
+
+            self._lock.acquire()    # for atomic processing of invalidations
+            try:
+                if self._load_status:
+                    self._cache.store(oid, ver, tid, None, data)
+                self._load_oid = None
+            finally:
+                self._lock.release()
+        finally:
+            self._load_lock.release()
+
+        return data, tid, ver
+
+    def loadBefore(self, oid, tid):
+        self._lock.acquire()
+        try:
+            t = self._cache.loadBefore(oid, tid)
+            if t is not None:
+                return t
+        finally:
+            self._lock.release()
+
+        t = self._server.loadBefore(oid, tid)
+        if t is None:
+            return None
+        data, start, end = t
+        if end is None:
+            # This method should not be used to get current data.  It
+            # doesn't use the _load_lock, so it is possble to overlap
+            # this load with an invalidation for the same object.
+
+            # XXX If we call again, we're guaranteed to get the
+            # post-invalidation data.  But if the data is still
+            # current, we'll still get end == None.
+
+            # Maybe the best thing to do is to re-run the test with
+            # the load lock in the case.  That's slow performance, but
+            # I don't think real application code will ever care about
+            # it.
+
+            return data, start, end
+        self._lock.acquire()
+        try:
+            self._cache.store(oid, "", start, end, data)
+        finally:
+            self._lock.release()
+
+        return data, start, end
+
+    def modifiedInVersion(self, oid):
+        """Storage API: return the version, if any, that modfied an object.
+
+        If no version modified the object, return an empty string.
+        """
+        self._lock.acquire()
+        try:
+            v = self._cache.modifiedInVersion(oid)
+            if v is not None:
+                return v
+        finally:
+            self._lock.release()
+        return self._server.modifiedInVersion(oid)
+
+    def new_oid(self):
+        """Storage API: return a new object identifier."""
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
+        # avoid multiple oid requests to server at the same time
+        self._oid_lock.acquire()
+        try:
+            if not self._oids:
+                self._oids = self._server.new_oids()
+                self._oids.reverse()
+            return self._oids.pop()
+        finally:
+            self._oid_lock.release()
+
+    def pack(self, t=None, referencesf=None, wait=1, days=0):
+        """Storage API: pack the storage.
+
+        Deviations from the Storage API: the referencesf argument is
+        ignored; two additional optional arguments wait and days are
+        provided:
+
+        wait -- a flag indicating whether to wait for the pack to
+            complete; defaults to true.
+
+        days -- a number of days to subtract from the pack time;
+            defaults to zero.
+        """
+        # XXX Is it okay that read-only connections allow pack()?
+        # rf argument ignored; server will provide it's own implementation
+        if t is None:
+            t = time.time()
+        t = t - (days * 86400)
+        return self._server.pack(t, wait)
+
+    def _check_serials(self):
+        """Internal helper to move data from _serials to _seriald."""
+        # XXX serials are always going to be the same, the only
+        # question is whether an exception has been raised.
+        if self._serials:
+            l = len(self._serials)
+            r = self._serials[:l]
+            del self._serials[:l]
+            for oid, s in r:
+                if isinstance(s, Exception):
+                    raise s
+                self._seriald[oid] = s
+            return r
+
+    def store(self, oid, serial, data, version, txn):
+        """Storage API: store data for an object."""
+        self._check_trans(txn)
+        self._server.storea(oid, serial, data, version, id(txn))
+        self._tbuf.store(oid, version, data)
+        return self._check_serials()
+
+    def tpc_vote(self, txn):
+        """Storage API: vote on a transaction."""
+        if txn is not self._transaction:
+            return
+        self._server.vote(id(txn))
+        return self._check_serials()
+
+    def tpc_begin(self, txn, tid=None, status=' '):
+        """Storage API: begin a transaction."""
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
+        self._tpc_cond.acquire()
+        self._midtxn_disconnect = 0
+        while self._transaction is not None:
+            # It is allowable for a client to call two tpc_begins in a
+            # row with the same transaction, and the second of these
+            # must be ignored.
+            if self._transaction == txn:
+                self._tpc_cond.release()
+                return
+            self._tpc_cond.wait(30)
+        self._transaction = txn
+        self._tpc_cond.release()
+
+        try:
+            self._server.tpc_begin(id(txn), txn.user, txn.description,
+                                   txn._extension, tid, status)
+        except:
+            # Client may have disconnected during the tpc_begin().
+            if self._server is not disconnected_stub:
+                self.end_transaction()
+            raise
+
+        self._tbuf.clear()
+        self._seriald.clear()
+        del self._serials[:]
+
+    def end_transaction(self):
+        """Internal helper to end a transaction."""
+        # the right way to set self._transaction to None
+        # calls notify() on _tpc_cond in case there are waiting threads
+        self._tpc_cond.acquire()
+        self._transaction = None
+        self._tpc_cond.notify()
+        self._tpc_cond.release()
+
+    def lastTransaction(self):
+        return self._cache.getLastTid()
+
+    def tpc_abort(self, txn):
+        """Storage API: abort a transaction."""
+        if txn is not self._transaction:
+            return
+        try:
+            # XXX Are there any transactions that should prevent an
+            # abort from occurring?  It seems wrong to swallow them
+            # all, yet you want to be sure that other abort logic is
+            # executed regardless.
+            try:
+                self._server.tpc_abort(id(txn))
+            except ClientDisconnected:
+                log2("ClientDisconnected in tpc_abort() ignored",
+                     level=BLATHER)
+        finally:
+            self._tbuf.clear()
+            self._seriald.clear()
+            del self._serials[:]
+            self.end_transaction()
+
+    def tpc_finish(self, txn, f=None):
+        """Storage API: finish a transaction."""
+        if txn is not self._transaction:
+            return
+        self._load_lock.acquire()
+        try:
+            if self._midtxn_disconnect:
+                raise ClientDisconnected(
+                       'Calling tpc_finish() on a disconnected transaction')
+
+            # The calls to tpc_finish() and _update_cache() should
+            # never run currently with another thread, because the
+            # tpc_cond condition variable prevents more than one
+            # thread from calling tpc_finish() at a time.
+            tid = self._server.tpc_finish(id(txn))
+            self._lock.acquire()  # for atomic processing of invalidations
+            try:
+                self._update_cache(tid)
+                if f is not None:
+                    f(tid)
+            finally:
+                self._lock.release()
+
+            r = self._check_serials()
+            assert r is None or len(r) == 0, "unhandled serialnos: %s" % r
+        finally:
+            self._load_lock.release()
+            self.end_transaction()
+
+    def _update_cache(self, tid):
+        """Internal helper to handle objects modified by a transaction.
+
+        This iterates over the objects in the transaction buffer and
+        update or invalidate the cache.
+        """
+        # Must be called with _lock already acquired.
+
+        # XXX not sure why _update_cache() would be called on
+        # a closed storage.
+        if self._cache is None:
+            return
+
+        for oid, version, data in self._tbuf:
+            self._cache.invalidate(oid, version, tid)
+            # If data is None, we just invalidate.
+            if data is not None:
+                s = self._seriald[oid]
+                if s != ResolvedSerial:
+                    assert s == tid, (s, tid)
+                    self._cache.store(oid, version, s, None, data)
+        self._tbuf.clear()
+
+    def undo(self, trans_id, txn):
+        """Storage API: undo a transaction.
+
+        This is executed in a transactional context.  It has no effect
+        until the transaction is committed.  It can be undone itself.
+
+        Zope uses this to implement undo unless it is not supported by
+        a storage.
+        """
+        self._check_trans(txn)
+        tid, oids = self._server.undo(trans_id, id(txn))
+        for oid in oids:
+            self._tbuf.invalidate(oid, '')
+        return tid, oids
+
+    def undoInfo(self, first=0, last=-20, specification=None):
+        """Storage API: return undo information."""
+        return self._server.undoInfo(first, last, specification)
+
+    def undoLog(self, first=0, last=-20, filter=None):
+        """Storage API: return a sequence of TransactionDescription objects.
+
+        The filter argument should be None or left unspecified, since
+        it is impossible to pass the filter function to the server to
+        be executed there.  If filter is not None, an empty sequence
+        is returned.
+        """
+        if filter is not None:
+            return []
+        return self._server.undoLog(first, last)
+
+    def versionEmpty(self, version):
+        """Storage API: return whether the version has no transactions."""
+        return self._server.versionEmpty(version)
+
+    def versions(self, max=None):
+        """Storage API: return a sequence of versions in the storage."""
+        return self._server.versions(max)
+
+    # Below are methods invoked by the StorageServer
+
+    def serialnos(self, args):
+        """Server callback to pass a list of changed (oid, serial) pairs."""
+        self._serials.extend(args)
+
+    def info(self, dict):
+        """Server callback to update the info dictionary."""
+        self._info.update(dict)
+
+    def invalidateVerify(self, args):
+        """Server callback to invalidate an (oid, version) pair.
+
+        This is called as part of cache validation.
+        """
+        # Invalidation as result of verify_cache().
+        # Queue an invalidate for the end the verification procedure.
+        if self._pickler is None:
+            # XXX This should never happen
+            return
+        self._pickler.dump(args)
+
+    def _process_invalidations(self, invs):
+        # Invalidations are sent by the ZEO server as a sequence of
+        # oid, version pairs.  The DB's invalidate() method expects a
+        # dictionary of oids.
+
+        self._lock.acquire()
+        try:
+            # versions maps version names to dictionary of invalidations
+            versions = {}
+            for oid, version, tid in invs:
+                if oid == self._load_oid:
+                    self._load_status = 0
+                self._cache.invalidate(oid, version, tid)
+                versions.setdefault((version, tid), {})[oid] = tid
+
+            if self._db is not None:
+                for (version, tid), d in versions.items():
+                    self._db.invalidate(tid, d, version=version)
+        finally:
+            self._lock.release()
+
+    def endVerify(self):
+        """Server callback to signal end of cache validation."""
+        if self._pickler is None:
+            return
+        # write end-of-data marker
+        self._pickler.dump((None, None))
+        self._pickler = None
+        self._tfile.seek(0)
+        f = self._tfile
+        self._tfile = None
+        self._process_invalidations(InvalidationLogIterator(f))
+        f.close()
+
+        log2("endVerify finishing")
+        self._server = self._pending_server
+        self._ready.set()
+        self._pending_conn = None
+        log2("endVerify finished")
+
+    def invalidateTransaction(self, tid, args):
+        """Invalidate objects modified by tid."""
+        self._lock.acquire()
+        try:
+            self._cache.setLastTid(tid)
+        finally:
+            self._lock.release()
+        if self._pickler is not None:
+            log2("Transactional invalidation during cache verification",
+                 level=BLATHER)
+            for t in args:
+                self._pickler.dump(t)
+            return
+        self._process_invalidations([(oid, version, tid)
+                                     for oid, version in args])
+
+    # The following are for compatibility with protocol version 2.0.0
+
+    def invalidateTrans(self, args):
+        return self.invalidateTransaction(None, args)
+
+    invalidate = invalidateVerify
+    end = endVerify
+    Invalidate = invalidateTrans
+
+def InvalidationLogIterator(fileobj):
+    unpickler = cPickle.Unpickler(fileobj)
+    while 1:
+        oid, version = unpickler.load()
+        if oid is None:
+            break
+        yield oid, version, None

lib/python/ZEO/ClientStub.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""RPC stubs for interface exported by ClientStorage."""
+
+class ClientStorage:
+
+    """An RPC stub class for the interface exported by ClientStorage.
+
+    This is the interface presented by ClientStorage to the
+    StorageServer; i.e. the StorageServer calls these methods and they
+    are executed in the ClientStorage.
+
+    See the ClientStorage class for documentation on these methods.
+
+    It is currently important that all methods here are asynchronous
+    (meaning they don't have a return value and the caller doesn't
+    wait for them to complete), *and* that none of them cause any
+    calls from the client to the storage.  This is due to limitations
+    in the zrpc subpackage.
+
+    The on-the-wire names of some of the methods don't match the
+    Python method names.  That's because the on-the-wire protocol was
+    fixed for ZEO 2 and we don't want to change it.  There are some
+    aliases in ClientStorage.py to make up for this.
+    """
+
+    def __init__(self, rpc):
+        """Constructor.
+
+        The argument is a connection: an instance of the
+        zrpc.connection.Connection class.
+        """
+        self.rpc = rpc
+
+    def beginVerify(self):
+        self.rpc.callAsync('beginVerify')
+
+    def invalidateVerify(self, args):
+        self.rpc.callAsync('invalidateVerify', args)
+
+    def endVerify(self):
+        self.rpc.callAsync('endVerify')
+
+    def invalidateTransaction(self, tid, args):
+        self.rpc.callAsyncNoPoll('invalidateTransaction', tid, args)
+
+    def serialnos(self, arg):
+        self.rpc.callAsync('serialnos', arg)
+
+    def info(self, arg):
+        self.rpc.callAsync('info', arg)

lib/python/ZEO/CommitLog.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Log a transaction's commit info during two-phase commit.
+
+A storage server allows multiple clients to commit transactions, but
+must serialize them as the actually execute at the server.  The
+concurrent commits are achieved by logging actions up until the
+tpc_vote().  At that point, the entire transaction is committed on the
+real storage.
+"""
+import cPickle
+import tempfile
+
+class CommitLog:
+
+    def __init__(self):
+        self.file = tempfile.TemporaryFile(suffix=".log")
+        self.pickler = cPickle.Pickler(self.file, 1)
+        self.pickler.fast = 1
+        self.stores = 0
+        self.read = 0
+
+    def size(self):
+        return self.file.tell()
+
+    def store(self, oid, serial, data, version):
+        self.pickler.dump((oid, serial, data, version))
+        self.stores += 1
+
+    def get_loader(self):
+        self.read = 1
+        self.file.seek(0)
+        return self.stores, cPickle.Unpickler(self.file)
+
+    def close(self):
+        if self.file:
+            self.file.close()
+            self.file = None

lib/python/ZEO/DEPENDENCIES.cfg

+BTrees
+ThreadedAsync
+ZConfig
+ZODB
+persistent
+transaction
+zdaemon

lib/python/ZEO/DebugServer.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""A debugging version of the server that records network activity."""
+
+import struct
+import time
+
+from ZEO.StorageServer import StorageServer, log
+from ZEO.zrpc.server import ManagedServerConnection
+
+# a bunch of codes
+NEW_CONN = 1
+CLOSE_CONN = 2
+DATA = 3
+ERROR = 4
+
+class DebugManagedServerConnection(ManagedServerConnection):
+
+    def __init__(self, sock, addr, obj, mgr):
+        # mgr is the DebugServer instance
+        self.mgr = mgr
+        self.__super_init(sock, addr, obj)
+        record_id = mgr._record_connection(addr)
+        self._record = lambda code, data: mgr._record(record_id, code, data)
+        self.obj.notifyConnected(self)
+
+    def close(self):
+        self._record(CLOSE_CONN, "")
+        ManagedServerConnection.close(self)
+
+    # override the lowest-level of asyncore's connection
+
+    def recv(self, buffer_size):
+        try:
+            data = self.socket.recv(buffer_size)
+            if not data:
+                # a closed connection is indicated by signaling
+                # a read condition, and having recv() return 0.
+                self.handle_close()
+                return ''
+            else:
+                self._record(DATA, data)
+                return data
+        except socket.error, why:
+            # winsock sometimes throws ENOTCONN
+            self._record(ERROR, why)
+            if why[0] in [ECONNRESET, ENOTCONN, ESHUTDOWN]:
+                self.handle_close()
+                return ''
+            else:
+                raise socket.error, why
+
+class DebugServer(StorageServer):
+
+    ZEOStorageClass = DebugZEOStorage
+    ManagedServerConnectionClass = DebugManagerConnection
+
+    def __init__(self, *args, **kwargs):
+        StorageServer.__init__(*args, **kwargs)
+        self._setup_record(kwargs["record"])
+        self._conn_counter = 1
+
+    def _setup_record(self, path):
+        try:
+            self._recordfile = open(path, "ab")
+        except IOError, msg:
+            self._recordfile = None
+            log("failed to open recordfile %s: %s" % (path, msg))
+
+    def _record_connection(self, addr):
+        cid = self._conn_counter
+        self._conn_counter += 1
+        self._record(cid, NEW_CONN, str(addr))
+        return cid
+
+    def _record(self, conn, code, data):
+        s = struct.pack(">iii", code, time.time(), len(data)) + data
+        self._recordfile.write(s)

lib/python/ZEO/Exceptions.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Exceptions for ZEO."""
+
+from ZODB.POSException import StorageError
+
+class ClientStorageError(StorageError):
+    """An error occured in the ZEO Client Storage."""
+
+class UnrecognizedResult(ClientStorageError):
+    """A server call returned an unrecognized result."""
+
+class ClientDisconnected(ClientStorageError):
+    """The database storage is disconnected from the storage."""
+
+class AuthError(StorageError):
+    """The client provided invalid authentication credentials."""

lib/python/ZEO/README.txt

+=======
+ZEO 2.0
+=======
+
+What's ZEO?
+-----------
+
+ZEO stands for Zope Enterprise Objects.  ZEO is an add-on for Zope
+that allows multiple processes to connect to a single ZODB storage.
+Those processes can live on different machines, but don't need to.
+ZEO 2 has many improvements over ZEO 1, and is incompatible with ZEO 1;
+if you upgrade an existing ZEO 1 installation, you must upgrade the
+server and all clients simultaneous.  If you received ZEO 2 as part of
+the ZODB 3 distribution, the ZEO 1 sources are provided in a separate
+directory (ZEO1).  Some documentation for ZEO is available in the ZODB 3
+package in the Doc subdirectory.  ZEO depends on the ZODB software; it
+can be used with the version of ZODB distributed with Zope 2.5.1 or
+later.  More information about ZEO can be found in the ZODB Wiki:
+
+    http://www.zope.org/Wikis/ZODB
+
+What's here?
+------------
+
+This list of filenames is mostly for ZEO developers::
+
+ ClientCache.py          client-side cache implementation
+ ClientStorage.py        client-side storage implementation
+ ClientStub.py           RPC stubs for callbacks from server to client
+ CommitLog.py            buffer used during two-phase commit on the server
+ Exceptions.py           definitions of exceptions
+ ICache.py               interface definition for the client-side cache
+ ServerStub.py           RPC stubs for the server
+ StorageServer.py        server-side storage implementation
+ TransactionBuffer.py    buffer used for transaction data in the client
+ __init__.py             near-empty file to make this directory a package
+ simul.py                command-line tool to simulate cache behavior
+ start.py                command-line tool to start the storage server
+ stats.py                command-line tool to process client cache traces
+ tests/                  unit tests and other test utilities
+ util.py                 utilities used by the server startup tool
+ version.txt             text file indicating the ZEO version
+ zrpc/                   subpackage implementing Remote Procedure Call (RPC)
+

lib/python/ZEO/ServerStub.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""RPC stubs for interface exported by StorageServer."""
+
+##
+# ZEO storage server.
+# <p>
+# Remote method calls can be synchronous or asynchronous.  If the call
+# is synchronous, the client thread blocks until the call returns.  A
+# single client can only have one synchronous request outstanding.  If
+# several threads share a single client, threads other than the caller
+# will block only if the attempt to make another synchronous call.
+# An asynchronous call does not cause the client thread to block.  An
+# exception raised by an asynchronous method is logged on the server,
+# but is not returned to the client.
+
+class StorageServer:
+
+    """An RPC stub class for the interface exported by ClientStorage.
+
+    This is the interface presented by the StorageServer to the
+    ClientStorage; i.e. the ClientStorage calls these methods and they
+    are executed in the StorageServer.
+
+    See the StorageServer module for documentation on these methods,
+    with the exception of _update(), which is documented here.
+    """
+
+    def __init__(self, rpc):
+        """Constructor.
+
+        The argument is a connection: an instance of the
+        zrpc.connection.Connection class.
+        """
+        self.rpc = rpc
+        # Wait until we know what version the other side is using.
+        while rpc.peer_protocol_version is None:
+            rpc.pending()
+        if rpc.peer_protocol_version == 'Z200':
+            self.lastTransaction = lambda: None
+            self.getInvalidations = lambda tid: None
+            self.getAuthProtocol = lambda: None
+
+    def extensionMethod(self, name):
+        return ExtensionMethodWrapper(self.rpc, name).call
+
+    ##
+    # Register current connection with a storage and a mode.
+    # In effect, it is like an open call.
+    # @param storage_name a string naming the storage.  This argument
+    #        is primarily for backwards compatibility with servers
+    #        that supported multiple storages.
+    # @param read_only boolean
+    # @exception ValueError unknown storage_name or already registered
+    # @exception ReadOnlyError storage is read-only and a read-write
+    #            connectio was requested
+
+    def register(self, storage_name, read_only):
+        self.rpc.call('register', storage_name, read_only)
+
+    ##
+    # Return dictionary of meta-data about the storage.
+    # @defreturn dict
+
+    def get_info(self):
+        return self.rpc.call('get_info')
+
+    ##
+    # Check whether the server requires authentication.  Returns
+    # the name of the protocol.
+    # @defreturn string
+
+    def getAuthProtocol(self):
+        return self.rpc.call('getAuthProtocol')
+
+    ##
+    # Return id of the last committed transaction
+    # @defreturn string
+
+    def lastTransaction(self):
+        # Not in protocol version 2.0.0; see __init__()
+        return self.rpc.call('lastTransaction')
+
+    ##
+    # Return invalidations for all transactions after tid.
+    # @param tid transaction id
+    # @defreturn 2-tuple, (tid, list)
+    # @return tuple containing the last committed transaction
+    #         and a list of oids that were invalidated.  Returns
+    #         None and an empty list if the server does not have
+    #         the list of oids available.
+
+    def getInvalidations(self, tid):
+        # Not in protocol version 2.0.0; see __init__()
+        return self.rpc.call('getInvalidations', tid)
+
+    ##
+    # Check whether serial numbers s and sv are current for oid.
+    # If one or both of the serial numbers are not current, the
+    # server will make an asynchronous invalidateVerify() call.
+    # @param oid object id
+    # @param s serial number on non-version data
+    # @param sv serial number of version data or None
+    # @defreturn async
+
+    def zeoVerify(self, oid, s, sv):
+        self.rpc.callAsync('zeoVerify', oid, s, sv)
+
+    ##
+    # Check whether current serial number is valid for oid and version.
+    # If the serial number is not current, the server will make an
+    # asynchronous invalidateVerify() call.
+    # @param oid object id
+    # @param version name of version for oid
+    # @param serial client's current serial number
+    # @defreturn async
+
+    def verify(self, oid, version, serial):
+        self.rpc.callAsync('verify', oid, version, serial)
+
+    ##
+    # Signal to the server that cache verification is done.
+    # @defreturn async
+
+    def endZeoVerify(self):
+        self.rpc.callAsync('endZeoVerify')
+
+    ##
+    # Generate a new set of oids.
+    # @param n number of new oids to return
+    # @defreturn list
+    # @return list of oids
+
+    def new_oids(self, n=None):
+        if n is None:
+            return self.rpc.call('new_oids')
+        else:
+            return self.rpc.call('new_oids', n)
+
+    ##
+    # Pack the storage.
+    # @param t pack time
+    # @param wait optional, boolean.  If true, the call will not
+    #             return until the pack is complete.
+
+    def pack(self, t, wait=None):
+        if wait is None:
+            self.rpc.call('pack', t)
+        else:
+            self.rpc.call('pack', t, wait)
+
+    ##
+    # Return current data for oid.  Version data is returned if
+    # present.
+    # @param oid object id
+    # @defreturn 5-tuple
+    # @return 5-tuple, current non-version data, serial number,
+    #         version name, version data, version data serial number
+    # @exception KeyError if oid is not found
+
+    def zeoLoad(self, oid):
+        return self.rpc.call('zeoLoad', oid)
+
+    ##
+    # Return current data for oid along with tid if transaction that
+    # wrote the date.
+    # @param oid object id
+    # @param version string, name of version
+    # @defreturn 4-tuple
+    # @return data, serial number, transaction id, version,
+    #         where version is the name of the version the data came
+    #         from or "" for non-version data
+    # @exception KeyError if oid is not found
+
+    def loadEx(self, oid, version):
+        return self.rpc.call("loadEx", oid, version)
+
+    ##
+    # Return non-current data along with transaction ids that identify
+    # the lifetime of the specific revision.
+    # @param oid object id
+    # @param tid a transaction id that provides an upper bound on
+    #            the lifetime of the revision.  That is, loadBefore
+    #            returns the revision that was current before tid committed.
+    # @defreturn 4-tuple
+    # @return data, serial numbr, start transaction id, end transaction id
+
+    def loadBefore(self, oid, tid):
+        return self.rpc.call("loadBefore", oid, tid)
+
+    ##
+    # Storage new revision of oid.
+    # @param oid object id
+    # @param serial serial number that this transaction read
+    # @param data new data record for oid
+    # @param version name of version or ""
+    # @param id id of current transaction
+    # @defreturn async
+
+    def storea(self, oid, serial, data, version, id):
+        self.rpc.callAsync('storea', oid, serial, data, version, id)
+
+    ##
+    # Start two-phase commit for a transaction
+    # @param id id used by client to identify current transaction.  The
+    #        only purpose of this argument is to distinguish among multiple
+    #        threads using a single ClientStorage.
+    # @param user name of user committing transaction (can be "")
+    # @param description string containing transaction metadata (can be "")
+    # @param ext dictionary of extended metadata (?)
+    # @param tid optional explicit tid to pass to underlying storage
+    # @param status optional status character, e.g "p" for pack
+    # @defreturn async
+
+    def tpc_begin(self, id, user, descr, ext, tid, status):
+        return self.rpc.call('tpc_begin', id, user, descr, ext, tid, status)
+
+    def vote(self, trans_id):
+        return self.rpc.call('vote', trans_id)
+
+    def tpc_finish(self, id):
+        return self.rpc.call('tpc_finish', id)
+
+    def tpc_abort(self, id):
+        self.rpc.callAsync('tpc_abort', id)
+
+    def abortVersion(self, src, id):
+        return self.rpc.call('abortVersion', src, id)
+
+    def commitVersion(self, src, dest, id):
+        return self.rpc.call('commitVersion', src, dest, id)
+
+    def history(self, oid, version, length=None):
+        if length is None:
+            return self.rpc.call('history', oid, version)
+        else:
+            return self.rpc.call('history', oid, version, length)
+
+    def load(self, oid, version):
+        return self.rpc.call('load', oid, version)
+
+    def getSerial(self, oid):
+        return self.rpc.call('getSerial', oid)
+
+    def loadSerial(self, oid, serial):
+        return self.rpc.call('loadSerial', oid, serial)
+
+    def modifiedInVersion(self, oid):
+        return self.rpc.call('modifiedInVersion', oid)
+
+    def new_oid(self, last=None):
+        if last is None:
+            return self.rpc.call('new_oid')
+        else:
+            return self.rpc.call('new_oid', last)
+
+    def store(self, oid, serial, data, version, trans):
+        return self.rpc.call('store', oid, serial, data, version, trans)
+
+    def undo(self, trans_id, trans):
+        return self.rpc.call('undo', trans_id, trans)
+
+    def undoLog(self, first, last):
+        return self.rpc.call('undoLog', first, last)
+
+    def undoInfo(self, first, last, spec):
+        return self.rpc.call('undoInfo', first, last, spec)
+
+    def versionEmpty(self, vers):
+        return self.rpc.call('versionEmpty', vers)
+
+    def versions(self, max=None):
+        if max is None:
+            return self.rpc.call('versions')
+        else:
+            return self.rpc.call('versions', max)
+
+class ExtensionMethodWrapper:
+    def __init__(self, rpc, name):
+        self.rpc = rpc
+        self.name = name
+
+    def call(self, *a, **kwa):
+        return self.rpc.call(self.name, *a, **kwa)

lib/python/ZEO/StorageServer.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""The StorageServer class and the exception that it may raise.
+
+This server acts as a front-end for one or more real storages, like
+file storage or Berkeley storage.
+
+XXX Need some basic access control-- a declaration of the methods
+exported for invocation by the server.
+"""
+
+import asyncore
+import cPickle
+import os
+import sys
+import threading
+import time
+import logging
+
+import transaction
+
+from ZEO import ClientStub
+from ZEO.CommitLog import CommitLog
+from ZEO.monitor import StorageStats, StatsServer
+from ZEO.zrpc.server import Dispatcher
+from ZEO.zrpc.connection import ManagedServerConnection, Delay, MTDelay
+from ZEO.zrpc.trigger import trigger
+from ZEO.Exceptions import AuthError
+
+from ZODB.ConflictResolution import ResolvedSerial
+from ZODB.POSException import StorageError, StorageTransactionError
+from ZODB.POSException import TransactionError, ReadOnlyError, ConflictError
+from ZODB.serialize import referencesf
+from ZODB.utils import u64, oid_repr
+from ZODB.loglevels import BLATHER
+
+logger = logging.getLogger('ZEO.StorageServer')
+
+# XXX This used to say "ZSS", which is now implied in the logger name, can this
+#     be either set to str(os.getpid()) (if that makes sense) or removed?
+_label = "" # default label used for logging.
+
+def set_label():
+    """Internal helper to reset the logging label (e.g. after fork())."""
+    global _label
+    _label = "%s" % os.getpid()
+
+def log(message, level=logging.INFO, label=None, exc_info=False):
+    """Internal helper to log a message."""
+    label = label or _label
+    if label:
+        message = "(%s) %s" % (label, message)
+    logger.log(level, message, exc_info=exc_info)
+
+class StorageServerError(StorageError):
+    """Error reported when an unpickleable exception is raised."""
+
+class ZEOStorage:
+    """Proxy to underlying storage for a single remote client."""
+
+    # Classes we instantiate.  A subclass might override.
+    ClientStorageStubClass = ClientStub.ClientStorage
+
+    # A list of extension methods.  A subclass with extra methods
+    # should override.
+    extensions = []
+
+    def __init__(self, server, read_only=0, auth_realm=None):
+        self.server = server
+        # timeout and stats will be initialized in register()
+        self.timeout = None
+        self.stats = None
+        self.connection = None
+        self.client = None
+        self.storage = None
+        self.storage_id = "uninitialized"
+        self.transaction = None
+        self.read_only = read_only
+        self.locked = 0
+        self.verifying = 0
+        self.store_failed = 0
+        self.log_label = _label
+        self.authenticated = 0
+        self.auth_realm = auth_realm
+        # The authentication protocol may define extra methods.
+        self._extensions = {}
+        for func in self.extensions:
+            self._extensions[func.func_name] = None
+
+    def finish_auth(self, authenticated):
+        if not self.auth_realm:
+            return 1
+        self.authenticated = authenticated
+        return authenticated
+
+    def set_database(self, database):
+        self.database = database
+
+    def notifyConnected(self, conn):
+        self.connection = conn # For restart_other() below
+        self.client = self.ClientStorageStubClass(conn)
+        addr = conn.addr
+        if isinstance(addr, type("")):
+            label = addr
+        else:
+            host, port = addr
+            label = str(host) + ":" + str(port)
+        self.log_label = _label + "/" + label
+
+    def notifyDisconnected(self):
+        # When this storage closes, we must ensure that it aborts
+        # any pending transaction.
+        if self.transaction is not None:
+            self.log("disconnected during transaction %s" % self.transaction)
+            self._abort()
+        else:
+            self.log("disconnected")
+        if self.stats is not None:
+            self.stats.clients -= 1
+
+    def __repr__(self):
+        tid = self.transaction and repr(self.transaction.id)
+        if self.storage:
+            stid = (self.storage._transaction and
+                    repr(self.storage._transaction.id))
+        else:
+            stid = None
+        name = self.__class__.__name__
+        return "<%s %X trans=%s s_trans=%s>" % (name, id(self), tid, stid)
+
+    def log(self, msg, level=logging.INFO, exc_info=False):
+        log(msg, level=level, label=self.log_label, exc_info=exc_info)
+
+    def setup_delegation(self):
+        """Delegate several methods to the storage"""
+        self.versionEmpty = self.storage.versionEmpty
+        self.versions = self.storage.versions
+        self.getSerial = self.storage.getSerial
+        self.history = self.storage.history
+        self.load = self.storage.load
+        self.loadSerial = self.storage.loadSerial
+        self.modifiedInVersion = self.storage.modifiedInVersion
+        try:
+            fn = self.storage.getExtensionMethods
+        except AttributeError:
+            # We must be running with a ZODB which
+            # predates adding getExtensionMethods to
+            # BaseStorage. Eventually this try/except
+            # can be removed
+            pass
+        else:
+            d = fn()
+            self._extensions.update(d)
+            for name in d.keys():
+                assert not hasattr(self, name)
+                setattr(self, name, getattr(self.storage, name))
+        self.lastTransaction = self.storage.lastTransaction
+
+    def _check_tid(self, tid, exc=None):
+        if self.read_only:
+            raise ReadOnlyError()
+        if self.transaction is None:
+            caller = sys._getframe().f_back.f_code.co_name
+            self.log("no current transaction: %s()" % caller,
+                     level=logging.WARNING)
+            if exc is not None:
+                raise exc(None, tid)
+            else:
+                return 0
+        if self.transaction.id != tid:
+            caller = sys._getframe().f_back.f_code.co_name
+            self.log("%s(%s) invalid; current transaction = %s" %
+                     (caller, repr(tid), repr(self.transaction.id)),
+                     logging.WARNING)
+            if exc is not None:
+                raise exc(self.transaction.id, tid)
+            else:
+                return 0
+        return 1
+
+    def getAuthProtocol(self):
+        """Return string specifying name of authentication module to use.
+
+        The module name should be auth_%s where %s is auth_protocol."""
+        protocol = self.server.auth_protocol
+        if not protocol or protocol == 'none':
+            return None
+        return protocol
+
+    def register(self, storage_id, read_only):
+        """Select the storage that this client will use
+
+        This method must be the first one called by the client.
+        For authenticated storages this method will be called by the client
+        immediately after authentication is finished.
+        """
+        if self.auth_realm and not self.authenticated:
+            raise AuthError, "Client was never authenticated with server!"
+
+        if self.storage is not None:
+            self.log("duplicate register() call")
+            raise ValueError, "duplicate register() call"
+        storage = self.server.storages.get(storage_id)
+        if storage is None:
+            self.log("unknown storage_id: %s" % storage_id)
+            raise ValueError, "unknown storage: %s" % storage_id
+
+        if not read_only and (self.read_only or storage.isReadOnly()):
+            raise ReadOnlyError()
+
+        self.read_only = self.read_only or read_only
+        self.storage_id = storage_id
+        self.storage = storage
+        self.setup_delegation()
+        self.timeout, self.stats = self.server.register_connection(storage_id,
+                                                                   self)
+
+    def get_info(self):
+        return {'length': len(self.storage),
+                'size': self.storage.getSize(),
+                'name': self.storage.getName(),
+                'supportsUndo': self.storage.supportsUndo(),
+                'supportsVersions': self.storage.supportsVersions(),
+                'extensionMethods': self.getExtensionMethods(),
+                }
+
+    def get_size_info(self):
+        return {'length': len(self.storage),
+                'size': self.storage.getSize(),
+                }
+
+    def getExtensionMethods(self):
+        return self._extensions
+
+    def loadEx(self, oid, version):
+        self.stats.loads += 1
+        return self.storage.loadEx(oid, version)
+
+    def loadBefore(self, oid, tid):
+        self.stats.loads += 1
+        return self.storage.loadBefore(oid, tid)
+
+    def zeoLoad(self, oid):
+        self.stats.loads += 1
+        v = self.storage.modifiedInVersion(oid)
+        if v:
+            pv, sv = self.storage.load(oid, v)
+        else:
+            pv = sv = None
+        try:
+            p, s = self.storage.load(oid, '')
+        except KeyError:
+            if sv:
+                # Created in version, no non-version data
+                p = s = None
+            else:
+                raise
+        return p, s, v, pv, sv
+
+    def getInvalidations(self, tid):
+        invtid, invlist = self.server.get_invalidations(tid)
+        if invtid is None:
+            return None
+        self.log("Return %d invalidations up to tid %s"
+                 % (len(invlist), u64(invtid)))
+        return invtid, invlist
+
+    def verify(self, oid, version, tid):
+        try:
+            t = self.storage.getTid(oid)
+        except KeyError:
+            self.client.invalidateVerify((oid, ""))
+        else:
+            if tid != t:
+                # This will invalidate non-version data when the
+                # client only has invalid version data.  Since this is
+                # an uncommon case, we avoid the cost of checking
+                # whether the serial number matches the current
+                # non-version data.
+                self.client.invalidateVerify((oid, version))
+
+    def zeoVerify(self, oid, s, sv):
+        if not self.verifying:
+            self.verifying = 1
+            self.stats.verifying_clients += 1
+        try:
+            os = self.storage.getTid(oid)
+        except KeyError:
+            self.client.invalidateVerify((oid, ''))
+            # XXX It's not clear what we should do now.  The KeyError
+            # could be caused by an object uncreation, in which case
+            # invalidation is right.  It could be an application bug
+            # that left a dangling reference, in which case it's bad.
+        else:
+            # If the client has version data, the logic is a bit more
+            # complicated.  If the current serial number matches the
+            # client serial number, then the non-version data must
+            # also be valid.  If the current serialno is for a
+            # version, then the non-version data can't change.
+
+            # If the version serialno isn't valid, then the
+            # non-version serialno may or may not be valid.  Rather
+            # than trying to figure it whether it is valid, we just
+            # invalidate it.  Sending an invalidation for the
+            # non-version data implies invalidating the version data
+            # too, since an update to non-version data can only occur
+            # after the version is aborted or committed.
+            if sv:
+                if sv != os:
+                    self.client.invalidateVerify((oid, ''))
+            else:
+                if s != os:
+                    self.client.invalidateVerify((oid, ''))
+
+    def endZeoVerify(self):
+        if self.verifying:
+            self.stats.verifying_clients -= 1
+        self.verifying = 0
+        self.client.endVerify()
+
+    def pack(self, time, wait=1):
+        # Yes, you can pack a read-only server or storage!
+        if wait:
+            return run_in_thread(self._pack_impl, time)
+        else:
+            # If the client isn't waiting for a reply, start a thread
+            # and forget about it.
+            t = threading.Thread(target=self._pack_impl, args=(time,))
+            t.start()
+            return None
+
+    def _pack_impl(self, time):
+        self.log("pack(time=%s) started..." % repr(time))
+        self.storage.pack(time, referencesf)
+        self.log("pack(time=%s) complete" % repr(time))
+        # Broadcast new size statistics
+        self.server.invalidate(0, self.storage_id, None,
+                               (), self.get_size_info())
+
+    def new_oids(self, n=100):
+        """Return a sequence of n new oids, where n defaults to 100"""
+        if self.read_only:
+            raise ReadOnlyError()
+        if n <= 0:
+            n = 1
+        return [self.storage.new_oid() for i in range(n)]
+
+    # undoLog and undoInfo are potentially slow methods
+
+    def undoInfo(self, first, last, spec):
+        return run_in_thread(self.storage.undoInfo, first, last, spec)
+
+    def undoLog(self, first, last):
+        return run_in_thread(self.storage.undoLog, first, last)
+
+    def tpc_begin(self, id, user, description, ext, tid=None, status=" "):
+        if self.read_only:
+            raise ReadOnlyError()
+        if self.transaction is not None:
+            if self.transaction.id == id:
+                self.log("duplicate tpc_begin(%s)" % repr(id))
+                return
+            else:
+                raise StorageTransactionError("Multiple simultaneous tpc_begin"
+                                              " requests from one client.")
+
+        self.transaction = t = transaction.Transaction()
+        t.id = id
+        t.user = user
+        t.description = description
+        t._extension = ext
+
+        self.serials = []
+        self.invalidated = []
+        self.txnlog = CommitLog()
+        self.tid = tid
+        self.status = status
+        self.store_failed = 0
+        self.stats.active_txns += 1
+
+    def tpc_finish(self, id):
+        if not self._check_tid(id):
+            return
+        assert self.locked
+        self.stats.active_txns -= 1
+        self.stats.commits += 1
+        self.storage.tpc_finish(self.transaction)
+        tid = self.storage.lastTransaction()
+        if self.invalidated:
+            self.server.invalidate(self, self.storage_id, tid,
+                                   self.invalidated, self.get_size_info())
+        self._clear_transaction()
+        # Return the tid, for cache invalidation optimization
+        return tid
+
+    def tpc_abort(self, id):
+        if not self._check_tid(id):
+            return
+        self.stats.active_txns -= 1
+        self.stats.aborts += 1
+        if self.locked:
+            self.storage.tpc_abort(self.transaction)
+        self._clear_transaction()
+
+    def _clear_transaction(self):
+        # Common code at end of tpc_finish() and tpc_abort()
+        self.transaction = None
+        self.txnlog.close()
+        if self.locked:
+            self.locked = 0
+            self.timeout.end(self)
+            self.stats.lock_time = None
+            self.log("Transaction released storage lock", BLATHER)
+            # _handle_waiting() can start another transaction (by
+            # restarting a waiting one) so must be done last
+            self._handle_waiting()
+
+    def _abort(self):
+        # called when a connection is closed unexpectedly
+        if not self.locked:
+            # Delete (d, zeo_storage) from the _waiting list, if found.
+            waiting = self.storage._waiting
+            for i in range(len(waiting)):
+                d, z = waiting[i]
+                if z is self:
+                    del waiting[i]
+                    self.log("Closed connection removed from waiting list."
+                             " Clients waiting: %d." % len(waiting))
+                    break
+
+        if self.transaction:
+            self.stats.active_txns -= 1
+            self.stats.aborts += 1
+            self.tpc_abort(self.transaction.id)
+
+    # The public methods of the ZEO client API do not do the real work.
+    # They defer work until after the storage lock has been acquired.
+    # Most of the real implementations are in methods beginning with
+    # an _.
+
+    def storea(self, oid, serial, data, version, id):
+        self._check_tid(id, exc=StorageTransactionError)
+        self.stats.stores += 1
+        self.txnlog.store(oid, serial, data, version)
+
+    # The following four methods return values, so they must acquire
+    # the storage lock and begin the transaction before returning.
+
+    def vote(self, id):
+        self._check_tid(id, exc=StorageTransactionError)
+        if self.locked:
+            return self._vote()
+        else:
+            return self._wait(lambda: self._vote())
+
+    def abortVersion(self, src, id):
+        self._check_tid(id, exc=StorageTransactionError)
+        if self.locked:
+            return self._abortVersion(src)
+        else:
+            return self._wait(lambda: self._abortVersion(src))
+
+    def commitVersion(self, src, dest, id):
+        self._check_tid(id, exc=StorageTransactionError)
+        if self.locked:
+            return self._commitVersion(src, dest)
+        else:
+            return self._wait(lambda: self._commitVersion(src, dest))
+
+    def undo(self, trans_id, id):
+        self._check_tid(id, exc=StorageTransactionError)
+        if self.locked:
+            return self._undo(trans_id)
+        else:
+            return self._wait(lambda: self._undo(trans_id))
+
+    def _tpc_begin(self, txn, tid, status):
+        self.locked = 1
+        self.timeout.begin(self)
+        self.stats.lock_time = time.time()
+        self.storage.tpc_begin(txn, tid, status)
+
+    def _store(self, oid, serial, data, version):
+        err = None
+        try:
+            newserial = self.storage.store(oid, serial, data, version,
+                                           self.transaction)
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except Exception, err:
+            self.store_failed = 1
+            if isinstance(err, ConflictError):
+                self.stats.conflicts += 1
+                self.log("conflict error oid=%s msg=%s" %
+                         (oid_repr(oid), str(err)), BLATHER)
+            if not isinstance(err, TransactionError):
+                # Unexpected errors are logged and passed to the client
+                self.log("store error: %s, %s" % sys.exc_info()[:2],
+                         logging.ERROR, exc_info=True)
+            # Try to pickle the exception.  If it can't be pickled,
+            # the RPC response would fail, so use something else.
+            pickler = cPickle.Pickler()
+            pickler.fast = 1
+            try:
+                pickler.dump(err, 1)
+            except:
+                msg = "Couldn't pickle storage exception: %s" % repr(err)
+                self.log(msg, logging.ERROR)
+                err = StorageServerError(msg)
+            # The exception is reported back as newserial for this oid
+            newserial = err
+        else:
+            if serial != "\0\0\0\0\0\0\0\0":
+                self.invalidated.append((oid, version))
+        if newserial == ResolvedSerial:
+            self.stats.conflicts_resolved += 1
+            self.log("conflict resolved oid=%s" % oid_repr(oid), BLATHER)
+        self.serials.append((oid, newserial))
+        return err is None
+
+    def _vote(self):
+        self.client.serialnos(self.serials)
+        # If a store call failed, then return to the client immediately.
+        # The serialnos() call will deliver an exception that will be
+        # handled by the client in its tpc_vote() method.
+        if self.store_failed:
+            return
+        return self.storage.tpc_vote(self.transaction)
+
+    def _abortVersion(self, src):
+        tid, oids = self.storage.abortVersion(src, self.transaction)
+        inv = [(oid, src) for oid in oids]
+        self.invalidated.extend(inv)
+        return tid, oids
+
+    def _commitVersion(self, src, dest):
+        tid, oids = self.storage.commitVersion(src, dest, self.transaction)
+        inv = [(oid, dest) for oid in oids]
+        self.invalidated.extend(inv)
+        if dest:
+            inv = [(oid, src) for oid in oids]
+            self.invalidated.extend(inv)
+        return tid, oids
+
+    def _undo(self, trans_id):
+        tid, oids = self.storage.undo(trans_id, self.transaction)
+        inv = [(oid, None) for oid in oids]
+        self.invalidated.extend(inv)
+        return tid, oids
+
+    # When a delayed transaction is restarted, the dance is
+    # complicated.  The restart occurs when one ZEOStorage instance
+    # finishes as a transaction and finds another instance is in the
+    # _waiting list.
+
+    # XXX It might be better to have a mechanism to explicitly send
+    # the finishing transaction's reply before restarting the waiting
+    # transaction.  If the restart takes a long time, the previous
+    # client will be blocked until it finishes.
+
+    def _wait(self, thunk):
+        # Wait for the storage lock to be acquired.
+        self._thunk = thunk
+        if self.storage._transaction:
+            d = Delay()
+            self.storage._waiting.append((d, self))
+            self.log("Transaction blocked waiting for storage. "
+                     "Clients waiting: %d." % len(self.storage._waiting))
+            return d
+        else:
+            self.log("Transaction acquired storage lock.", BLATHER)
+            return self._restart()
+
+    def _restart(self, delay=None):
+        # Restart when the storage lock is available.
+        if self.txnlog.stores == 1:
+            template = "Preparing to commit transaction: %d object, %d bytes"
+        else:
+            template = "Preparing to commit transaction: %d objects, %d bytes"
+        self.log(template % (self.txnlog.stores, self.txnlog.size()),
+                 level=BLATHER)
+        self._tpc_begin(self.transaction, self.tid, self.status)
+        loads, loader = self.txnlog.get_loader()
+        for i in range(loads):
+            # load oid, serial, data, version
+            if not self._store(*loader.load()):
+                break
+        resp = self._thunk()
+        if delay is not None:
+            delay.reply(resp)
+        else:
+            return resp
+
+    def _handle_waiting(self):
+        # Restart any client waiting for the storage lock.
+        while self.storage._waiting:
+            delay, zeo_storage = self.storage._waiting.pop(0)
+            if self._restart_other(zeo_storage, delay):
+                if self.storage._waiting:
+                    n = len(self.storage._waiting)
+                    self.log("Blocked transaction restarted.  "
+                             "Clients waiting: %d" % n)
+                else:
+                    self.log("Blocked transaction restarted.")
+                return
+
+    def _restart_other(self, zeo_storage, delay):
+        # Return True if the server restarted.
+        # call the restart() method on the appropriate server.
+        try:
+            zeo_storage._restart(delay)
+        except:
+            self.log("Unexpected error handling waiting transaction",
+                     level=logging.WARNING, exc_info=True)
+            zeo_storage.connection.close()
+            return 0
+        else:
+            return 1
+
+class StorageServer:
+
+    """The server side implementation of ZEO.
+
+    The StorageServer is the 'manager' for incoming connections.  Each
+    connection is associated with its own ZEOStorage instance (defined
+    below).  The StorageServer may handle multiple storages; each
+    ZEOStorage instance only handles a single storage.
+    """
+
+    # Classes we instantiate.  A subclass might override.
+
+    DispatcherClass = Dispatcher
+    ZEOStorageClass = ZEOStorage
+    ManagedServerConnectionClass = ManagedServerConnection
+
+    def __init__(self, addr, storages, read_only=0,
+                 invalidation_queue_size=100,
+                 transaction_timeout=None,
+                 monitor_address=None,
+                 auth_protocol=None,
+                 auth_database=None,
+                 auth_realm=None):
+        """StorageServer constructor.
+
+        This is typically invoked from the start.py script.
+
+        Arguments (the first two are required and positional):
+
+        addr -- the address at which the server should listen.  This
+            can be a tuple (host, port) to signify a TCP/IP connection
+            or a pathname string to signify a Unix domain socket
+            connection.  A hostname may be a DNS name or a dotted IP
+            address.
+
+        storages -- a dictionary giving the storage(s) to handle.  The
+            keys are the storage names, the values are the storage
+            instances, typically FileStorage or Berkeley storage
+            instances.  By convention, storage names are typically
+            strings representing small integers starting at '1'.
+
+        read_only -- an optional flag saying whether the server should
+            operate in read-only mode.  Defaults to false.  Note that
+            even if the server is operating in writable mode,
+            individual storages may still be read-only.  But if the
+            server is in read-only mode, no write operations are
+            allowed, even if the storages are writable.  Note that
+            pack() is considered a read-only operation.
+
+        invalidation_queue_size -- The storage server keeps a queue
+            of the objects modified by the last N transactions, where
+            N == invalidation_queue_size.  This queue is used to
+            speed client cache verification when a client disconnects
+            for a short period of time.
+
+        transaction_timeout -- The maximum amount of time to wait for
+            a transaction to commit after acquiring the storage lock.
+            If the transaction takes too long, the client connection
+            will be closed and the transaction aborted.
+
+        monitor_address -- The address at which the monitor server
+            should listen.  If specified, a monitor server is started.
+            The monitor server provides server statistics in a simple
+            text format.
+
+        auth_protocol -- The name of the authentication protocol to use.
+            Examples are "digest" and "srp".
+
+        auth_database -- The name of the password database filename.
+            It should be in a format compatible with the authentication
+            protocol used; for instance, "sha" and "srp" require different
+            formats.
+
+            Note that to implement an authentication protocol, a server
+            and client authentication mechanism must be implemented in a
+            auth_* module, which should be stored inside the "auth"
+            subdirectory. This module may also define a DatabaseClass
+            variable that should indicate what database should be used
+            by the authenticator.
+        """
+
+        self.addr = addr
+        self.storages = storages
+        set_label()
+        msg = ", ".join(
+            ["%s:%s:%s" % (name, storage.isReadOnly() and "RO" or "RW",
+                           storage.getName())
+             for name, storage in storages.items()])
+        log("%s created %s with storages: %s" %
+            (self.__class__.__name__, read_only and "RO" or "RW", msg))
+        for s in storages.values():
+            s._waiting = []
+        self.read_only = read_only
+        self.auth_protocol = auth_protocol
+        self.auth_database = auth_database
+        self.auth_realm = auth_realm
+        self.database = None
+        if auth_protocol:
+            self._setup_auth(auth_protocol)
+        # A list of at most invalidation_queue_size invalidations.
+        # The list is kept in sorted order with the most recent
+        # invalidation at the front.  The list never has more than
+        # self.invq_bound elements.
+        self.invq = []
+        self.invq_bound = invalidation_queue_size
+        self.connections = {}
+        self.dispatcher = self.DispatcherClass(addr,
+                                               factory=self.new_connection)
+        self.stats = {}
+        self.timeouts = {}
+        for name in self.storages.keys():
+            self.stats[name] = StorageStats()
+            if transaction_timeout is None:
+                # An object with no-op methods
+                timeout = StubTimeoutThread()
+            else:
+                timeout = TimeoutThread(transaction_timeout)
+                timeout.start()
+            self.timeouts[name] = timeout
+        if monitor_address:
+            self.monitor = StatsServer(monitor_address, self.stats)
+        else:
+            self.monitor = None
+
+    def _setup_auth(self, protocol):
+        # Can't be done in global scope, because of cyclic references
+        from ZEO.auth import get_module
+
+        name = self.__class__.__name__
+
+        module = get_module(protocol)
+        if not module:
+            log("%s: no such an auth protocol: %s" % (name, protocol))
+            return
+
+        storage_class, client, db_class = module
+
+        if not storage_class or not issubclass(storage_class, ZEOStorage):
+            log(("%s: %s isn't a valid protocol, must have a StorageClass" %
+                 (name, protocol)))
+            self.auth_protocol = None
+            return
+        self.ZEOStorageClass = storage_class
+
+        log("%s: using auth protocol: %s" % (name, protocol))
+
+        # We create a Database instance here for use with the authenticator
+        # modules. Having one instance allows it to be shared between multiple
+        # storages, avoiding the need to bloat each with a new authenticator
+        # Database that would contain the same info, and also avoiding any
+        # possibly synchronization issues between them.
+        self.database = db_class(self.auth_database)
+        if self.database.realm != self.auth_realm:
+            raise ValueError("password database realm %r "
+                             "does not match storage realm %r"
+                             % (self.database.realm, self.auth_realm))
+
+
+    def new_connection(self, sock, addr):
+        """Internal: factory to create a new connection.
+
+        This is called by the Dispatcher class in ZEO.zrpc.server
+        whenever accept() returns a socket for a new incoming
+        connection.
+        """
+        if self.auth_protocol and self.database:
+            zstorage = self.ZEOStorageClass(self, self.read_only,
+                                            auth_realm=self.auth_realm)
+            zstorage.set_database(self.database)
+        else:
+            zstorage = self.ZEOStorageClass(self, self.read_only)
+
+        c = self.ManagedServerConnectionClass(sock, addr, zstorage, self)
+        log("new connection %s: %s" % (addr, repr(c)))
+        return c
+
+    def register_connection(self, storage_id, conn):
+        """Internal: register a connection with a particular storage.
+
+        This is called by ZEOStorage.register().
+
+        The dictionary self.connections maps each storage name to a
+        list of current connections for that storage; this information
+        is needed to handle invalidation.  This function updates this
+        dictionary.
+
+        Returns the timeout and stats objects for the appropriate storage.
+        """
+        l = self.connections.get(storage_id)
+        if l is None:
+            l = self.connections[storage_id] = []
+        l.append(conn)
+        stats = self.stats[storage_id]
+        stats.clients += 1
+        return self.timeouts[storage_id], stats
+
+    def invalidate(self, conn, storage_id, tid, invalidated=(), info=None):
+        """Internal: broadcast info and invalidations to clients.
+
+        This is called from several ZEOStorage methods.
+
+        This can do three different things:
+
+        - If the invalidated argument is non-empty, it broadcasts
+          invalidateTransaction() messages to all clients of the given
+          storage except the current client (the conn argument).
+
+        - If the invalidated argument is empty and the info argument
+          is a non-empty dictionary, it broadcasts info() messages to
+          all clients of the given storage, including the current
+          client.
+
+        - If both the invalidated argument and the info argument are
+          non-empty, it broadcasts invalidateTransaction() messages to all
+          clients except the current, and sends an info() message to
+          the current client.
+
+        """
+        if invalidated:
+            if len(self.invq) >= self.invq_bound:
+                self.invq.pop()
+            self.invq.insert(0, (tid, invalidated))
+        for p in self.connections.get(storage_id, ()):
+            if invalidated and p is not conn:
+                p.client.invalidateTransaction(tid, invalidated)
+            elif info is not None:
+                p.client.info(info)
+
+    def get_invalidations(self, tid):
+        """Return a tid and list of all objects invalidation since tid.
+
+        The tid is the most recent transaction id seen by the client.
+
+        Returns None if it is unable to provide a complete list
+        of invalidations for tid.  In this case, client should
+        do full cache verification.
+        """
+
+        if not self.invq:
+            log("invq empty")
+            return None, []
+
+        earliest_tid = self.invq[-1][0]
+        if earliest_tid > tid:
+            log("tid to old for invq %s < %s" % (u64(tid), u64(earliest_tid)))
+            return None, []
+
+        oids = {}
+        for _tid, L in self.invq:
+            if _tid <= tid:
+                break
+            for key in L:
+                oids[key] = 1
+        latest_tid = self.invq[0][0]
+        return latest_tid, oids.keys()
+
+    def close_server(self):
+        """Close the dispatcher so that there are no new connections.
+
+        This is only called from the test suite, AFAICT.
+        """
+        self.dispatcher.close()
+        if self.monitor is not None:
+            self.monitor.close()
+        for storage in self.storages.values():
+            storage.close()
+        # Force the asyncore mainloop to exit by hackery, i.e. close
+        # every socket in the map.  loop() will return when the map is
+        # empty.
+        for s in asyncore.socket_map.values():
+            try:
+                s.close()
+            except:
+                pass
+
+    def close_conn(self, conn):
+        """Internal: remove the given connection from self.connections.
+
+        This is the inverse of register_connection().
+        """
+        for cl in self.connections.values():
+            if conn.obj in cl:
+                cl.remove(conn.obj)
+
+class StubTimeoutThread:
+
+    def begin(self, client):
+        pass
+
+    def end(self, client):
+        pass
+
+class TimeoutThread(threading.Thread):
+    """Monitors transaction progress and generates timeouts."""
+
+    # There is one TimeoutThread per storage, because there's one
+    # transaction lock per storage.
+
+    def __init__(self, timeout):
+        threading.Thread.__init__(self)
+        self.setDaemon(1)
+        self._timeout = timeout
+        self._client = None
+        self._deadline = None
+        self._cond = threading.Condition() # Protects _client and _deadline
+        self._trigger = trigger()
+
+    def begin(self, client):
+        # Called from the restart code the "main" thread, whenever the
+        # storage lock is being acquired.  (Serialized by asyncore.)
+        self._cond.acquire()
+        try:
+            assert self._client is None
+            self._client = client
+            self._deadline = time.time() + self._timeout
+            self._cond.notify()
+        finally:
+            self._cond.release()
+
+    def end(self, client):
+        # Called from the "main" thread whenever the storage lock is
+        # being released.  (Serialized by asyncore.)
+        self._cond.acquire()
+        try:
+            assert self._client is not None
+            assert self._client is client
+            self._client = None
+            self._deadline = None
+        finally:
+            self._cond.release()
+
+    def run(self):
+        # Code running in the thread.
+        while 1:
+            self._cond.acquire()
+            try:
+                while self._deadline is None:
+                    self._cond.wait()
+                howlong = self._deadline - time.time()
+                if howlong <= 0:
+                    # Prevent reporting timeout more than once
+                    self._deadline = None
+                client = self._client # For the howlong <= 0 branch below
+            finally:
+                self._cond.release()
+            if howlong <= 0:
+                client.log("Transaction timeout after %s seconds" %
+                           self._timeout)
+                self._trigger.pull_trigger(lambda: client.connection.close())
+            else:
+                time.sleep(howlong)
+
+def run_in_thread(method, *args):
+    t = SlowMethodThread(method, args)
+    t.start()
+    return t.delay
+
+class SlowMethodThread(threading.Thread):
+    """Thread to run potentially slow storage methods.
+
+    Clients can use the delay attribute to access the MTDelay object
+    used to send a zrpc response at the right time.
+    """
+
+    # Some storage methods can take a long time to complete.  If we
+    # run these methods via a standard asyncore read handler, they
+    # will block all other server activity until they complete.  To
+    # avoid blocking, we spawn a separate thread, return an MTDelay()
+    # object, and have the thread reply() when it finishes.
+
+    def __init__(self, method, args):
+        threading.Thread.__init__(self)
+        self._method = method
+        self._args = args
+        self.delay = MTDelay()
+
+    def run(self):
+        try:
+            result = self._method(*self._args)
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except Exception:
+            self.delay.error(sys.exc_info())
+        else:
+            self.delay.reply(result)

lib/python/ZEO/TransactionBuffer.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""A TransactionBuffer store transaction updates until commit or abort.
+
+A transaction may generate enough data that it is not practical to
+always hold pending updates in memory.  Instead, a TransactionBuffer
+is used to store the data until a commit or abort.
+"""
+
+# A faster implementation might store trans data in memory until it
+# reaches a certain size.
+
+import cPickle
+import tempfile
+from threading import Lock
+
+class TransactionBuffer:
+
+    # Valid call sequences:
+    #
+    #     ((store | invalidate)* begin_iterate next* clear)* close
+    #
+    # get_size can be called any time
+
+    # The TransactionBuffer is used by client storage to hold update
+    # data until the tpc_finish().  It is normally used by a single
+    # thread, because only one thread can be in the two-phase commit
+    # at one time.
+
+    # It is possible, however, for one thread to close the storage
+    # while another thread is in the two-phase commit.  We must use
+    # a lock to guard against this race, because unpredictable things
+    # can happen in Python if one thread closes a file that another
+    # thread is reading.  In a debug build, an assert() can fail.
+
+    # XXX If an operation is performed on a closed TransactionBuffer,
+    # it has no effect and does not raise an exception.  The only time
+    # this should occur is when a ClientStorage is closed in one
+    # thread while another thread is in its tpc_finish().  It's not
+    # clear what should happen in this case.  If the tpc_finish()
+    # completes without error, the Connection using it could have
+    # inconsistent data.  This should have minimal effect, though,
+    # because the Connection is connected to a closed storage.
+
+    def __init__(self):
+        self.file = tempfile.TemporaryFile(suffix=".tbuf")
+        self.lock = Lock()
+        self.closed = 0
+        self.count = 0
+        self.size = 0
+        # It's safe to use a fast pickler because the only objects
+        # stored are builtin types -- strings or None.
+        self.pickler = cPickle.Pickler(self.file, 1)
+        self.pickler.fast = 1
+
+    def close(self):
+        self.lock.acquire()
+        try:
+            self.closed = 1
+            try:
+                self.file.close()
+            except OSError:
+                pass
+        finally:
+            self.lock.release()
+
+    def store(self, oid, version, data):
+        self.lock.acquire()
+        try:
+            self._store(oid, version, data)
+        finally:
+            self.lock.release()
+
+    def _store(self, oid, version, data):
+        """Store oid, version, data for later retrieval"""
+        if self.closed:
+            return
+        self.pickler.dump((oid, version, data))
+        self.count += 1
+        # Estimate per-record cache size
+        self.size = self.size + len(data) + 31
+        if version:
+            # Assume version data has same size as non-version data
+            self.size = self.size + len(version) + len(data) + 12
+
+    def invalidate(self, oid, version):
+        self.lock.acquire()
+        try:
+            if self.closed:
+                return
+            self.pickler.dump((oid, version, None))
+            self.count += 1
+        finally:
+            self.lock.release()
+
+    def clear(self):
+        """Mark the buffer as empty"""
+        self.lock.acquire()
+        try:
+            if self.closed:
+                return
+            self.file.seek(0)
+            self.count = 0
+            self.size = 0
+        finally:
+            self.lock.release()
+
+    def __iter__(self):
+        self.lock.acquire()
+        try:
+            if self.closed:
+                return
+            self.file.flush()
+            self.file.seek(0)
+            return TBIterator(self.file, self.count)
+        finally:
+            self.lock.release()
+
+class TBIterator(object):
+
+    def __init__(self, f, count):
+        self.file = f
+        self.count = count
+        self.unpickler = cPickle.Unpickler(f)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        """Return next tuple of data or None if EOF"""
+        if self.count == 0:
+            self.file.seek(0)
+            self.size = 0
+            raise StopIteration
+        oid_ver_data = self.unpickler.load()
+        self.count -= 1
+        return oid_ver_data

lib/python/ZEO/__init__.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""ZEO -- Zope Enterprise Objects.
+
+See the file README.txt in this directory for an overview.
+
+ZEO is now part of ZODB; ZODB's home on the web is
+
+    http://www.zope.org/Wikis/ZODB
+
+"""
+
+# The next line must use double quotes, so replace.py recognizes it.
+version = "2.3a3"

lib/python/ZEO/auth/__init__.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+_auth_modules = {}
+
+def get_module(name):
+    if name == 'sha':
+        from auth_sha import StorageClass, SHAClient, Database
+        return StorageClass, SHAClient, Database
+    elif name == 'digest':
+        from auth_digest import StorageClass, DigestClient, DigestDatabase
+        return StorageClass, DigestClient, DigestDatabase
+    else:
+        return _auth_modules.get(name)
+
+def register_module(name, storage_class, client, db):
+    if _auth_modules.has_key(name):
+        raise TypeError, "%s is already registred" % name
+    _auth_modules[name] = storage_class, client, db

lib/python/ZEO/auth/auth_digest.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Digest authentication for ZEO
+
+This authentication mechanism follows the design of HTTP digest
+authentication (RFC 2069).  It is a simple challenge-response protocol
+that does not send passwords in the clear, but does not offer strong
+security.  The RFC discusses many of the limitations of this kind of
+protocol.
+
+Guard the password database as if it contained plaintext passwords.
+It stores the hash of a username and password.  This does not expose
+the plaintext password, but it is sensitive nonetheless.  An attacker
+with the hash can impersonate the real user.  This is a limitation of
+the simple digest scheme.
+
+HTTP is a stateless protocol, and ZEO is a stateful protocol.  The
+security requirements are quite different as a result.  The HTTP
+protocol uses a nonce as a challenge.  The ZEO protocol requires a
+separate session key that is used for message authentication.  We
+generate a second nonce for this purpose; the hash of nonce and
+user/realm/password is used as the session key.  XXX I'm not sure if
+this is a sound approach; SRP would be preferred.
+"""
+
+import os
+import random
+import sha
+import struct
+import time
+
+from ZEO.auth.base import Database, Client
+from ZEO.StorageServer import ZEOStorage
+from ZEO.Exceptions import AuthError
+
+def get_random_bytes(n=8):
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+    else:
+        L = [chr(random.randint(0, 255)) for i in range(n)]
+        s = "".join(L)
+    return s
+
+def hexdigest(s):
+    return sha.new(s).hexdigest()
+
+class DigestDatabase(Database):
+    def __init__(self, filename, realm=None):
+        Database.__init__(self, filename, realm)
+
+        # Initialize a key used to build the nonce for a challenge.
+        # We need one key for the lifetime of the server, so it
+        # is convenient to store in on the database.
+        self.noncekey = get_random_bytes(8)
+
+    def _store_password(self, username, password):
+        dig = hexdigest("%s:%s:%s" % (username, self.realm, password))
+        self._users[username] = dig
+
+def session_key(h_up, nonce):
+    # The hash itself is a bit too short to be a session key.
+    # HMAC wants a 64-byte key.  We don't want to use h_up
+    # directly because it would never change over time.  Instead
+    # use the hash plus part of h_up.
+    return sha.new("%s:%s" % (h_up, nonce)).digest() + h_up[:44]
+
+class StorageClass(ZEOStorage):
+    def set_database(self, database):
+        assert isinstance(database, DigestDatabase)
+        self.database = database
+        self.noncekey = database.noncekey
+
+    def _get_time(self):
+        # Return a string representing the current time.
+        t = int(time.time())
+        return struct.pack("i", t)
+
+    def _get_nonce(self):
+        # RFC 2069 recommends a nonce of the form
+        # H(client-IP ":" time-stamp ":" private-key)
+        dig = sha.sha()
+        dig.update(str(self.connection.addr))
+        dig.update(self._get_time())
+        dig.update(self.noncekey)
+        return dig.hexdigest()
+
+    def auth_get_challenge(self):
+        """Return realm, challenge, and nonce."""
+        self._challenge = self._get_nonce()
+        self._key_nonce = self._get_nonce()
+        return self.auth_realm, self._challenge, self._key_nonce
+
+    def auth_response(self, resp):
+        # verify client response
+        user, challenge, response = resp
+
+        # Since zrpc is a stateful protocol, we just store the nonce
+        # we sent to the client.  It will need to generate a new
+        # nonce for a new connection anyway.
+        if self._challenge != challenge:
+            raise ValueError, "invalid challenge"
+
+        # lookup user in database
+        h_up = self.database.get_password(user)
+
+        # regeneration resp from user, password, and nonce
+        check = hexdigest("%s:%s" % (h_up, challenge))
+        if check == response:
+            self.connection.setSessionKey(session_key(h_up, self._key_nonce))
+        return self.finish_auth(check == response)
+
+    extensions = [auth_get_challenge, auth_response]
+
+class DigestClient(Client):
+    extensions = ["auth_get_challenge", "auth_response"]
+
+    def start(self, username, realm, password):
+        _realm, challenge, nonce = self.stub.auth_get_challenge()
+        if _realm != realm:
+            raise AuthError("expected realm %r, got realm %r"
+                            % (_realm, realm))
+        h_up = hexdigest("%s:%s:%s" % (username, realm, password))
+
+        resp_dig = hexdigest("%s:%s" % (h_up, challenge))
+        result = self.stub.auth_response((username, challenge, resp_dig))
+        if result:
+            return session_key(h_up, nonce)
+        else:
+            return None

lib/python/ZEO/auth/base.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Base classes for defining an authentication protocol.
+
+Database -- abstract base class for password database
+Client -- abstract base class for authentication client
+"""
+
+import os
+import sha
+
+class Client:
+    # Subclass should override to list the names of methods that
+    # will be called on the server.
+    extensions = []
+
+    def __init__(self, stub):
+        self.stub = stub
+        for m in self.extensions:
+            setattr(self.stub, m, self.stub.extensionMethod(m))
+
+def sort(L):
+    """Sort a list in-place and return it."""
+    L.sort()
+    return L
+
+class Database:
+    """Abstracts a password database.
+
+    This class is used both in the authentication process (via
+    get_password()) and by client scripts that manage the password
+    database file.
+
+    The password file is a simple, colon-separated text file mapping
+    usernames to password hashes. The hashes are SHA hex digests
+    produced from the password string.
+    """
+    realm = None
+    def __init__(self, filename, realm=None):
+        """Creates a new Database
+
+        filename: a string containing the full pathname of
+            the password database file. Must be readable by the user
+            running ZEO. Must be writeable by any client script that
+            accesses the database.
+
+        realm: the realm name (a string)
+        """
+        self._users = {}
+        self.filename = filename
+        self.load()
+        if realm:
+            if self.realm and self.realm != realm:
+                raise ValueError, ("Specified realm %r differs from database "
+                                   "realm %r" % (realm or '', self.realm))
+            else:
+                self.realm = realm
+
+    def save(self, fd=None):
+        filename = self.filename
+
+        if not fd:
+            fd = open(filename, 'w')
+        if self.realm:
+            print >> fd, "realm", self.realm
+
+        for username in sort(self._users.keys()):
+            print >> fd, "%s: %s" % (username, self._users[username])
+
+    def load(self):
+        filename = self.filename
+        if not filename:
+            return
+
+        if not os.path.exists(filename):
+            return
+
+        fd = open(filename)
+        L = fd.readlines()
+
+        if not L:
+            return
+
+        if L[0].startswith("realm "):
+            line = L.pop(0).strip()
+            self.realm = line[len("realm "):]
+
+        for line in L:
+            username, hash = line.strip().split(":", 1)
+            self._users[username] = hash.strip()
+
+    def _store_password(self, username, password):
+        self._users[username] = self.hash(password)
+
+    def get_password(self, username):
+        """Returns password hash for specified username.
+
+        Callers must check for LookupError, which is raised in
+        the case of a non-existent user specified."""
+        if not self._users.has_key(username):
+            raise LookupError, "No such user: %s" % username
+        return self._users[username]
+
+    def hash(self, s):
+        return sha.new(s).hexdigest()
+
+    def add_user(self, username, password):
+        if self._users.has_key(username):
+            raise LookupError, "User %s already exists" % username
+        self._store_password(username, password)
+
+    def del_user(self, username):
+        if not self._users.has_key(username):
+            raise LookupError, "No such user: %s" % username
+        del self._users[username]
+
+    def change_password(self, username, password):
+        if not self._users.has_key(username):
+            raise LookupError, "No such user: %s" % username
+        self._store_password(username, password)

lib/python/ZEO/auth/hmac.py

+"""HMAC (Keyed-Hashing for Message Authentication) Python module.
+
+Implements the HMAC algorithm as described by RFC 2104.
+"""
+
+def _strxor(s1, s2):
+    """Utility method. XOR the two strings s1 and s2 (must have same length).
+    """
+    return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
+
+# The size of the digests returned by HMAC depends on the underlying
+# hashing module used.
+digest_size = None
+
+class HMAC:
+    """RFC2104 HMAC class.
+
+    This supports the API for Cryptographic Hash Functions (PEP 247).
+    """
+
+    def __init__(self, key, msg = None, digestmod = None):
+        """Create a new HMAC object.
+
+        key:       key for the keyed hash object.
+        msg:       Initial input for the hash, if provided.
+        digestmod: A module supporting PEP 247. Defaults to the md5 module.
+        """
+        if digestmod is None:
+            import md5
+            digestmod = md5
+
+        self.digestmod = digestmod
+        self.outer = digestmod.new()
+        self.inner = digestmod.new()
+        self.digest_size = digestmod.digest_size
+
+        blocksize = 64
+        ipad = "\x36" * blocksize
+        opad = "\x5C" * blocksize
+
+        if len(key) > blocksize:
+            key = digestmod.new(key).digest()
+
+        key = key + chr(0) * (blocksize - len(key))
+        self.outer.update(_strxor(key, opad))
+        self.inner.update(_strxor(key, ipad))
+        if msg is not None:
+            self.update(msg)
+
+##    def clear(self):
+##        raise NotImplementedError, "clear() method not available in HMAC."
+
+    def update(self, msg):
+        """Update this hashing object with the string msg.
+        """
+        self.inner.update(msg)
+
+    def copy(self):
+        """Return a separate copy of this hashing object.
+
+        An update to this copy won't affect the original object.
+        """
+        other = HMAC("")
+        other.digestmod = self.digestmod
+        other.inner = self.inner.copy()
+        other.outer = self.outer.copy()
+        return other
+
+    def digest(self):
+        """Return the hash value of this hashing object.
+
+        This returns a string containing 8-bit data.  The object is
+        not altered in any way by this function; you can continue
+        updating the object after calling this function.
+        """
+        h = self.outer.copy()
+        h.update(self.inner.digest())
+        return h.digest()
+
+    def hexdigest(self):
+        """Like digest(), but returns a string of hexadecimal digits instead.
+        """
+        return "".join([hex(ord(x))[2:].zfill(2)
+                        for x in tuple(self.digest())])
+
+def new(key, msg = None, digestmod = None):
+    """Create a new hashing object and return it.
+
+    key: The starting key for the hash.
+    msg: if available, will immediately be hashed into the object's starting
+    state.
+
+    You can now feed arbitrary strings into the object using its update()
+    method, and can ask for the hash value at any time by calling its digest()
+    method.
+    """
+    return HMAC(key, msg, digestmod)

lib/python/ZEO/cache.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Disk-based client cache for ZEO.
+
+ClientCache exposes an API used by the ZEO client storage.  FileCache
+stores objects one disk using a 2-tuple of oid and tid as key.
+
+The upper cache's API is similar to a storage API with methods like
+load(), store(), and invalidate().  It manages in-memory data
+structures that allow it to map this richer API onto the simple
+key-based API of the lower-level cache.
+"""
+
+import bisect
+import logging
+import os
+import struct
+import tempfile
+import time
+
+from ZODB.utils import z64, u64
+
+##
+# A disk-based cache for ZEO clients.
+# <p>
+# This class provides an interface to a persistent, disk-based cache
+# used by ZEO clients to store copies of database records from the
+# server.
+# <p>
+# The details of the constructor as unspecified at this point.
+# <p>
+# Each entry in the cache is valid for a particular range of transaction
+# ids.  The lower bound is the transaction that wrote the data.  The
+# upper bound is the next transaction that wrote a revision of the
+# object.  If the data is current, the upper bound is stored as None;
+# the data is considered current until an invalidate() call is made.
+# <p>
+# It is an error to call store() twice with the same object without an
+# intervening invalidate() to set the upper bound on the first cache
+# entry.  <em>Perhaps it will be necessary to have a call the removes
+# something from the cache outright, without keeping a non-current
+# entry.</em>
+# <h3>Cache verification</h3>
+# <p>
+# When the client is connected to the server, it receives
+# invalidations every time an object is modified.  Whe the client is
+# disconnected, it must perform cache verification to make sure its
+# cached data is synchronized with the storage's current state.
+# <p>
+# quick verification
+# full verification
+# <p>
+
+class ClientCache:
+    """A simple in-memory cache."""
+
+    ##
+    # Do we put the constructor here?
+    # @param path path of persistent snapshot of cache state
+    # @param size maximum size of object data, in bytes
+
+    def __init__(self, path=None, size=None, trace=False):
+        self.path = path
+        self.size = size
+        self.log = logging.getLogger("zeo.cache")
+
+        if trace and path:
+            self._setup_trace()
+        else:
+            self._trace = self._notrace
+
+        # Last transaction seen by the cache, either via setLastTid()
+        # or by invalidate().
+        self.tid = None
+
+        # The cache stores objects in a dict mapping (oid, tid) pairs
+        # to Object() records (see below).  The tid is the transaction
+        # id that wrote the object.  An object record includes data,
+        # serialno, and end tid.  It has auxillary data structures to
+        # compute the appropriate tid, given the oid and a transaction id
+        # representing an arbitrary point in history.
+        #
+        # The serialized form of the cache just stores the Object()
+        # records.  The in-memory form can be reconstructed from these
+        # records.
+
+        # Maps oid to current tid.  Used to find compute key for objects.
+        self.current = {}
+        # Maps oid to list of (start_tid, end_tid) pairs in sorted order.
+        # Used to find matching key for load of non-current data.
+        self.noncurrent = {}
+        # Map oid to version, tid pair.  If there is no entry, the object
+        # is not modified in a version.
+        self.version = {}
+
+        # A double-linked list is used to manage the cache.  It makes
+        # decisions about which objects to keep and which to evict.
+        self.fc = FileCache(size or 10**6, self.path, self)
+
+    def open(self):
+        self.fc.scan(self.install)
+
+    def install(self, f, ent):
+        # Called by cache storage layer to insert object
+        o = Object.fromFile(f, ent.key, header_only=True)
+        if o is None:
+            return
+        oid = o.key[0]
+        if o.version:
+            self.version[oid] = o.version, o.start_tid
+        elif o.end_tid is None:
+            self.current[oid] = o.start_tid
+        else:
+            L = self.noncurrent.setdefault(oid, [])
+            bisect.insort_left(L, (o.start_tid, o.end_tid))
+
+    def close(self):
+        self.fc.close()
+
+    ##
+    # Set the last transaction seen by the cache.
+    # @param tid a transaction id
+    # @exception ValueError attempt to set a new tid less than the current tid
+
+    def setLastTid(self, tid):
+        self.fc.settid(tid)
+
+    ##
+    # Return the last transaction seen by the cache.
+    # @return a transaction id
+    # @defreturn string
+
+    def getLastTid(self):
+        if self.fc.tid == z64:
+            return None
+        else:
+            return self.fc.tid
+
+    ##
+    # Return the current data record for oid and version.
+    # @param oid object id
+    # @param version a version string
+    # @return data record, serial number, tid or None if the object is not
+    #         in the cache
+    # @defreturn 3-tuple: (string, string, string)
+
+    def load(self, oid, version=""):
+        tid = None
+        if version:
+            p = self.version.get(oid)
+            if p is None:
+                return None
+            elif p[0] == version:
+                tid = p[1]
+            # Otherwise, we know the cache has version data but not
+            # for the requested version.  Thus, we know it is safe
+            # to return the non-version data from the cache.
+        if tid is None:
+            tid = self.current.get(oid)
+        if tid is None:
+            self._trace(0x20, oid, version)
+            return None
+        o = self.fc.access((oid, tid))
+        if o is None:
+            return None
+        self._trace(0x22, oid, version, o.start_tid, o.end_tid, len(o.data))
+        return o.data, tid, o.version
+
+    ##
+    # Return a non-current revision of oid that was current before tid.
+    # @param oid object id
+    # @param tid id of transaction that wrote next revision of oid
+    # @return data record, serial number, start tid, and end tid
+    # @defreturn 4-tuple: (string, string, string, string)
+
+    def loadBefore(self, oid, tid):
+        L = self.noncurrent.get(oid)
+        if L is None:
+            self._trace(0x24, oid, tid)
+            return None
+        # A pair with None as the second element will always be less
+        # than any pair with the same first tid.
+        i = bisect.bisect_left(L, (tid, None))
+        # The least element left of tid was written before tid.  If
+        # there is no element, the cache doesn't have old enough data.
+        if i == 0:
+            self._trace(0x24, oid, tid)
+            return
+        lo, hi = L[i-1]
+        # XXX lo should always be less than tid
+        if not lo < tid <= hi:
+            self._trace(0x24, oid, tid)
+            return None
+        o = self.fc.access((oid, lo))
+        self._trace(0x26, oid, tid)
+        return o.data, o.start_tid, o.end_tid
+
+    ##
+    # Return the version an object is modified in or None for an
+    # object that is not modified in a version.
+    # @param oid object id
+    # @return name of version in which the object is modified
+    # @defreturn string or None
+
+    def modifiedInVersion(self, oid):
+        p = self.version.get(oid)
+        if p is None:
+            return None
+        version, tid = p
+        return version
+
+    ##
+    # Store a new data record in the cache.
+    # @param oid object id
+    # @param version name of version that oid was modified in.  The cache
+    #                only stores current version data, so end_tid should
+    #                be None.
+    # @param start_tid the id of the transaction that wrote this revision
+    # @param end_tid the id of the transaction that created the next
+    #                revision of oid.  If end_tid is None, the data is
+    #                current.
+    # @param data the actual data
+    # @exception ValueError tried to store non-current version data
+
+    def store(self, oid, version, start_tid, end_tid, data):
+        # It's hard for the client to avoid storing the same object
+        # more than once.  One case is whether the client requests
+        # version data that doesn't exist.  It checks the cache for
+        # the requested version, doesn't find it, then asks the server
+        # for that data.  The server returns the non-version data,
+        # which may already by in the cache.
+        if (oid, start_tid) in self.fc:
+            return
+        o = Object((oid, start_tid), version, data, start_tid, end_tid)
+        if version:
+            if end_tid is not None:
+                raise ValueError("cache only stores current version data")
+            if oid in self.version:
+                if self.version[oid] != (version, start_tid):
+                    raise ValueError("data already exists for version %r"
+                                     % self.version[oid][0])
+            self.version[oid] = version, start_tid
+            self._trace(0x50, oid, version, start_tid, dlen=len(data))
+        else:
+            if end_tid is None:
+                _cur_start = self.current.get(oid)
+                if _cur_start:
+                    if _cur_start != start_tid:
+                        raise ValueError(
+                            "already have current data for oid")
+                    else:
+                        return
+                self.current[oid] = start_tid
+                self._trace(0x52, oid, version, start_tid, dlen=len(data))
+            else:
+                L = self.noncurrent.setdefault(oid, [])
+                p = start_tid, end_tid
+                if p in L:
+                    return # duplicate store
+                bisect.insort_left(L, (start_tid, end_tid))
+                self._trace(0x54, oid, version, start_tid, end_tid,
+                            dlen=len(data))
+        self.fc.add(o)
+
+    ##
+    # Mark the current data for oid as non-current.  If there is no
+    # current data for oid, do nothing.
+    # @param oid object id
+    # @param version name of version to invalidate.
+    # @param tid the id of the transaction that wrote a new revision of oid
+
+    def invalidate(self, oid, version, tid):
+        if tid > self.fc.tid:
+            self.fc.settid(tid)
+        if oid in self.version:
+            self._trace(0x1A, oid, version, tid)
+            dllversion, dlltid = self.version[oid]
+            assert not version or version == dllversion, (version, dllversion)
+            # remove() will call unlink() to delete from self.version
+            self.fc.remove((oid, dlltid))
+            # And continue on, we must also remove any non-version data
+            # from the cache.  This is a bit of a failure of the current
+            # cache consistency approach as the new tid of the version
+            # data gets confused with the old tid of the non-version data.
+            # I could sort this out, but it seems simpler to punt and
+            # have the cache invalidation too much for versions.
+
+        if oid not in self.current:
+            self._trace(0x10, oid, version, tid)
+            return
+        cur_tid = self.current.pop(oid)
+        # XXX Want to fetch object without marking it as accessed
+        o = self.fc.access((oid, cur_tid))
+        if o is None:
+            # XXX is this possible?
+            return None
+        o.end_tid = tid
+        self.fc.update(o)
+        self._trace(0x1C, oid, version, tid)
+        L = self.noncurrent.setdefault(oid, [])
+        bisect.insort_left(L, (cur_tid, tid))
+
+    ##
+    # Return the number of object revisions in the cache.
+
+    # XXX just return len(self.cache)?
+
+    def __len__(self):
+        n = len(self.current) + len(self.version)
+        if self.noncurrent:
+            n += sum(map(len, self.noncurrent))
+        return n
+
+    ##
+    # Generates over, version, serial triples for all objects in the
+    # cache.  This generator is used by cache verification.
+
+    def contents(self):
+        # XXX May need to materialize list instead of iterating,
+        # depends on whether the caller may change the cache.
+        for o in self.fc:
+            oid, tid = o.key
+            if oid in self.version:
+                obj = self.fc.access(o.key)
+                yield oid, tid, obj.version
+            else:
+                yield oid, tid, ""
+
+    def dump(self):
+        from ZODB.utils import oid_repr
+        print "cache size", len(self)
+        L = list(self.contents())
+        L.sort()
+        for oid, tid, version in L:
+            print oid_repr(oid), oid_repr(tid), repr(version)
+        print "dll contents"
+        L = list(self.fc)
+        L.sort(lambda x,y:cmp(x.key, y.key))
+        for x in L:
+            end_tid = x.end_tid or z64
+            print oid_repr(x.key[0]), oid_repr(x.key[1]), oid_repr(end_tid)
+        print
+
+    def _evicted(self, o):
+        # Called by Object o to signal its eviction
+        oid, tid = o.key
+        if o.end_tid is None:
+            if o.version:
+                del self.version[oid]
+            else:
+                del self.current[oid]
+        else:
+            # XXX Although we use bisect to keep the list sorted,
+            # we never expect the list to be very long.  So the
+            # brute force approach should normally be fine.
+            L = self.noncurrent[oid]
+            L.remove((o.start_tid, o.end_tid))
+
+    def _setup_trace(self):
+        tfn = self.path + ".trace"
+        self.tracefile = None
+        try:
+            self.tracefile = open(tfn, "ab")
+            self._trace(0x00)
+        except IOError, msg:
+            self.tracefile = None
+            self.log.warning("Could not write to trace file %s: %s",
+                             tfn, msg)
+
+    def _notrace(self, *arg, **kwargs):
+        pass
+
+    def _trace(self,
+               code, oid="", version="", tid="", end_tid=z64, dlen=0,
+               # The next two are just speed hacks.
+               time_time=time.time, struct_pack=struct.pack):
+        # The code argument is two hex digits; bits 0 and 7 must be zero.
+        # The first hex digit shows the operation, the second the outcome.
+        # If the second digit is in "02468" then it is a 'miss'.
+        # If it is in "ACE" then it is a 'hit'.
+        # This method has been carefully tuned to be as fast as possible.
+        # Note: when tracing is disabled, this method is hidden by a dummy.
+        if version:
+            code |= 0x80
+        encoded = (dlen + 255) & 0x7fffff00 | code
+        if tid is None:
+            tid = z64
+        if end_tid is None:
+            end_tid = z64
+        try:
+            self.tracefile.write(
+                struct_pack(">iiH8s8s",
+                            time_time(),
+                            encoded,
+                            len(oid),
+                            tid, end_tid) + oid)
+        except:
+            print `tid`, `end_tid`
+            raise
+
+##
+# An Object stores the cached data for a single object.
+# <p>
+# The cached data includes the actual object data, the key, and three
+# data fields that describe the validity period and version of the
+# object.  The key contains the oid and a redundant start_tid.  The
+# actual size of an object is variable, depending on the size of the
+# data and whether it is in a version.
+# <p>
+# The serialized format does not include the key, because it is stored
+# in the header used by the cache's storage format.
+
+class Object(object):
+    __slots__ = (# pair, object id, txn id -- something usable as a dict key
+                 # the second part of the part is equal to start_tid below
+                 "key",
+
+                 "start_tid", # string, id of txn that wrote the data
+                 "end_tid", # string, id of txn that wrote next revision
+                            # or None
+                 "version", # string, name of version
+                 "data", # string, the actual data record for the object
+
+                 "size", # total size of serialized object
+                )
+
+    def __init__(self, key, version, data, start_tid, end_tid):
+        self.key = key
+        self.version = version
+        self.data = data
+        self.start_tid = start_tid
+        self.end_tid = end_tid
+        # The size of a the serialized object on disk, include the
+        # 14-byte header, the length of data and version, and a
+        # copy of the 8-byte oid.
+        if data is not None:
+            self.size = 22 + len(data) + len(version)
+
+    # The serialization format uses an end tid of "\0" * 8, the least
+    # 8-byte string, to represent None.  It isn't possible for an
+    # end_tid to be 0, because it must always be strictly greater
+    # than the start_tid.
+
+    fmt = ">8shi"
+
+    def serialize(self, f):
+        # Write standard form of Object to file, f.
+        self.serialize_header(f)
+        f.write(self.data)
+        f.write(self.key[0])
+
+    def serialize_header(self, f):
+        s = struct.pack(self.fmt, self.end_tid or "\0" * 8,
+                        len(self.version), len(self.data))
+        f.write(s)
+        f.write(self.version)
+
+    def fromFile(cls, f, key, header_only=False):
+        s = f.read(struct.calcsize(cls.fmt))
+        if not s:
+            return None
+        oid, start_tid = key
+        end_tid, vlen, dlen = struct.unpack(cls.fmt, s)
+        if end_tid == z64:
+            end_tid = None
+        version = f.read(vlen)
+        if vlen != len(version):
+            raise ValueError("corrupted record, version")
+        if header_only:
+            data = None
+        else:
+            data = f.read(dlen)
+            if dlen != len(data):
+                raise ValueError("corrupted record, data")
+            s = f.read(8)
+            if s != oid:
+                raise ValueError("corrupted record, oid")
+        return cls((oid, start_tid), version, data, start_tid, end_tid)
+
+    fromFile = classmethod(fromFile)
+
+def sync(f):
+    f.flush()
+    if hasattr(os, 'fsync'):
+        os.fsync(f.fileno())
+
+class Entry(object):
+    __slots__ = (# object key -- something usable as a dict key.
+                 'key',
+
+                 # Offset from start of file to the object's data
+                 # record; this includes all overhead bytes (status
+                 # byte, size bytes, etc).  The size of the data
+                 # record is stored in the file near the start of the
+                 # record, but for efficiency we also keep size in a
+                 # dict (filemap; see later).
+                 'offset',
+                )
+
+    def __init__(self, key=None, offset=None):
+        self.key = key
+        self.offset = offset
+
+
+magic = "ZEC3"
+
+OBJECT_HEADER_SIZE = 1 + 4 + 16
+
+##
+# FileCache stores a cache in a single on-disk file.
+#
+# On-disk cache structure
+#
+# The file begins with a 12-byte header.  The first four bytes are the
+# file's magic number - ZEC3 - indicating zeo cache version 3.  The
+# next eight bytes are the last transaction id.
+#
+# The file is a contiguous sequence of blocks.  All blocks begin with
+# a one-byte status indicator:
+#
+# 'a'
+#       Allocated.  The block holds an object; the next 4 bytes are >I
+#       format total block size.
+#
+# 'f'
+#       Free.  The block is free; the next 4 bytes are >I format total
+#       block size.
+#
+# '1', '2', '3', '4'
+#       The block is free, and consists of 1, 2, 3 or 4 bytes total.
+#
+# 'Z'
+#       File header.  The file starts with a magic number, currently
+#       'ZEC3' and an 8-byte transaction id.
+#
+# "Total" includes the status byte, and size bytes.  There are no
+# empty (size 0) blocks.
+
+
+# XXX This needs a lot more hair.
+# The structure of an allocated block is more complicated:
+#
+#     1 byte allocation status ('a').
+#     4 bytes block size, >I format.
+#     16 bytes oid + tid, string.
+#     size-OBJECT_HEADER_SIZE bytes, the object pickle.
+
+# The cache's currentofs goes around the file, circularly, forever.
+# It's always the starting offset of some block.
+#
+# When a new object is added to the cache, it's stored beginning at
+# currentofs, and currentofs moves just beyond it.  As many contiguous
+# blocks needed to make enough room for the new object are evicted,
+# starting at currentofs.  Exception:  if currentofs is close enough
+# to the end of the file that the new object can't fit in one
+# contiguous chunk, currentofs is reset to 0 first.
+
+# Do all possible to ensure that the bytes we wrote are really on
+# disk.
+
+class FileCache(object):
+
+    def __init__(self, maxsize, fpath, parent, reuse=True):
+        # Maximum total of object sizes we keep in cache.
+        self.maxsize = maxsize
+        # Current total of object sizes in cache.
+        self.currentsize = 0
+        self.parent = parent
+        self.tid = None
+
+        # Map offset in file to pair (data record size, Entry).
+        # Entry is None iff the block starting at offset is free.
+        # filemap always contains a complete account of what's in the
+        # file -- study method _verify_filemap for executable checking
+        # of the relevant invariants.  An offset is at the start of a
+        # block iff it's a key in filemap.
+        self.filemap = {}
+
+        # Map key to Entry.  There's one entry for each object in the
+        # cache file.  After
+        #     obj = key2entry[key]
+        # then
+        #     obj.key == key
+        # is true.
+        self.key2entry = {}
+
+        # Always the offset into the file of the start of a block.
+        # New and relocated objects are always written starting at
+        # currentofs.
+        self.currentofs = 12
+
+        self.fpath = fpath
+        if not reuse or not fpath or not os.path.exists(fpath):
+            self.new = True
+            if fpath:
+                self.f = file(fpath, 'wb+')
+            else:
+                self.f = tempfile.TemporaryFile()
+            # Make sure the OS really saves enough bytes for the file.
+            self.f.seek(self.maxsize - 1)
+            self.f.write('x')
+            self.f.truncate()
+            # Start with one magic header block
+            self.f.seek(0)
+            self.f.write(magic)
+            self.f.write(z64)
+            # and one free block.
+            self.f.write('f' + struct.pack(">I", self.maxsize - 12))
+            self.sync()
+            self.filemap[12] = self.maxsize - 12, None
+        else:
+            self.new = False
+            self.f = None
+
+        # Statistics:  _n_adds, _n_added_bytes,
+        #              _n_evicts, _n_evicted_bytes
+        self.clearStats()
+
+    # Scan the current contents of the cache file, calling install
+    # for each object found in the cache.  This method should only
+    # be called once to initialize the cache from disk.
+
+    def scan(self, install):
+        if self.new:
+            return
+        fsize = os.path.getsize(self.fpath)
+        self.f = file(self.fpath, 'rb+')
+        _magic = self.f.read(4)
+        if _magic != magic:
+            raise ValueError("unexpected magic number: %r" % _magic)
+        self.tid = self.f.read(8)
+        # Remember the largest free block.  That seems a
+        # decent place to start currentofs.
+        max_free_size = max_free_offset = 0
+        ofs = 12
+        while ofs < fsize:
+            self.f.seek(ofs)
+            ent = None
+            status = self.f.read(1)
+            if status == 'a':
+                size, rawkey = struct.unpack(">I16s", self.f.read(20))
+                key = rawkey[:8], rawkey[8:]
+                assert key not in self.key2entry
+                self.key2entry[key] = ent = Entry(key, ofs)
+                install(self.f, ent)
+            elif status == 'f':
+                size, = struct.unpack(">I", self.f.read(4))
+            elif status in '1234':
+                size = int(status)
+            else:
+                assert 0, hex(ord(status))
+
+            self.filemap[ofs] = size, ent
+            if ent is None and size > max_free_size:
+                max_free_size, max_free_offset = size, ofs
+
+            ofs += size
+
+        assert ofs == fsize
+        if __debug__:
+            self._verify_filemap()
+        self.currentofs = max_free_offset
+
+    def clearStats(self):
+        self._n_adds = self._n_added_bytes = 0
+        self._n_evicts = self._n_evicted_bytes = 0
+        self._n_removes = self._n_removed_bytes = 0
+        self._n_accesses = 0
+
+    def getStats(self):
+        return (self._n_adds, self._n_added_bytes,
+                self._n_evicts, self._n_evicted_bytes,
+                self._n_removes, self._n_removed_bytes,
+                self._n_accesses
+               )
+
+    def __len__(self):
+        return len(self.key2entry)
+
+    def __iter__(self):
+        return self.key2entry.itervalues()
+
+    def __contains__(self, key):
+        return key in self.key2entry
+
+    def sync(self):
+        sync(self.f)
+
+    def close(self):
+        if self.f:
+            self.sync()
+            self.f.close()
+            self.f = None
+
+    # Evict objects as necessary to free up at least nbytes bytes,
+    # starting at currentofs.  If currentofs is closer than nbytes to
+    # the end of the file, currentofs is reset to 0.  The number of
+    # bytes actually freed may be (and probably will be) greater than
+    # nbytes, and is _makeroom's return value.  The file is not
+    # altered by _makeroom.  filemap is updated to reflect the
+    # evictions, and it's the caller's responsibilty both to fiddle
+    # the file, and to update filemap, to account for all the space
+    # freed (starting at currentofs when _makeroom returns, and
+    # spanning the number of bytes retured by _makeroom).
+
+    def _makeroom(self, nbytes):
+        assert 0 < nbytes <= self.maxsize
+        if self.currentofs + nbytes > self.maxsize:
+            self.currentofs = 12
+        ofs = self.currentofs
+        while nbytes > 0:
+            size, e = self.filemap.pop(ofs)
+            if e is not None:
+                self._evictobj(e, size)
+            ofs += size
+            nbytes -= size
+        return ofs - self.currentofs
+
+    # Write Object obj, with data, to file starting at currentofs.
+    # nfreebytes are already available for overwriting, and it's
+    # guranteed that's enough.  obj.offset is changed to reflect the
+    # new data record position, and filemap is updated to match.
+
+    def _writeobj(self, obj, nfreebytes):
+        size = OBJECT_HEADER_SIZE + obj.size
+        assert size <= nfreebytes
+        excess = nfreebytes - size
+        # If there's any excess (which is likely), we need to record a
+        # free block following the end of the data record.  That isn't
+        # expensive -- it's all a contiguous write.
+        if excess == 0:
+            extra = ''
+        elif excess < 5:
+            extra = "01234"[excess]
+        else:
+            extra = 'f' + struct.pack(">I", excess)
+
+        self.f.seek(self.currentofs)
+        self.f.writelines(('a',
+                           struct.pack(">I8s8s", size,
+                                       obj.key[0], obj.key[1])))
+        obj.serialize(self.f)
+        self.f.write(extra)
+        e = Entry(obj.key, self.currentofs)
+        self.key2entry[obj.key] = e
+        self.filemap[self.currentofs] = size, e
+        self.currentofs += size
+        if excess:
+            # We need to record the free block in filemap, but there's
+            # no need to advance currentofs beyond it.  Instead it
+            # gives some breathing room for the next object to get
+            # written.
+            self.filemap[self.currentofs] = excess, None
+
+    def add(self, object):
+        size = OBJECT_HEADER_SIZE + object.size
+        if size > self.maxsize:
+            return
+        assert size <= self.maxsize
+
+        assert object.key not in self.key2entry
+        assert len(object.key[0]) == 8
+        assert len(object.key[1]) == 8
+
+        self._n_adds += 1
+        self._n_added_bytes += size
+
+        available = self._makeroom(size)
+        self._writeobj(object, available)
+
+    def _verify_filemap(self, display=False):
+        a = 12
+        f = self.f
+        while a < self.maxsize:
+            f.seek(a)
+            status = f.read(1)
+            if status in 'af':
+                size, = struct.unpack(">I", f.read(4))
+            else:
+                size = int(status)
+            if display:
+                if a == self.currentofs:
+                    print '*****',
+                print "%c%d" % (status, size),
+            size2, obj = self.filemap[a]
+            assert size == size2
+            assert (obj is not None) == (status == 'a')
+            if obj is not None:
+                assert obj.offset == a
+                assert self.key2entry[obj.key] is obj
+            a += size
+        if display:
+            print
+        assert a == self.maxsize
+
+    def _evictobj(self, e, size):
+        self._n_evicts += 1
+        self._n_evicted_bytes += size
+        # Load the object header into memory so we know how to
+        # update the parent's in-memory data structures.
+        self.f.seek(e.offset + OBJECT_HEADER_SIZE)
+        o = Object.fromFile(self.f, e.key, header_only=True)
+        self.parent._evicted(o)
+
+    ##
+    # Return object for key or None if not in cache.
+
+    def access(self, key):
+        self._n_accesses += 1
+        e = self.key2entry.get(key)
+        if e is None:
+            return None
+        offset = e.offset
+        size, e2 = self.filemap[offset]
+        assert e is e2
+
+        self.f.seek(offset + OBJECT_HEADER_SIZE)
+        return Object.fromFile(self.f, key)
+
+    ##
+    # Remove object for key from cache, if present.
+
+    def remove(self, key):
+        # If an object is being explicitly removed, we need to load
+        # its header into memory and write a free block marker to the
+        # disk where the object was stored.  We need to load the
+        # header to update the in-memory data structures held by
+        # ClientCache.
+
+        # XXX Or we could just keep the header in memory at all times.
+
+        e = self.key2entry.get(key)
+        if e is None:
+            return
+        offset = e.offset
+        size, e2 = self.filemap[offset]
+        self.f.seek(offset + OBJECT_HEADER_SIZE)
+        o = Object.fromFile(self.f, key, header_only=True)
+        self.f.seek(offset + OBJECT_HEADER_SIZE)
+        self.f.write('f')
+        self.f.flush()
+        self.parent._evicted(o)
+        self.filemap[offset] = size, None
+
+    ##
+    # Update on-disk representation of obj.
+    #
+    # This method should be called when the object header is modified.
+
+    def update(self, obj):
+
+        e = self.key2entry[obj.key]
+        self.f.seek(e.offset + OBJECT_HEADER_SIZE)
+        obj.serialize_header(self.f)
+
+    def settid(self, tid):
+        if self.tid is not None and tid <= self.tid:
+            raise ValueError("new last tid (%s) must be greater than "
+                             "previous one (%s)" % (u64(tid),
+                                                    u64(self.tid)))
+        self.tid = tid
+        self.f.seek(4)
+        self.f.write(tid)
+        self.f.flush()

lib/python/ZEO/component.xml

+<component>
+
+  <sectiontype name="zeo">
+
+    <description>
+      The content of a ZEO section describe operational parameters
+      of a ZEO server except for the storage(s) to be served.
+    </description>
+
+    <key name="address" datatype="socket-address"
+         required="yes">
+      <description>
+        The address at which the server should listen.  This can be in
+        the form 'host:port' to signify a TCP/IP connection or a
+        pathname string to signify a Unix domain socket connection (at
+        least one '/' is required).  A hostname may be a DNS name or a
+        dotted IP address.  If the hostname is omitted, the platform's
+        default behavior is used when binding the listening socket (''
+        is passed to socket.bind() as the hostname portion of the
+        address).
+      </description>
+    </key>
+
+    <key name="read-only" datatype="boolean"
+         required="no"
+         default="false">
+      <description>
+        Flag indicating whether the server should operate in read-only
+        mode.  Defaults to false.  Note that even if the server is
+        operating in writable mode, individual storages may still be
+        read-only.  But if the server is in read-only mode, no write
+        operations are allowed, even if the storages are writable.  Note
+        that pack() is considered a read-only operation.
+      </description>
+    </key>
+
+    <key name="invalidation-queue-size" datatype="integer"
+         required="no"
+         default="100">
+      <description>
+        The storage server keeps a queue of the objects modified by the
+        last N transactions, where N == invalidation_queue_size.  This
+        queue is used to speed client cache verification when a client
+        disconnects for a short period of time.
+      </description>
+    </key>
+
+    <key name="monitor-address" datatype="socket-address"
+         required="no">
+      <description>
+        The address at which the monitor server should listen.  If
+        specified, a monitor server is started.  The monitor server
+        provides server statistics in a simple text format.  This can
+        be in the form 'host:port' to signify a TCP/IP connection or a
+        pathname string to signify a Unix domain socket connection (at
+        least one '/' is required).  A hostname may be a DNS name or a
+        dotted IP address.  If the hostname is omitted, the platform's
+        default behavior is used when binding the listening socket (''
+        is passed to socket.bind() as the hostname portion of the
+        address).
+      </description>
+    </key>
+
+    <key name="transaction-timeout" datatype="integer"
+         required="no">
+      <description>
+        The maximum amount of time to wait for a transaction to commit
+        after acquiring the storage lock, specified in seconds.  If the
+        transaction takes too long, the client connection will be closed
+        and the transaction aborted.
+      </description>
+    </key>
+
+    <key name="authentication-protocol" required="no">
+      <description>
+        The name of the protocol used for authentication.  The
+        only protocol provided with ZEO is "digest," but extensions
+        may provide other protocols.
+      </description>
+    </key>
+
+    <key name="authentication-database" required="no">
+      <description>
+        The path of the database containing authentication credentials.
+      </description>
+    </key>
+
+    <key name="authentication-realm" required="no">
+      <description>
+        The authentication realm of the server.  Some authentication
+        schemes use a realm to identify the logical set of usernames
+        that are accepted by this server.
+      </description>
+    </key>
+
+  </sectiontype>
+
+</component>

lib/python/ZEO/mkzeoinst.py

+#!python
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""%(program)s -- create a ZEO instance.
+
+Usage: %(program)s home [port]
+
+Given an "instance home directory" <home> and some configuration
+options (all of which have default values), create the following:
+
+<home>/etc/zeo.conf     -- ZEO config file
+<home>/var/             -- Directory for data files: Data.fs etc.
+<home>/log/             -- Directory for log files: zeo.log and zeoctl.log
+<home>/bin/runzeo       -- the zeo server runner
+<home>/bin/zeoctl       -- start/stop script (a shim for zeoctl.py)
+
+The script will not overwrite existing files; instead, it will issue a
+warning if an existing file is found that differs from the file that
+would be written if it didn't exist.
+"""
+
+# WARNING!  Several templates and functions here are reused by ZRS.
+# So be careful with changes.
+
+import os
+import sys
+import stat
+import getopt
+
+zeo_conf_template = """\
+# ZEO configuration file
+
+%%define INSTANCE %(instance_home)s
+
+<zeo>
+  address %(port)d
+  read-only false
+  invalidation-queue-size 100
+  # monitor-address PORT
+  # transaction-timeout SECONDS
+</zeo>
+
+<filestorage 1>
+  path $INSTANCE/var/Data.fs
+</filestorage>
+
+<eventlog>
+  level info
+  <logfile>
+    path $INSTANCE/log/zeo.log
+  </logfile>
+</eventlog>
+
+<runner>
+  program $INSTANCE/bin/runzeo
+  socket-name $INSTANCE/etc/%(package)s.zdsock
+  daemon true
+  forever false
+  backoff-limit 10
+  exit-codes 0, 2
+  directory $INSTANCE
+  default-to-interactive true
+  # user zope
+  python %(python)s
+  zdrun %(zodb3_home)s/zdaemon/zdrun.py
+
+  # This logfile should match the one in the %(package)s.conf file.
+  # It is used by zdctl's logtail command, zdrun/zdctl doesn't write it.
+  logfile $INSTANCE/log/%(package)s.log
+</runner>
+"""
+
+zeoctl_template = """\
+#!/bin/sh
+# %(PACKAGE)s instance control script
+
+# The following two lines are for chkconfig.  On Red Hat Linux (and
+# some other systems), you can copy or symlink this script into
+# /etc/rc.d/init.d/ and then use chkconfig(8) to automatically start
+# %(PACKAGE)s at boot time.
+
+# chkconfig: 345 90 10
+# description: start a %(PACKAGE)s server
+
+PYTHON="%(python)s"
+ZODB3_HOME="%(zodb3_home)s"
+
+CONFIG_FILE="%(instance_home)s/etc/%(package)s.conf"
+
+PYTHONPATH="$ZODB3_HOME"
+export PYTHONPATH
+
+ZEOCTL="$ZODB3_HOME/ZEO/zeoctl.py"
+
+exec "$PYTHON" "$ZEOCTL" -C "$CONFIG_FILE" ${1+"$@"}
+"""
+
+runzeo_template = """\
+#!/bin/sh
+# %(PACKAGE)s instance start script
+
+PYTHON="%(python)s"
+ZODB3_HOME="%(zodb3_home)s"
+
+CONFIG_FILE="%(instance_home)s/etc/%(package)s.conf"
+
+PYTHONPATH="$ZODB3_HOME"
+export PYTHONPATH
+
+RUNZEO="$ZODB3_HOME/ZEO/runzeo.py"
+
+exec "$PYTHON" "$RUNZEO" -C "$CONFIG_FILE" ${1+"$@"}
+"""
+
+def main():
+    ZEOInstanceBuilder().run()
+    print "All done."
+
+class ZEOInstanceBuilder:
+    def run(self):
+        try:
+            opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
+        except getopt.error, msg:
+            print msg
+            sys.exit(2)
+        program = os.path.basename(sys.argv[0])
+        if opts:
+            # There's only the help options, so just dump some help:
+            msg = __doc__ % {"program": program}
+            print msg
+            sys.exit()
+        if len(args) not in [1, 2]:
+            print "Usage: %s home [port]" % program
+            sys.exit(2)
+
+        instance_home = args[0]
+        if not os.path.isabs(instance_home):
+            instance_home = os.path.abspath(instance_home)
+
+        for entry in sys.path:
+            if os.path.exists(os.path.join(entry, 'ZODB')):
+                zodb3_home = entry
+                break
+        else:
+            print "Can't find the Zope home (not in sys.path)"
+            sys.exit(2)
+
+        if args[1:]:
+            port = int(args[1])
+        else:
+            port = 9999
+
+        params = self.get_params(zodb3_home, instance_home, port)
+        self.create(instance_home, params)
+
+    def get_params(self, zodb3_home, instance_home, port):
+        return {
+            "package": "zeo",
+            "PACKAGE": "ZEO",
+            "zodb3_home": zodb3_home,
+            "instance_home": instance_home,
+            "port": port,
+            "python": sys.executable,
+            }
+
+    def create(self, home, params):
+        makedir(home)
+        makedir(home, "etc")
+        makedir(home, "var")
+        makedir(home, "log")
+        makedir(home, "bin")
+        makefile(zeo_conf_template, home, "etc", "zeo.conf", **params)
+        makexfile(zeoctl_template, home, "bin", "zeoctl", **params)
+        makexfile(runzeo_template, home, "bin", "runzeo", **params)
+
+
+def which(program):
+    strpath = os.getenv("PATH")
+    binpath = strpath.split(os.pathsep)
+    for dir in binpath:
+        path = os.path.join(dir, program)
+        if os.path.isfile(path) and os.access(path, os.X_OK):
+            if not os.path.isabs(path):
+                path = os.path.abspath(path)
+            return path
+    raise IOError, "can't find %r on path %r" % (program, strpath)
+
+def makedir(*args):
+    path = ""
+    for arg in args:
+        path = os.path.join(path, arg)
+    mkdirs(path)
+    return path
+
+def mkdirs(path):
+    if os.path.isdir(path):
+        return
+    head, tail = os.path.split(path)
+    if head and tail and not os.path.isdir(head):
+        mkdirs(head)
+    os.mkdir(path)
+    print "Created directory", path
+
+def makefile(template, *args, **kwds):
+    path = makedir(*args[:-1])
+    path = os.path.join(path, args[-1])
+    data = template % kwds
+    if os.path.exists(path):
+        f = open(path)
+        olddata = f.read().strip()
+        f.close()
+        if olddata:
+            if olddata != data.strip():
+                print "Warning: not overwriting existing file %r" % path
+            return path
+    f = open(path, "w")
+    f.write(data)
+    f.close()
+    print "Wrote file", path
+    return path
+
+def makexfile(template, *args, **kwds):
+    path = makefile(template, *args, **kwds)
+    umask = os.umask(022)
+    os.umask(umask)
+    mode = 0777 & ~umask
+    if stat.S_IMODE(os.stat(path)[stat.ST_MODE]) != mode:
+        os.chmod(path, mode)
+        print "Changed mode for %s to %o" % (path, mode)
+    return path
+
+if __name__ == "__main__":
+    main()

lib/python/ZEO/monitor.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Monitor behavior of ZEO server and record statistics.
+
+$Id: monitor.py,v 1.6 2004/04/25 11:34:15 gintautasm Exp $
+"""
+
+import asyncore
+import socket
+import time
+import types
+import logging
+
+import ZEO
+
+class StorageStats:
+    """Per-storage usage statistics."""
+
+    def __init__(self):
+        self.loads = 0
+        self.stores = 0
+        self.commits = 0
+        self.aborts = 0
+        self.active_txns = 0
+        self.clients = 0
+        self.verifying_clients = 0
+        self.lock_time = None
+        self.conflicts = 0
+        self.conflicts_resolved = 0
+        self.start = time.ctime()
+
+    def parse(self, s):
+        # parse the dump format
+        lines = s.split("\n")
+        for line in lines:
+            field, value = line.split(":", 1)
+            if field == "Server started":
+                self.start = value
+            elif field == "Clients":
+                self.clients = int(value)
+            elif field == "Clients verifying":
+                self.verifying_clients = int(value)
+            elif field == "Active transactions":
+                self.active_txns = int(value)
+            elif field == "Commit lock held for":
+                # This assumes
+                self.lock_time = time.time() - int(value)
+            elif field == "Commits":
+                self.commits = int(value)
+            elif field == "Aborts":
+                self.aborts = int(value)
+            elif field == "Loads":
+                self.loads = int(value)
+            elif field == "Stores":
+                self.stores = int(value)
+            elif field == "Conflicts":
+                self.conflicts = int(value)
+            elif field == "Conflicts resolved":
+                self.conflicts_resolved = int(value)
+
+    def dump(self, f):
+        print >> f, "Server started:", self.start
+        print >> f, "Clients:", self.clients
+        print >> f, "Clients verifying:", self.verifying_clients
+        print >> f, "Active transactions:", self.active_txns
+        if self.lock_time:
+            howlong = time.time() - self.lock_time
+            print >> f, "Commit lock held for:", int(howlong)
+        print >> f, "Commits:", self.commits
+        print >> f, "Aborts:", self.aborts
+        print >> f, "Loads:", self.loads
+        print >> f, "Stores:", self.stores
+        print >> f, "Conflicts:", self.conflicts
+        print >> f, "Conflicts resolved:", self.conflicts_resolved
+
+class StatsClient(asyncore.dispatcher):
+
+    def __init__(self, sock, addr):
+        asyncore.dispatcher.__init__(self, sock)
+        self.buf = []
+        self.closed = 0
+
+    def close(self):
+        self.closed = 1
+        # The socket is closed after all the data is written.
+        # See handle_write().
+
+    def write(self, s):
+        self.buf.append(s)
+
+    def writable(self):
+        return len(self.buf)
+
+    def readable(self):
+        # XXX what goes here?
+        return 0
+
+    def handle_write(self):
+        s = "".join(self.buf)
+        self.buf = []
+        n = self.socket.send(s)
+        if n < len(s):
+            self.buf.append(s[:n])
+
+        if self.closed and not self.buf:
+            asyncore.dispatcher.close(self)
+
+class StatsServer(asyncore.dispatcher):
+
+    StatsConnectionClass = StatsClient
+
+    def __init__(self, addr, stats):
+        asyncore.dispatcher.__init__(self)
+        self.addr = addr
+        self.stats = stats
+        if type(self.addr) == types.TupleType:
+            self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        else:
+            self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.set_reuse_addr()
+        logger = logging.getLogger('ZEO.monitor')
+        logger.info("listening on %s", repr(self.addr))
+        self.bind(self.addr)
+        self.listen(5)
+
+    def writable(self):
+        return 0
+
+    def readable(self):
+        return 1
+
+    def handle_accept(self):
+        try:
+            sock, addr = self.accept()
+        except socket.error:
+            return
+        f = self.StatsConnectionClass(sock, addr)
+        self.dump(f)
+        f.close()
+
+    def dump(self, f):
+        print >> f, "ZEO monitor server version %s" % ZEO.version
+        print >> f, time.ctime()
+        print >> f
+
+        L = self.stats.keys()
+        L.sort()
+        for k in L:
+            stats = self.stats[k]
+            print >> f, "Storage:", k
+            stats.dump(f)
+            print >> f

lib/python/ZEO/runzeo.py

+#!python
+##############################################################################
+#
+# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Start the ZEO storage server.
+
+Usage: %s [-C URL] [-a ADDRESS] [-f FILENAME] [-h]
+
+Options:
+-C/--configuration URL -- configuration file or URL
+-a/--address ADDRESS -- server address of the form PORT, HOST:PORT, or PATH
+                        (a PATH must contain at least one "/")
+-f/--filename FILENAME -- filename for FileStorage
+-t/--timeout TIMEOUT -- transaction timeout in seconds (default no timeout)
+-h/--help -- print this usage message and exit
+-m/--monitor ADDRESS -- address of monitor server ([HOST:]PORT or PATH)
+
+Unless -C is specified, -a and -f are required.
+"""
+
+# The code here is designed to be reused by other, similar servers.
+# For the forseeable future, it must work under Python 2.1 as well as
+# 2.2 and above.
+
+import os
+import sys
+import signal
+import socket
+import logging
+
+import ZConfig, ZConfig.datatypes
+import ZEO
+from zdaemon.zdoptions import ZDOptions
+
+logger = logging.getLogger('ZEO.runzeo')
+_pid = str(os.getpid())
+
+def log(msg, level=logging.INFO, exc_info=False):
+    """Internal: generic logging function."""
+    message = "(%s) %s" % (_pid, msg)
+    logger.log(level, message, exc_info=exc_info)
+
+
+def parse_address(arg):
+    # XXX Not part of the official ZConfig API
+    obj = ZConfig.datatypes.SocketAddress(arg)
+    return obj.family, obj.address
+
+
+class ZEOOptionsMixin:
+
+    storages = None
+
+    def handle_address(self, arg):
+        self.family, self.address = parse_address(arg)
+
+    def handle_monitor_address(self, arg):
+        self.monitor_family, self.monitor_address = parse_address(arg)
+
+    def handle_filename(self, arg):
+        from ZODB.config import FileStorage # That's a FileStorage *opener*!
+        class FSConfig:
+            def __init__(self, name, path):
+                self._name = name
+                self.path = path
+                self.create = 0
+                self.read_only = 0
+                self.stop = None
+                self.quota = None
+            def getSectionName(self):
+                return self._name
+        if not self.storages:
+            self.storages = []
+        name = str(1 + len(self.storages))
+        conf = FileStorage(FSConfig(name, arg))
+        self.storages.append(conf)
+
+    def add_zeo_options(self):
+        self.add(None, None, "a:", "address=", self.handle_address)
+        self.add(None, None, "f:", "filename=", self.handle_filename)
+        self.add("family", "zeo.address.family")
+        self.add("address", "zeo.address.address",
+                 required="no server address specified; use -a or -C")
+        self.add("read_only", "zeo.read_only", default=0)
+        self.add("invalidation_queue_size", "zeo.invalidation_queue_size",
+                 default=100)
+        self.add("transaction_timeout", "zeo.transaction_timeout",
+                 "t:", "timeout=", float)
+        self.add("monitor_address", "zeo.monitor_address.address",
+                 "m:", "monitor=", self.handle_monitor_address)
+        self.add('auth_protocol', 'zeo.authentication_protocol',
+                 None, 'auth-protocol=', default=None)
+        self.add('auth_database', 'zeo.authentication_database',
+                 None, 'auth-database=')
+        self.add('auth_realm', 'zeo.authentication_realm',
+                 None, 'auth-realm=')
+
+class ZEOOptions(ZDOptions, ZEOOptionsMixin):
+
+    logsectionname = "eventlog"
+    schemadir = os.path.dirname(ZEO.__file__)
+
+    def __init__(self):
+        ZDOptions.__init__(self)
+        self.add_zeo_options()
+        self.add("storages", "storages",
+                 required="no storages specified; use -f or -C")
+
+
+class ZEOServer:
+
+    def __init__(self, options):
+        self.options = options
+
+    def main(self):
+        self.setup_default_logging()
+        self.check_socket()
+        self.clear_socket()
+        try:
+            self.open_storages()
+            self.setup_signals()
+            self.create_server()
+            self.loop_forever()
+        finally:
+            self.close_storages()
+            self.clear_socket()
+
+    def setup_default_logging(self):
+        if self.options.config_logger is not None:
+            return
+        # No log file is configured; default to stderr.
+        logger = logging.getLogger()
+        handler = logging.StreamHandler()
+        handler.setLevel(logging.INFO)
+        logger.addHandler(handler)
+
+    def check_socket(self):
+        if self.can_connect(self.options.family, self.options.address):
+            self.options.usage("address %s already in use" %
+                               repr(self.options.address))
+
+    def can_connect(self, family, address):
+        s = socket.socket(family, socket.SOCK_STREAM)
+        try:
+            s.connect(address)
+        except socket.error:
+            return 0
+        else:
+            s.close()
+            return 1
+
+    def clear_socket(self):
+        if isinstance(self.options.address, type("")):
+            try:
+                os.unlink(self.options.address)
+            except os.error:
+                pass
+
+    def open_storages(self):
+        self.storages = {}
+        for opener in self.options.storages:
+            log("opening storage %r using %s"
+                % (opener.name, opener.__class__.__name__))
+            self.storages[opener.name] = opener.open()
+
+    def setup_signals(self):
+        """Set up signal handlers.
+
+        The signal handler for SIGFOO is a method handle_sigfoo().
+        If no handler method is defined for a signal, the signal
+        action is not changed from its initial value.  The handler
+        method is called without additional arguments.
+        """
+        if os.name != "posix":
+            return
+        if hasattr(signal, 'SIGXFSZ'):
+            signal.signal(signal.SIGXFSZ, signal.SIG_IGN) # Special case
+        init_signames()
+        for sig, name in signames.items():
+            method = getattr(self, "handle_" + name.lower(), None)
+            if method is not None:
+                def wrapper(sig_dummy, frame_dummy, method=method):
+                    method()
+                signal.signal(sig, wrapper)
+
+    def create_server(self):
+        from ZEO.StorageServer import StorageServer
+        self.server = StorageServer(
+            self.options.address,
+            self.storages,
+            read_only=self.options.read_only,
+            invalidation_queue_size=self.options.invalidation_queue_size,
+            transaction_timeout=self.options.transaction_timeout,
+            monitor_address=self.options.monitor_address,
+            auth_protocol=self.options.auth_protocol,
+            auth_database=self.options.auth_database,  # XXX option spelling
+            auth_realm=self.options.auth_realm)
+
+    def loop_forever(self):
+        import ThreadedAsync.LoopCallback
+        ThreadedAsync.LoopCallback.loop()
+
+    def handle_sigterm(self):
+        log("terminated by SIGTERM")
+        sys.exit(0)
+
+    def handle_sigint(self):
+        log("terminated by SIGINT")
+        sys.exit(0)
+
+    def handle_sighup(self):
+        log("restarted by SIGHUP")
+        sys.exit(1)
+
+    def handle_sigusr2(self):
+        # XXX this used to reinitialize zLOG. How do I achieve
+        #     the same effect with Python's logging package?
+        #     Should we restart as with SIGHUP?
+        log("received SIGUSR2, but it was not handled!", level=logging.WARNING)
+
+    def close_storages(self):
+        for name, storage in self.storages.items():
+            log("closing storage %r" % name)
+            try:
+                storage.close()
+            except: # Keep going
+                log("failed to close storage %r" % name,
+                    level=logging.EXCEPTION, exc_info=True)
+
+
+# Signal names
+
+signames = None
+
+def signame(sig):
+    """Return a symbolic name for a signal.
+
+    Return "signal NNN" if there is no corresponding SIG name in the
+    signal module.
+    """
+
+    if signames is None:
+        init_signames()
+    return signames.get(sig) or "signal %d" % sig
+
+def init_signames():
+    global signames
+    signames = {}
+    for name, sig in signal.__dict__.items():
+        k_startswith = getattr(name, "startswith", None)
+        if k_startswith is None:
+            continue
+        if k_startswith("SIG") and not k_startswith("SIG_"):
+            signames[sig] = name
+
+
+# Main program
+
+def main(args=None):
+    options = ZEOOptions()
+    options.realize(args)
+    s = ZEOServer(options)
+    s.main()
+
+if __name__ == "__main__":
+    main()

lib/python/ZEO/schema.xml

+<schema>
+
+  <!-- note that zeoctl.xml is a closely related schema which should
+       match this schema, but should require the "runner" section -->
+
+  <description>
+    This schema describes the configuration of the ZEO storage server
+    process.
+  </description>
+
+  <!-- Use the storage types defined by ZODB. -->
+  <import package="ZODB"/>
+
+  <!-- Use the ZEO server information structure. -->
+  <import package="ZEO"/>
+
+  <import package="ZConfig.components.logger"/>
+
+  <!-- runner control -->
+  <import package="zdaemon"/>
+
+
+  <section type="zeo" name="*" required="yes" attribute="zeo" />
+
+  <section type="runner" name="*" required="no" attribute="runner" />
+
+  <multisection name="+" type="ZODB.storage"
+                attribute="storages"
+                required="yes">
+    <description>
+      One or more storages that are provided by the ZEO server.  The
+      section names are used as the storage names, and must be unique
+      within each ZEO storage server.  Traditionally, these names
+      represent small integers starting at '1'.
+    </description>
+  </multisection>
+
+  <section name="*" type="eventlog" attribute="eventlog" required="no" />
+
+</schema>

lib/python/ZEO/simul.py

+#! /usr/bin/env python
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Cache simulation.
+
+Usage: simul.py [-bflyz] [-X] [-s size] tracefile
+
+Use one of -b, -f, -l, -y or -z select the cache simulator:
+-b: buddy system allocator
+-f: simple free list allocator
+-l: idealized LRU (no allocator)
+-y: variation on the existing ZEO cache that copies to current file
+-z: existing ZEO cache (default)
+
+Options:
+-s size: cache size in MB (default 20 MB)
+-X: enable heuristic checking for misaligned records: oids > 2**32
+    will be rejected; this requires the tracefile to be seekable
+
+Note: the buddy system allocator rounds the cache size up to a power of 2
+"""
+
+import sys
+import time
+import getopt
+import struct
+
+def usage(msg):
+    print >>sys.stderr, msg
+    print >>sys.stderr, __doc__
+
+def main():
+    # Parse options
+    MB = 1000*1000
+    cachelimit = 20*MB
+    simclass = ZEOCacheSimulation
+    heuristic = 0
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "bflyzs:X")
+    except getopt.error, msg:
+        usage(msg)
+        return 2
+    for o, a in opts:
+        if o == '-b':
+            simclass = BuddyCacheSimulation
+        if o == '-f':
+            simclass = SimpleCacheSimulation
+        if o == '-l':
+            simclass = LRUCacheSimulation
+        if o == '-y':
+            simclass = AltZEOCacheSimulation
+        if o == '-z':
+            simclass = ZEOCacheSimulation
+        if o == '-s':
+            cachelimit = int(float(a)*MB)
+        if o == '-X':
+            heuristic = 1
+    if len(args) != 1:
+        usage("exactly one file argument required")
+        return 2
+    filename = args[0]
+
+    # Open file
+    if filename.endswith(".gz"):
+        # Open gzipped file
+        try:
+            import gzip
+        except ImportError:
+            print >>sys.stderr,  "can't read gzipped files (no module gzip)"
+            return 1
+        try:
+            f = gzip.open(filename, "rb")
+        except IOError, msg:
+            print >>sys.stderr,  "can't open %s: %s" % (filename, msg)
+            return 1
+    elif filename == "-":
+        # Read from stdin
+        f = sys.stdin
+    else:
+        # Open regular file
+        try:
+            f = open(filename, "rb")
+        except IOError, msg:
+            print >>sys.stderr,  "can't open %s: %s" % (filename, msg)
+            return 1
+
+    # Create simulation object
+    sim = simclass(cachelimit)
+
+    # Print output header
+    sim.printheader()
+
+    # Read trace file, simulating cache behavior
+    offset = 0
+    records = 0
+    f_read = f.read
+    struct_unpack = struct.unpack
+    while 1:
+        # Read a record and decode it
+        r = f_read(10)
+        if len(r) < 10:
+            break
+        offset += 10
+        ts, code, lenoid = struct_unpack(">iiH", r)
+        if ts == 0:
+            # Must be a misaligned record caused by a crash
+            ##print "Skipping 8 bytes at offset", offset-8
+            continue
+        r = f_read(8 + lenoid)
+        if len(r) < 8 + lenoid:
+            break
+        offset += 8 + lenoid
+        records += 1
+        serial, oid = struct_unpack(">8s%ds" % lenoid, r)
+        # Decode the code
+        dlen, version, code, current = (code & 0x7fffff00,
+                                        code & 0x80,
+                                        code & 0x7e,
+                                        code & 0x01)
+        # And pass it to the simulation
+        sim.event(ts, dlen, version, code, current, oid, serial)
+
+    # Finish simulation
+    sim.finish()
+
+    # Exit code from main()
+    return 0
+
+class Simulation:
+
+    """Base class for simulations.
+
+    The driver program calls: event(), printheader(), finish().
+
+    The standard event() method calls these additional methods:
+    write(), load(), inval(), report(), restart(); the standard
+    finish() method also calls report().
+
+    """
+
+    def __init__(self, cachelimit):
+        self.cachelimit = cachelimit
+        # Initialize global statistics
+        self.epoch = None
+        self.total_loads = 0
+        self.total_hits = 0 # Subclass must increment
+        self.total_invals = 0
+        self.total_writes = 0
+        # Reset per-run statistics and set up simulation data
+        self.restart()
+
+    def restart(self):
+        # Reset per-run statistics
+        self.loads = 0
+        self.hits = 0 # Subclass must increment
+        self.invals = 0
+        self.writes = 0
+        self.ts0 = None
+
+    def event(self, ts, dlen, _version, code, _current, oid, _serial):
+        # Record first and last timestamp seen
+        if self.ts0 is None:
+            self.ts0 = ts
+            if self.epoch is None:
+                self.epoch = ts
+        self.ts1 = ts
+
+        # Simulate cache behavior.  Use load hits, updates and stores
+        # only (each load miss is followed immediately by a store
+        # unless the object in fact did not exist).  Updates always write.
+        if dlen and code & 0x70 in (0x20, 0x30, 0x50):
+            if code == 0x3A:
+                # Update
+                self.writes += 1
+                self.total_writes += 1
+                self.write(oid, dlen)
+            else:
+                # Load hit or store -- these are really the load requests
+                self.loads += 1
+                self.total_loads += 1
+                self.load(oid, dlen)
+        elif code & 0x70 == 0x10:
+            # Invalidate
+            self.inval(oid)
+        elif code == 0x00:
+            # Restart
+            self.report()
+            self.restart()
+
+    def write(self, oid, size):
+        pass
+
+    def load(self, oid, size):
+        pass
+
+    def inval(self, oid):
+        pass
+
+    format = "%12s %9s %8s %8s %6s %6s %6s %6s"
+
+    # Subclass should override extraname to name known instance variables;
+    # if extraname is 'foo', both self.foo and self.total_foo must exist:
+    extraname = "*** please override ***"
+
+    def printheader(self):
+        print "%s, cache size %s bytes" % (self.__class__.__name__,
+                                           addcommas(self.cachelimit))
+        print self.format % (
+            "START TIME", "DURATION", "LOADS", "HITS",
+            "INVALS", "WRITES", self.extraname.upper(), "HITRATE")
+
+    nreports = 0
+
+    def report(self):
+        if self.loads:
+            self.nreports += 1
+            print self.format % (
+                time.ctime(self.ts0)[4:-8],
+                duration(self.ts1 - self.ts0),
+                self.loads, self.hits, self.invals, self.writes,
+                getattr(self, self.extraname),
+                hitrate(self.loads, self.hits))
+
+    def finish(self):
+        self.report()
+        if self.nreports > 1:
+            print (self.format + " OVERALL") % (
+                time.ctime(self.epoch)[4:-8],
+                duration(self.ts1 - self.epoch),
+                self.total_loads,
+                self.total_hits,
+                self.total_invals,
+                self.total_writes,
+                getattr(self, "total_" + self.extraname),
+                hitrate(self.total_loads, self.total_hits))
+
+class ZEOCacheSimulation(Simulation):
+
+    """Simulate the current (ZEO 1.0 and 2.0) ZEO cache behavior.
+
+    This assumes the cache is not persistent (we don't know how to
+    simulate cache validation.)
+
+    """
+
+    extraname = "flips"
+
+    def __init__(self, cachelimit):
+        # Initialize base class
+        Simulation.__init__(self, cachelimit)
+        # Initialize additional global statistics
+        self.total_flips = 0
+
+    def restart(self):
+        # Reset base class
+        Simulation.restart(self)
+        # Reset additional per-run statistics
+        self.flips = 0
+        # Set up simulation
+        self.filesize = [4, 4] # account for magic number
+        self.fileoids = [{}, {}]
+        self.current = 0 # index into filesize, fileoids
+
+    def load(self, oid, size):
+        if (self.fileoids[self.current].get(oid) or
+            self.fileoids[1 - self.current].get(oid)):
+            self.hits += 1
+            self.total_hits += 1
+        else:
+            self.write(oid, size)
+
+    def write(self, oid, size):
+        # Fudge because size is rounded up to multiples of 256.  (31
+        # is header overhead per cache record; 127 is to compensate
+        # for rounding up to multiples of 256.)
+        size = size + 31 - 127
+        if self.filesize[self.current] + size > self.cachelimit / 2:
+            # Cache flip
+            self.flips += 1
+            self.total_flips += 1
+            self.current = 1 - self.current
+            self.filesize[self.current] = 4
+            self.fileoids[self.current] = {}
+        self.filesize[self.current] += size
+        self.fileoids[self.current][oid] = 1
+
+    def inval(self, oid):
+        if self.fileoids[self.current].get(oid):
+            self.invals += 1
+            self.total_invals += 1
+            del self.fileoids[self.current][oid]
+        elif self.fileoids[1 - self.current].get(oid):
+            self.invals += 1
+            self.total_invals += 1
+            del self.fileoids[1 - self.current][oid]
+
+class AltZEOCacheSimulation(ZEOCacheSimulation):
+
+    """A variation of the ZEO cache that copies to the current file.
+
+    When a hit is found in the non-current cache file, it is copied to
+    the current cache file.  Exception: when the copy would cause a
+    cache flip, we don't copy (this is part laziness, part concern
+    over causing extraneous flips).
+    """
+
+    def load(self, oid, size):
+        if self.fileoids[self.current].get(oid):
+            self.hits += 1
+            self.total_hits += 1
+        elif self.fileoids[1 - self.current].get(oid):
+            self.hits += 1
+            self.total_hits += 1
+            # Simulate a write, unless it would cause a flip
+            size = size + 31 - 127
+            if self.filesize[self.current] + size <= self.cachelimit / 2:
+                self.filesize[self.current] += size
+                self.fileoids[self.current][oid] = 1
+                del self.fileoids[1 - self.current][oid]
+        else:
+            self.write(oid, size)
+
+class LRUCacheSimulation(Simulation):
+
+    extraname = "evicts"
+
+    def __init__(self, cachelimit):
+        # Initialize base class
+        Simulation.__init__(self, cachelimit)
+        # Initialize additional global statistics
+        self.total_evicts = 0
+
+    def restart(self):
+        # Reset base class
+        Simulation.restart(self)
+        # Reset additional per-run statistics
+        self.evicts = 0
+        # Set up simulation
+        self.cache = {}
+        self.size = 0
+        self.head = Node(None, None)
+        self.head.linkbefore(self.head)
+
+    def load(self, oid, size):
+        node = self.cache.get(oid)
+        if node is not None:
+            self.hits += 1
+            self.total_hits += 1
+            node.linkbefore(self.head)
+        else:
+            self.write(oid, size)
+
+    def write(self, oid, size):
+        node = self.cache.get(oid)
+        if node is not None:
+            node.unlink()
+            assert self.head.next is not None
+            self.size -= node.size
+        node = Node(oid, size)
+        self.cache[oid] = node
+        node.linkbefore(self.head)
+        self.size += size
+        # Evict LRU nodes
+        while self.size > self.cachelimit:
+            self.evicts += 1
+            self.total_evicts += 1
+            node = self.head.next
+            assert node is not self.head
+            node.unlink()
+            assert self.head.next is not None
+            del self.cache[node.oid]
+            self.size -= node.size
+
+    def inval(self, oid):
+        node = self.cache.get(oid)
+        if node is not None:
+            assert node.oid == oid
+            self.invals += 1
+            self.total_invals += 1
+            node.unlink()
+            assert self.head.next is not None
+            del self.cache[oid]
+            self.size -= node.size
+            assert self.size >= 0
+
+class Node:
+
+    """Node in a doubly-linked list, storing oid and size as payload.
+
+    A node can be linked or unlinked; in the latter case, next and
+    prev are None.  Initially a node is unlinked.
+
+    """
+    # Make it a new-style class in Python 2.2 and up; no effect in 2.1
+    __metaclass__ = type
+    __slots__ = ['prev', 'next', 'oid', 'size']
+
+    def __init__(self, oid, size):
+        self.oid = oid
+        self.size = size
+        self.prev = self.next = None
+
+    def unlink(self):
+        prev = self.prev
+        next = self.next
+        if prev is not None:
+            assert next is not None
+            assert prev.next is self
+            assert next.prev is self
+            prev.next = next
+            next.prev = prev
+            self.prev = self.next = None
+        else:
+            assert next is None
+
+    def linkbefore(self, next):
+        self.unlink()
+        prev = next.prev
+        if prev is None:
+            assert next.next is None
+            prev = next
+        self.prev = prev
+        self.next = next
+        prev.next = next.prev = self
+
+class BuddyCacheSimulation(LRUCacheSimulation):
+
+    def __init__(self, cachelimit):
+        LRUCacheSimulation.__init__(self, roundup(cachelimit))
+
+    def restart(self):
+        LRUCacheSimulation.restart(self)
+        self.allocator = self.allocatorFactory(self.cachelimit)
+
+    def allocatorFactory(self, size):
+        return BuddyAllocator(size)
+
+    # LRUCacheSimulation.load() is just fine
+
+    def write(self, oid, size):
+        node = self.cache.get(oid)
+        if node is not None:
+            node.unlink()
+            assert self.head.next is not None
+            self.size -= node.size
+            self.allocator.free(node)
+        while 1:
+            node = self.allocator.alloc(size)
+            if node is not None:
+                break
+            # Failure to allocate.  Evict something and try again.
+            node = self.head.next
+            assert node is not self.head
+            self.evicts += 1
+            self.total_evicts += 1
+            node.unlink()
+            assert self.head.next is not None
+            del self.cache[node.oid]
+            self.size -= node.size
+            self.allocator.free(node)
+        node.oid = oid
+        self.cache[oid] = node
+        node.linkbefore(self.head)
+        self.size += node.size
+
+    def inval(self, oid):
+        node = self.cache.get(oid)
+        if node is not None:
+            assert node.oid == oid
+            self.invals += 1
+            self.total_invals += 1
+            node.unlink()
+            assert self.head.next is not None
+            del self.cache[oid]
+            self.size -= node.size
+            assert self.size >= 0
+            self.allocator.free(node)
+
+class SimpleCacheSimulation(BuddyCacheSimulation):
+
+    def allocatorFactory(self, size):
+        return SimpleAllocator(size)
+
+    def finish(self):
+        BuddyCacheSimulation.finish(self)
+        self.allocator.report()
+
+MINSIZE = 256
+
+class BuddyAllocator:
+
+    def __init__(self, cachelimit):
+        cachelimit = roundup(cachelimit)
+        self.cachelimit = cachelimit
+        self.avail = {} # Map rounded-up sizes to free list node heads
+        self.nodes = {} # Map address to node
+        k = MINSIZE
+        while k <= cachelimit:
+            self.avail[k] = n = Node(None, None) # Not BlockNode; has no addr
+            n.linkbefore(n)
+            k += k
+        node = BlockNode(None, cachelimit, 0)
+        self.nodes[0] = node
+        node.linkbefore(self.avail[cachelimit])
+
+    def alloc(self, size):
+        size = roundup(size)
+        k = size
+        while k <= self.cachelimit:
+            head = self.avail[k]
+            node = head.next
+            if node is not head:
+                break
+            k += k
+        else:
+            return None # Store is full, or block is too large
+        node.unlink()
+        size2 = node.size
+        while size2 > size:
+            size2 = size2 / 2
+            assert size2 >= size
+            node.size = size2
+            buddy = BlockNode(None, size2, node.addr + size2)
+            self.nodes[buddy.addr] = buddy
+            buddy.linkbefore(self.avail[size2])
+        node.oid = 1 # Flag as in-use
+        return node
+
+    def free(self, node):
+        assert node is self.nodes[node.addr]
+        assert node.prev is node.next is None
+        node.oid = None # Flag as free
+        while node.size < self.cachelimit:
+            buddy_addr = node.addr ^ node.size
+            buddy = self.nodes[buddy_addr]
+            assert buddy.addr == buddy_addr
+            if buddy.oid is not None or buddy.size != node.size:
+                break
+            # Merge node with buddy
+            buddy.unlink()
+            if buddy.addr < node.addr: # buddy prevails
+                del self.nodes[node.addr]
+                node = buddy
+            else: # node prevails
+                del self.nodes[buddy.addr]
+            node.size *= 2
+        assert node is self.nodes[node.addr]
+        node.linkbefore(self.avail[node.size])
+
+    def dump(self, msg=""):
+        if msg:
+            print msg,
+        size = MINSIZE
+        blocks = bytes = 0
+        while size <= self.cachelimit:
+            head = self.avail[size]
+            node = head.next
+            count = 0
+            while node is not head:
+                count += 1
+                node = node.next
+            if count:
+                print "%d:%d" % (size, count),
+            blocks += count
+            bytes += count*size
+            size += size
+        print "-- %d, %d" % (bytes, blocks)
+
+def roundup(size):
+    k = MINSIZE
+    while k < size:
+        k += k
+    return k
+
+class SimpleAllocator:
+
+    def __init__(self, arenasize):
+        self.arenasize = arenasize
+        self.avail = BlockNode(None, 0, 0) # Weird: empty block as list head
+        self.rover = self.avail
+        node = BlockNode(None, arenasize, 0)
+        node.linkbefore(self.avail)
+        self.taglo = {0: node}
+        self.taghi = {arenasize: node}
+        # Allocator statistics
+        self.nallocs = 0
+        self.nfrees = 0
+        self.allocloops = 0
+        self.freebytes = arenasize
+        self.freeblocks = 1
+        self.allocbytes = 0
+        self.allocblocks = 0
+
+    def report(self):
+        print ("NA=%d AL=%d NF=%d ABy=%d ABl=%d FBy=%d FBl=%d" %
+               (self.nallocs, self.allocloops,
+                self.nfrees,
+                self.allocbytes, self.allocblocks,
+                self.freebytes, self.freeblocks))
+
+    def alloc(self, size):
+        self.nallocs += 1
+        # First fit algorithm
+        rover = stop = self.rover
+        while 1:
+            self.allocloops += 1
+            if rover.size >= size:
+                break
+            rover = rover.next
+            if rover is stop:
+                return None # We went round the list without finding space
+        if rover.size == size:
+            self.rover = rover.next
+            rover.unlink()
+            del self.taglo[rover.addr]
+            del self.taghi[rover.addr + size]
+            self.freeblocks -= 1
+            self.allocblocks += 1
+            self.freebytes -= size
+            self.allocbytes += size
+            return rover
+        # Take space from the beginning of the roving pointer
+        assert rover.size > size
+        node = BlockNode(None, size, rover.addr)
+        del self.taglo[rover.addr]
+        rover.size -= size
+        rover.addr += size
+        self.taglo[rover.addr] = rover
+        #self.freeblocks += 0 # No change here
+        self.allocblocks += 1
+        self.freebytes -= size
+        self.allocbytes += size
+        return node
+
+    def free(self, node):
+        self.nfrees += 1
+        self.freeblocks += 1
+        self.allocblocks -= 1
+        self.freebytes += node.size
+        self.allocbytes -= node.size
+        node.linkbefore(self.avail)
+        self.taglo[node.addr] = node
+        self.taghi[node.addr + node.size] = node
+        x = self.taghi.get(node.addr)
+        if x is not None:
+            # Merge x into node
+            x.unlink()
+            self.freeblocks -= 1
+            del self.taglo[x.addr]
+            del self.taghi[x.addr + x.size]
+            del self.taglo[node.addr]
+            node.addr = x.addr
+            node.size += x.size
+            self.taglo[node.addr] = node
+        x = self.taglo.get(node.addr + node.size)
+        if x is not None:
+            # Merge x into node
+            x.unlink()
+            self.freeblocks -= 1
+            del self.taglo[x.addr]
+            del self.taghi[x.addr + x.size]
+            del self.taghi[node.addr + node.size]
+            node.size += x.size
+            self.taghi[node.addr + node.size] = node
+        # It's possible that either one of the merges above invalidated
+        # the rover.
+        # It's simplest to simply reset the rover to the newly freed block.
+        self.rover = node
+
+    def dump(self, msg=""):
+        if msg:
+            print msg,
+        count = 0
+        bytes = 0
+        node = self.avail.next
+        while node is not self.avail:
+            bytes += node.size
+            count += 1
+            node = node.next
+        print count, "free blocks,", bytes, "free bytes"
+        self.report()
+
+class BlockNode(Node):
+
+    __slots__ = ['addr']
+
+    def __init__(self, oid, size, addr):
+        Node.__init__(self, oid, size)
+        self.addr = addr
+
+def testallocator(factory=BuddyAllocator):
+    # Run one of Knuth's experiments as a test
+    import random
+    import heapq # This only runs with Python 2.3, folks :-)
+    reportfreq = 100
+    cachelimit = 2**17
+    cache = factory(cachelimit)
+    queue = []
+    T = 0
+    blocks = 0
+    while T < 5000:
+        while queue and queue[0][0] <= T:
+            time, node = heapq.heappop(queue)
+            assert time == T
+            ##print "free addr=%d, size=%d" % (node.addr, node.size)
+            cache.free(node)
+            blocks -= 1
+        size = random.randint(100, 2000)
+        lifetime = random.randint(1, 100)
+        node = cache.alloc(size)
+        if node is None:
+            print "out of mem"
+            cache.dump("T=%4d: %d blocks;" % (T, blocks))
+            break
+        else:
+            ##print "alloc addr=%d, size=%d" % (node.addr, node.size)
+            blocks += 1
+            heapq.heappush(queue, (T + lifetime, node))
+        T = T+1
+        if T % reportfreq == 0:
+            cache.dump("T=%4d: %d blocks;" % (T, blocks))
+
+def hitrate(loads, hits):
+    return "%5.1f%%" % (100.0 * hits / max(1, loads))
+
+def duration(secs):
+
+    mm, ss = divmod(secs, 60)
+    hh, mm = divmod(mm, 60)
+    if hh:
+        return "%d:%02d:%02d" % (hh, mm, ss)
+    if mm:
+        return "%d:%02d" % (mm, ss)
+    return "%d" % ss
+
+def addcommas(n):
+    sign, s = '', str(n)
+    if s[0] == '-':
+        sign, s = '-', s[1:]
+    i = len(s) - 3
+    while i > 0:
+        s = s[:i] + ',' + s[i:]
+        i -= 3
+    return sign + s
+
+if __name__ == "__main__":
+    sys.exit(main())

lib/python/ZEO/stats.py

+#! /usr/bin/env python
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Trace file statistics analyzer.
+
+Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile
+-h: print histogram of object load frequencies
+-i: summarizing interval in minutes (default 15; max 60)
+-q: quiet; don't print summaries
+-s: print histogram of object sizes
+-S: don't print statistics
+-v: verbose; print each record
+-X: enable heuristic checking for misaligned records: oids > 2**32
+    will be rejected; this requires the tracefile to be seekable
+"""
+
+"""File format:
+
+Each record is 24 bytes, with the following layout.  Numbers are
+big-endian integers.
+
+Offset  Size  Contents
+
+0       4     timestamp (seconds since 1/1/1970)
+4       3     data size, in 256-byte increments, rounded up
+7       1     code (see below)
+8       2     object id length
+10      8     serial number
+18  variable  object id
+
+The code at offset 7 packs three fields:
+
+Mask    bits  Contents
+
+0x80    1     set if there was a non-empty version string
+0x7e    6     function and outcome code
+0x01    1     current cache file (0 or 1)
+
+The function and outcome codes are documented in detail at the end of
+this file in the 'explain' dictionary.  Note that the keys there (and
+also the arguments to _trace() in ClientStorage.py) are 'code & 0x7e',
+i.e. the low bit is always zero.
+"""
+
+import sys
+import time
+import getopt
+import struct
+from types import StringType
+
+def usage(msg):
+    print >>sys.stderr, msg
+    print >>sys.stderr, __doc__
+
+def main():
+    # Parse options
+    verbose = 0
+    quiet = 0
+    dostats = 1
+    print_size_histogram = 0
+    print_histogram = 0
+    interval = 900 # Every 15 minutes
+    heuristic = 0
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hi:qsSvX")
+    except getopt.error, msg:
+        usage(msg)
+        return 2
+    for o, a in opts:
+        if o == '-h':
+            print_histogram = 1
+        if o == "-i":
+            interval = int(60 * float(a))
+            if interval <= 0:
+                interval = 60
+            elif interval > 3600:
+                interval = 3600
+        if o == "-q":
+            quiet = 1
+            verbose = 0
+        if o == "-s":
+            print_size_histogram = 1
+        if o == "-S":
+            dostats = 0
+        if o == "-v":
+            verbose = 1
+        if o == '-X':
+            heuristic = 1
+    if len(args) != 1:
+        usage("exactly one file argument required")
+        return 2
+    filename = args[0]
+
+    # Open file
+    if filename.endswith(".gz"):
+        # Open gzipped file
+        try:
+            import gzip
+        except ImportError:
+            print >>sys.stderr,  "can't read gzipped files (no module gzip)"
+            return 1
+        try:
+            f = gzip.open(filename, "rb")
+        except IOError, msg:
+            print >>sys.stderr,  "can't open %s: %s" % (filename, msg)
+            return 1
+    elif filename == '-':
+        # Read from stdin
+        f = sys.stdin
+    else:
+        # Open regular file
+        try:
+            f = open(filename, "rb")
+        except IOError, msg:
+            print >>sys.stderr,  "can't open %s: %s" % (filename, msg)
+            return 1
+
+    # Read file, gathering statistics, and printing each record if verbose
+    rt0 = time.time()
+    # bycode -- map code to count of occurrences
+    bycode = {}
+    # records -- number of records
+    records = 0
+    # version -- number of records with versions
+    versions = 0
+    t0 = te = None
+    # datarecords -- number of records with dlen set
+    datarecords = 0
+    datasize = 0L
+    # oids -- maps oid to number of times it was loaded
+    oids = {}
+    # bysize -- maps data size to number of loads
+    bysize = {}
+    # bysize -- maps data size to number of writes
+    bysizew = {}
+    total_loads = 0
+    byinterval = {}
+    thisinterval = None
+    h0 = he = None
+    offset = 0
+    f_read = f.read
+    struct_unpack = struct.unpack
+    try:
+        while 1:
+            r = f_read(8)
+            if len(r) < 8:
+                break
+            offset += 8
+            ts, code = struct_unpack(">ii", r)
+            if ts == 0:
+                # Must be a misaligned record caused by a crash
+                if not quiet:
+                    print "Skipping 8 bytes at offset", offset-8
+                continue
+            r = f_read(18)
+            if len(r) < 10:
+                break
+            offset += 10
+            records += 1
+            oidlen, start_tid, end_tid = struct_unpack(">H8s8s", r)
+            oid = f_read(oidlen)
+            if len(oid) != oidlen:
+                break
+            offset += oidlen
+            if t0 is None:
+                t0 = ts
+                thisinterval = t0 / interval
+                h0 = he = ts
+            te = ts
+            if ts / interval != thisinterval:
+                if not quiet:
+                    dumpbyinterval(byinterval, h0, he)
+                byinterval = {}
+                thisinterval = ts / interval
+                h0 = ts
+            he = ts
+            dlen, code = code & 0x7fffff00, code & 0xff
+            if dlen:
+                datarecords += 1
+                datasize += dlen
+            version = '-'
+            if code & 0x80:
+                version = 'V'
+                versions += 1
+            code = code & 0x7e
+            bycode[code] = bycode.get(code, 0) + 1
+            byinterval[code] = byinterval.get(code, 0) + 1
+            if dlen:
+                if code & 0x70 == 0x20: # All loads
+                    bysize[dlen] = d = bysize.get(dlen) or {}
+                    d[oid] = d.get(oid, 0) + 1
+                elif code & 0x70 == 0x50: # All stores
+                    bysizew[dlen] = d = bysizew.get(dlen) or {}
+                    d[oid] = d.get(oid, 0) + 1
+            if verbose:
+                print "%s %d %02x %s %016x %016x %1s %s" % (
+                    time.ctime(ts)[4:-5],
+                    current,
+                    code,
+                    oid_repr(oid),
+                    U64(start_tid),
+                    U64(end_tid),
+                    version,
+                    dlen and str(dlen) or "")
+            if code & 0x70 == 0x20:
+                oids[oid] = oids.get(oid, 0) + 1
+                total_loads += 1
+            if code == 0x00:
+                if not quiet:
+                    dumpbyinterval(byinterval, h0, he)
+                byinterval = {}
+                thisinterval = ts / interval
+                h0 = he = ts
+                if not quiet:
+                    print time.ctime(ts)[4:-5],
+                    print '='*20, "Restart", '='*20
+    except KeyboardInterrupt:
+        print "\nInterrupted.  Stats so far:\n"
+
+    f.close()
+    rte = time.time()
+    if not quiet:
+        dumpbyinterval(byinterval, h0, he)
+
+    # Error if nothing was read
+    if not records:
+        print >>sys.stderr, "No records processed"
+        return 1
+
+    # Print statistics
+    if dostats:
+        print
+        print "Read %s records (%s bytes) in %.1f seconds" % (
+            addcommas(records), addcommas(records*24), rte-rt0)
+        print "Versions:   %s records used a version" % addcommas(versions)
+        print "First time: %s" % time.ctime(t0)
+        print "Last time:  %s" % time.ctime(te)
+        print "Duration:   %s seconds" % addcommas(te-t0)
+        print "Data recs:  %s (%.1f%%), average size %.1f KB" % (
+            addcommas(datarecords),
+            100.0 * datarecords / records,
+            datasize / 1024.0 / datarecords)
+        print "Hit rate:   %.1f%% (load hits / loads)" % hitrate(bycode)
+        print
+        codes = bycode.keys()
+        codes.sort()
+        print "%13s %4s %s" % ("Count", "Code", "Function (action)")
+        for code in codes:
+            print "%13s  %02x  %s" % (
+                addcommas(bycode.get(code, 0)),
+                code,
+                explain.get(code) or "*** unknown code ***")
+
+    # Print histogram
+    if print_histogram:
+        print
+        print "Histogram of object load frequency"
+        total = len(oids)
+        print "Unique oids: %s" % addcommas(total)
+        print "Total loads: %s" % addcommas(total_loads)
+        s = addcommas(total)
+        width = max(len(s), len("objects"))
+        fmt = "%5d %" + str(width) + "s %5.1f%% %5.1f%% %5.1f%%"
+        hdr = "%5s %" + str(width) + "s %6s %6s %6s"
+        print hdr % ("loads", "objects", "%obj", "%load", "%cum")
+        cum = 0.0
+        for binsize, count in histogram(oids):
+            obj_percent = 100.0 * count / total
+            load_percent = 100.0 * count * binsize / total_loads
+            cum += load_percent
+            print fmt % (binsize, addcommas(count),
+                         obj_percent, load_percent, cum)
+
+    # Print size histogram
+    if print_size_histogram:
+        print
+        print "Histograms of object sizes"
+        print
+        dumpbysize(bysizew, "written", "writes")
+        dumpbysize(bysize, "loaded", "loads")
+
+def dumpbysize(bysize, how, how2):
+    print
+    print "Unique sizes %s: %s" % (how, addcommas(len(bysize)))
+    print "%10s %6s %6s" % ("size", "objs", how2)
+    sizes = bysize.keys()
+    sizes.sort()
+    for size in sizes:
+        loads = 0
+        for n in bysize[size].itervalues():
+            loads += n
+        print "%10s %6d %6d" % (addcommas(size),
+                                len(bysize.get(size, "")),
+                                loads)
+
+def dumpbyinterval(byinterval, h0, he):
+    loads = 0
+    hits = 0
+    for code in byinterval.keys():
+        if code & 0x70 == 0x20:
+            n = byinterval[code]
+            loads += n
+            if code in (0x22, 0x26):
+                hits += n
+    if not loads:
+        return
+    if loads:
+        hr = 100.0 * hits / loads
+    else:
+        hr = 0.0
+    print "%s-%s %10s loads, %10s hits,%5.1f%% hit rate" % (
+        time.ctime(h0)[4:-8], time.ctime(he)[14:-8],
+        addcommas(loads), addcommas(hits), hr)
+
+def hitrate(bycode):
+    loads = 0
+    hits = 0
+    for code in bycode.keys():
+        if code & 0x70 == 0x20:
+            n = bycode[code]
+            loads += n
+            if code in (0x22, 0x26):
+                hits += n
+    if loads:
+        return 100.0 * hits / loads
+    else:
+        return 0.0
+
+def histogram(d):
+    bins = {}
+    for v in d.itervalues():
+        bins[v] = bins.get(v, 0) + 1
+    L = bins.items()
+    L.sort()
+    return L
+
+def U64(s):
+    h, v = struct.unpack(">II", s)
+    return (long(h) << 32) + v
+
+def oid_repr(oid):
+    if isinstance(oid, StringType) and len(oid) == 8:
+        return '%16x' % U64(oid)
+    else:
+        return repr(oid)
+
+def addcommas(n):
+    sign, s = '', str(n)
+    if s[0] == '-':
+        sign, s = '-', s[1:]
+    i = len(s) - 3
+    while i > 0:
+        s = s[:i] + ',' + s[i:]
+        i -= 3
+    return sign + s
+
+explain = {
+    # The first hex digit shows the operation, the second the outcome.
+    # If the second digit is in "02468" then it is a 'miss'.
+    # If it is in "ACE" then it is a 'hit'.
+
+    0x00: "_setup_trace (initialization)",
+
+    0x10: "invalidate (miss)",
+    0x1A: "invalidate (hit, version)",
+    0x1C: "invalidate (hit, saving non-current)",
+
+    0x20: "load (miss)",
+    0x22: "load (hit)",
+    0x24: "load (non-current, miss)",
+    0x26: "load (non-current, hit)",
+
+    0x50: "store (version)",
+    0x52: "store (current, non-version)",
+    0x54: "store (non-current)",
+
+    }
+
+if __name__ == "__main__":
+    sys.exit(main())

lib/python/ZEO/tests/Cache.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Tests of the ZEO cache"""
+
+from ZODB.tests.MinPO import MinPO
+from ZODB.tests.StorageTestBase import zodb_unpickle
+
+from transaction import Transaction
+
+class TransUndoStorageWithCache:
+
+    def checkUndoInvalidation(self):
+        oid = self._storage.new_oid()
+        revid = self._dostore(oid, data=MinPO(23))
+        revid = self._dostore(oid, revid=revid, data=MinPO(24))
+        revid = self._dostore(oid, revid=revid, data=MinPO(25))
+
+        info = self._storage.undoInfo()
+        if not info:
+            # XXX perhaps we have an old storage implementation that
+            # does do the negative nonsense
+            info = self._storage.undoInfo(0, 20)
+        tid = info[0]['id']
+
+        # We may need to bail at this point if the storage doesn't
+        # support transactional undo
+        if not self._storage.supportsTransactionalUndo():
+            return
+
+        # Now start an undo transaction
+        t = Transaction()
+        t.note('undo1')
+        self._storage.tpc_begin(t)
+
+        tid, oids = self._storage.undo(tid, t)
+
+        # Make sure this doesn't load invalid data into the cache
+        self._storage.load(oid, '')
+
+        self._storage.tpc_vote(t)
+        self._storage.tpc_finish(t)
+
+        assert len(oids) == 1
+        assert oids[0] == oid
+        data, revid = self._storage.load(oid, '')
+        obj = zodb_unpickle(data)
+        assert obj == MinPO(24)
+
+class StorageWithCache:
+
+    def checkAbortVersionInvalidation(self):
+        oid = self._storage.new_oid()
+        revid = self._dostore(oid, data=MinPO(1))
+        revid = self._dostore(oid, revid=revid, data=MinPO(2))
+        revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
+        revid = self._dostore(oid, revid=revid, data=MinPO(4), version="foo")
+        t = Transaction()
+        self._storage.tpc_begin(t)
+        self._storage.abortVersion("foo", t)
+        self._storage.load(oid, "foo")
+        self._storage.tpc_vote(t)
+        self._storage.tpc_finish(t)
+        data, revid = self._storage.load(oid, "foo")
+        obj = zodb_unpickle(data)
+        assert obj == MinPO(2), obj
+
+    def checkCommitEmptyVersionInvalidation(self):
+        oid = self._storage.new_oid()
+        revid = self._dostore(oid, data=MinPO(1))
+        revid = self._dostore(oid, revid=revid, data=MinPO(2))
+        revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
+        t = Transaction()
+        self._storage.tpc_begin(t)
+        self._storage.commitVersion("foo", "", t)
+        self._storage.load(oid, "")
+        self._storage.tpc_vote(t)
+        self._storage.tpc_finish(t)
+        data, revid = self._storage.load(oid, "")
+        obj = zodb_unpickle(data)
+        assert obj == MinPO(3), obj
+
+    def checkCommitVersionInvalidation(self):
+        oid = self._storage.new_oid()
+        revid = self._dostore(oid, data=MinPO(1))
+        revid = self._dostore(oid, revid=revid, data=MinPO(2))
+        revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
+        t = Transaction()
+        self._storage.tpc_begin(t)
+        self._storage.commitVersion("foo", "bar", t)
+        self._storage.load(oid, "")
+        self._storage.tpc_vote(t)
+        self._storage.tpc_finish(t)
+        data, revid = self._storage.load(oid, "bar")
+        obj = zodb_unpickle(data)
+        assert obj == MinPO(3), obj

lib/python/ZEO/tests/CommitLockTests.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Tests of the distributed commit lock."""
+
+import threading
+import time
+
+from persistent.TimeStamp import TimeStamp
+import transaction
+from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
+
+import ZEO.ClientStorage
+from ZEO.Exceptions import ClientDisconnected
+from ZEO.tests.TestThread import TestThread
+
+ZERO = '\0'*8
+
+class DummyDB:
+    def invalidate(self, *args, **kwargs):
+        pass
+
+class WorkerThread(TestThread):
+
+    # run the entire test in a thread so that the blocking call for
+    # tpc_vote() doesn't hang the test suite.
+
+    def __init__(self, testcase, storage, trans, method="tpc_finish"):
+        self.storage = storage
+        self.trans = trans
+        self.method = method
+        self.ready = threading.Event()
+        TestThread.__init__(self, testcase)
+
+    def testrun(self):
+        try:
+            self.storage.tpc_begin(self.trans)
+            oid = self.storage.new_oid()
+            p = zodb_pickle(MinPO("c"))
+            self.storage.store(oid, ZERO, p, '', self.trans)
+            oid = self.storage.new_oid()
+            p = zodb_pickle(MinPO("c"))
+            self.storage.store(oid, ZERO, p, '', self.trans)
+            self.myvote()
+            if self.method == "tpc_finish":
+                self.storage.tpc_finish(self.trans)
+            else:
+                self.storage.tpc_abort(self.trans)
+        except ClientDisconnected:
+            pass
+
+    def myvote(self):
+        # The vote() call is synchronous, which makes it difficult to
+        # coordinate the action of multiple threads that all call
+        # vote().  This method sends the vote call, then sets the
+        # event saying vote was called, then waits for the vote
+        # response.  It digs deep into the implementation of the client.
+
+        # This method is a replacement for:
+        #     self.ready.set()
+        #     self.storage.tpc_vote(self.trans)
+
+        rpc = self.storage._server.rpc
+        msgid = rpc._deferred_call('vote', id(self.trans))
+        self.ready.set()
+        rpc._deferred_wait(msgid)
+        self.storage._check_serials()
+
+class CommitLockTests:
+
+    NUM_CLIENTS = 5
+
+    # The commit lock tests verify that the storage successfully
+    # blocks and restarts transactions when there is contention for a
+    # single storage.  There are a lot of cases to cover.
+
+    # The general flow of these tests is to start a transaction by
+    # getting far enough into 2PC to acquire the commit lock.  Then
+    # begin one or more other connections that also want to commit.
+    # This causes the commit lock code to be exercised.  Once the
+    # other connections are started, the first transaction completes.
+
+    def _cleanup(self):
+        for store, trans in self._storages:
+            store.tpc_abort(trans)
+            store.close()
+        self._storages = []
+
+    def _start_txn(self):
+        txn = transaction.Transaction()
+        self._storage.tpc_begin(txn)
+        oid = self._storage.new_oid()
+        self._storage.store(oid, ZERO, zodb_pickle(MinPO(1)), '', txn)
+        return oid, txn
+
+    def _begin_threads(self):
+        # Start a second transaction on a different connection without
+        # blocking the test thread.  Returns only after each thread has
+        # set it's ready event.
+        self._storages = []
+        self._threads = []
+
+        for i in range(self.NUM_CLIENTS):
+            storage = self._duplicate_client()
+            txn = transaction.Transaction()
+            tid = self._get_timestamp()
+
+            t = WorkerThread(self, storage, txn)
+            self._threads.append(t)
+            t.start()
+            t.ready.wait()
+
+            # Close on the connections abnormally to test server response
+            if i == 0:
+                storage.close()
+            else:
+                self._storages.append((storage, txn))
+
+    def _finish_threads(self):
+        for t in self._threads:
+            t.cleanup()
+
+    def _duplicate_client(self):
+        "Open another ClientStorage to the same server."
+        # XXX argh it's hard to find the actual address
+        # The rpc mgr addr attribute is a list.  Each element in the
+        # list is a socket domain (AF_INET, AF_UNIX, etc.) and an
+        # address.
+        addr = self._storage._addr
+        new = ZEO.ClientStorage.ClientStorage(addr, wait=1)
+        new.registerDB(DummyDB(), None)
+        return new
+
+    def _get_timestamp(self):
+        t = time.time()
+        t = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
+        return `t`
+
+class CommitLockVoteTests(CommitLockTests):
+
+    def checkCommitLockVoteFinish(self):
+        oid, txn = self._start_txn()
+        self._storage.tpc_vote(txn)
+
+        self._begin_threads()
+
+        self._storage.tpc_finish(txn)
+        self._storage.load(oid, '')
+
+        self._finish_threads()
+
+        self._dostore()
+        self._cleanup()
+
+    def checkCommitLockVoteAbort(self):
+        oid, txn = self._start_txn()
+        self._storage.tpc_vote(txn)
+
+        self._begin_threads()
+
+        self._storage.tpc_abort(txn)
+
+        self._finish_threads()
+
+        self._dostore()
+        self._cleanup()
+
+    def checkCommitLockVoteClose(self):
+        oid, txn = self._start_txn()
+        self._storage.tpc_vote(txn)
+
+        self._begin_threads()
+
+        self._storage.close()
+
+        self._finish_threads()
+        self._cleanup()
+
+class CommitLockUndoTests(CommitLockTests):
+
+    def _get_trans_id(self):
+        self._dostore()
+        L = self._storage.undoInfo()
+        return L[0]['id']
+
+    def _begin_undo(self, trans_id, txn):
+        rpc = self._storage._server.rpc
+        return rpc._deferred_call('undo', trans_id, id(txn))
+
+    def _finish_undo(self, msgid):
+        return self._storage._server.rpc._deferred_wait(msgid)
+
+    def checkCommitLockUndoFinish(self):
+        trans_id = self._get_trans_id()
+        oid, txn = self._start_txn()
+        msgid = self._begin_undo(trans_id, txn)
+
+        self._begin_threads()
+
+        self._finish_undo(msgid)
+        self._storage.tpc_vote(txn)
+        self._storage.tpc_finish(txn)
+        self._storage.load(oid, '')
+
+        self._finish_threads()
+
+        self._dostore()
+        self._cleanup()
+
+    def checkCommitLockUndoAbort(self):
+        trans_id = self._get_trans_id()
+        oid, txn = self._start_txn()
+        msgid = self._begin_undo(trans_id, txn)
+
+        self._begin_threads()
+
+        self._finish_undo(msgid)
+        self._storage.tpc_vote(txn)
+        self._storage.tpc_abort(txn)
+
+        self._finish_threads()
+
+        self._dostore()
+        self._cleanup()
+
+    def checkCommitLockUndoClose(self):
+        trans_id = self._get_trans_id()
+        oid, txn = self._start_txn()
+        msgid = self._begin_undo(trans_id, txn)
+
+        self._begin_threads()
+
+        self._finish_undo(msgid)
+        self._storage.tpc_vote(txn)
+        self._storage.close()
+
+        self._finish_threads()
+
+        self._cleanup()

lib/python/ZEO/tests/ConnectionTests.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import os
+import sys
+import time
+import random
+import asyncore
+import tempfile
+import threading
+import logging
+
+import ZEO.ServerStub
+from ZEO.ClientStorage import ClientStorage
+from ZEO.Exceptions import ClientDisconnected
+from ZEO.zrpc.marshal import Marshaller
+from ZEO.tests import forker
+
+from ZODB.DB import DB
+from ZODB.POSException import ReadOnlyError, ConflictError
+from ZODB.tests.StorageTestBase import StorageTestBase
+from ZODB.tests.MinPO import MinPO
+from ZODB.tests.StorageTestBase \
+     import zodb_pickle, zodb_unpickle, handle_all_serials, handle_serials
+
+import transaction
+from transaction import Transaction
+
+logger = logging.getLogger('ZEO.tests.ConnectionTests')
+
+ZERO = '\0'*8
+
+class TestServerStub(ZEO.ServerStub.StorageServer):
+    __super_getInvalidations = ZEO.ServerStub.StorageServer.getInvalidations
+
+    def getInvalidations(self, tid):
+        # squirrel the results away for inspection by test case
+        self._last_invals = self.__super_getInvalidations(tid)
+        return self._last_invals
+
+class TestClientStorage(ClientStorage):
+
+    test_connection = False
+
+    StorageServerStubClass = TestServerStub
+
+    def verify_cache(self, stub):
+        self.end_verify = threading.Event()
+        self.verify_result = ClientStorage.verify_cache(self, stub)
+
+    def endVerify(self):
+        ClientStorage.endVerify(self)
+        self.end_verify.set()
+
+    def testConnection(self, conn):
+        try:
+            return ClientStorage.testConnection(self, conn)
+        finally:
+            self.test_connection = True
+
+class DummyDB:
+    def invalidate(self, *args, **kwargs):
+        pass
+
+class CommonSetupTearDown(StorageTestBase):
+    """Common boilerplate"""
+
+    __super_setUp = StorageTestBase.setUp
+    __super_tearDown = StorageTestBase.tearDown
+    keep = 0
+    invq = None
+    timeout = None
+    monitor = 0
+    db_class = DummyDB
+
+    def setUp(self):
+        """Test setup for connection tests.
+
+        This starts only one server; a test may start more servers by
+        calling self._newAddr() and then self.startServer(index=i)
+        for i in 1, 2, ...
+        """
+        self.__super_setUp()
+        logging.info("setUp() %s", self.id())
+        self.file = tempfile.mktemp()
+        self.addr = []
+        self._pids = []
+        self._servers = []
+        self.conf_paths = []
+        self.caches = []
+        self._newAddr()
+        self.startServer()
+
+    def tearDown(self):
+        """Try to cause the tests to halt"""
+        logging.info("tearDown() %s" % self.id())
+        for p in self.conf_paths:
+            os.remove(p)
+        if getattr(self, '_storage', None) is not None:
+            self._storage.close()
+            if hasattr(self._storage, 'cleanup'):
+                logging.debug("cleanup storage %s" %
+                         self._storage.__name__)
+                self._storage.cleanup()
+        for adminaddr in self._servers:
+            if adminaddr is not None:
+                forker.shutdown_zeo_server(adminaddr)
+        if hasattr(os, 'waitpid'):
+            # Not in Windows Python until 2.3
+            for pid in self._pids:
+                os.waitpid(pid, 0)
+        for c in self.caches:
+            for i in 0, 1:
+                for ext in "", ".trace":
+                    base = "%s-%s.zec%s" % (c, "1", ext)
+                    path = os.path.join(tempfile.tempdir, base)
+                    # On Windows before 2.3, we don't have a way to wait for
+                    # the spawned server(s) to close, and they inherited
+                    # file descriptors for our open files.  So long as those
+                    # processes are alive, we can't delete the files.  Try
+                    # a few times then give up.
+                    need_to_delete = False
+                    if os.path.exists(path):
+                        need_to_delete = True
+                        for dummy in range(5):
+                            try:
+                                os.unlink(path)
+                            except:
+                                time.sleep(0.5)
+                            else:
+                                need_to_delete = False
+                                break
+                    if need_to_delete:
+                        os.unlink(path)  # sometimes this is just gonna fail
+        self.__super_tearDown()
+
+    def _newAddr(self):
+        self.addr.append(self._getAddr())
+
+    def _getAddr(self):
+        # port+1 is also used, so only draw even port numbers
+        return 'localhost', random.randrange(25000, 30000, 2)
+
+    def getConfig(self, path, create, read_only):
+        raise NotImplementedError
+
+    cache_id = 1
+
+    def openClientStorage(self, cache=None, cache_size=200000, wait=1,
+                          read_only=0, read_only_fallback=0,
+                          username=None, password=None, realm=None):
+        if cache is None:
+            cache = str(self.__class__.cache_id)
+            self.__class__.cache_id += 1
+        self.caches.append(cache)
+        storage = TestClientStorage(self.addr,
+                                    client=cache,
+                                    var=tempfile.tempdir,
+                                    cache_size=cache_size,
+                                    wait=wait,
+                                    min_disconnect_poll=0.1,
+                                    read_only=read_only,
+                                    read_only_fallback=read_only_fallback,
+                                    username=username,
+                                    password=password,
+                                    realm=realm)
+        storage.registerDB(DummyDB(), None)
+        return storage
+
+    def getServerConfig(self, addr, ro_svr):
+        zconf = forker.ZEOConfig(addr)
+        if ro_svr:
+            zconf.read_only = 1
+        if self.monitor:
+            zconf.monitor_address = ("", 42000)
+        if self.invq:
+            zconf.invalidation_queue_size = self.invq
+        if self.timeout:
+            zconf.transaction_timeout = self.timeout
+        return zconf
+
+    def startServer(self, create=1, index=0, read_only=0, ro_svr=0, keep=None):
+        addr = self.addr[index]
+        logging.info("startServer(create=%d, index=%d, read_only=%d) @ %s" %
+                     (create, index, read_only, addr))
+        path = "%s.%d" % (self.file, index)
+        sconf = self.getConfig(path, create, read_only)
+        zconf = self.getServerConfig(addr, ro_svr)
+        if keep is None:
+            keep = self.keep
+        zeoport, adminaddr, pid, path = forker.start_zeo_server(
+            sconf, zconf, addr[1], keep)
+        self.conf_paths.append(path)
+        self._pids.append(pid)
+        self._servers.append(adminaddr)
+
+    def shutdownServer(self, index=0):
+        logging.info("shutdownServer(index=%d) @ %s" %
+                     (index, self._servers[index]))
+        adminaddr = self._servers[index]
+        if adminaddr is not None:
+            forker.shutdown_zeo_server(adminaddr)
+            self._servers[index] = None
+
+    def pollUp(self, timeout=30.0, storage=None):
+        if storage is None:
+            storage = self._storage
+        # Poll until we're connected
+        now = time.time()
+        giveup = now + timeout
+        while not storage.is_connected():
+            asyncore.poll(0.1)
+            now = time.time()
+            if now > giveup:
+                self.fail("timed out waiting for storage to connect")
+
+    def pollDown(self, timeout=30.0):
+        # Poll until we're disconnected
+        now = time.time()
+        giveup = now + timeout
+        while self._storage.is_connected():
+            asyncore.poll(0.1)
+            now = time.time()
+            if now > giveup:
+                self.fail("timed out waiting for storage to disconnect")
+
+
+class ConnectionTests(CommonSetupTearDown):
+    """Tests that explicitly manage the server process.
+
+    To test the cache or re-connection, these test cases explicit
+    start and stop a ZEO storage server.
+    """
+
+    def checkMultipleAddresses(self):
+        for i in range(4):
+            self._newAddr()
+        self._storage = self.openClientStorage('test', 100000)
+        oid = self._storage.new_oid()
+        obj = MinPO(12)
+        self._dostore(oid, data=obj)
+        self._storage.close()
+
+    def checkMultipleServers(self):
+        # XXX crude test at first -- just start two servers and do a
+        # commit at each one.
+
+        self._newAddr()
+        self._storage = self.openClientStorage('test', 100000)
+        self._dostore()
+
+        self.shutdownServer(index=0)
+        self.startServer(index=1)
+
+        # If we can still store after shutting down one of the
+        # servers, we must be reconnecting to the other server.
+
+        did_a_store = 0
+        for i in range(10):
+            try:
+                self._dostore()
+                did_a_store = 1
+                break
+            except ClientDisconnected:
+                time.sleep(0.5)
+        self.assert_(did_a_store)
+        self._storage.close()
+
+    def checkReadOnlyClient(self):
+        # Open a read-only client to a read-write server; stores fail
+
+        # Start a read-only client for a read-write server
+        self._storage = self.openClientStorage(read_only=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+        self._storage.close()
+
+    def checkReadOnlyServer(self):
+        # Open a read-only client to a read-only *server*; stores fail
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        # Start a read-only server
+        self.startServer(create=0, index=0, ro_svr=1)
+        # Start a read-only client
+        self._storage = self.openClientStorage(read_only=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+        self._storage.close()
+        # Get rid of the 'test left new threads behind' warning
+        time.sleep(0.1)
+
+    def checkReadOnlyFallbackWritable(self):
+        # Open a fallback client to a read-write server; stores succeed
+
+        # Start a read-only-fallback client for a read-write server
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should succeed here
+        self._dostore()
+        self._storage.close()
+
+    def checkReadOnlyFallbackReadOnlyServer(self):
+        # Open a fallback client to a read-only *server*; stores fail
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        # Start a read-only server
+        self.startServer(create=0, index=0, ro_svr=1)
+        # Start a read-only-fallback client
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        self.assert_(self._storage.isReadOnly())
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+        self._storage.close()
+
+    # XXX Compare checkReconnectXXX() here to checkReconnection()
+    # further down.  Is the code here hopelessly naive, or is
+    # checkReconnection() overwrought?
+
+    def checkReconnectWritable(self):
+        # A read-write client reconnects to a read-write server
+
+        # Start a client
+        self._storage = self.openClientStorage()
+        # Stores should succeed here
+        self._dostore()
+
+        # Shut down the server
+        self.shutdownServer()
+        self._servers = []
+        # Poll until the client disconnects
+        self.pollDown()
+        # Stores should fail now
+        self.assertRaises(ClientDisconnected, self._dostore)
+
+        # Restart the server
+        self.startServer(create=0)
+        # Poll until the client connects
+        self.pollUp()
+        # Stores should succeed here
+        self._dostore()
+        self._storage.close()
+
+    def checkDisconnectionError(self):
+        # Make sure we get a ClientDisconnected when we try to read an
+        # object when we're not connected to a storage server and the
+        # object is not in the cache.
+        self.shutdownServer()
+        self._storage = self.openClientStorage('test', 1000, wait=0)
+        self.assertRaises(ClientDisconnected,
+                          self._storage.load, 'fredwash', '')
+        self._storage.close()
+
+    def checkDisconnectedAbort(self):
+        self._storage = self.openClientStorage()
+        self._dostore()
+        oids = [self._storage.new_oid() for i in range(5)]
+        txn = Transaction()
+        self._storage.tpc_begin(txn)
+        for oid in oids:
+            data = zodb_pickle(MinPO(oid))
+            self._storage.store(oid, None, data, '', txn)
+        self.shutdownServer()
+        self.assertRaises(ClientDisconnected, self._storage.tpc_vote, txn)
+        self._storage.tpc_abort(txn)
+        self.startServer(create=0)
+        self._storage._wait()
+        self._dostore()
+
+        # This test is supposed to cover the following error, although
+        # I don't have much confidence that it does.  The likely
+        # explanation for the error is that the _tbuf contained
+        # objects that weren't in the _seriald, because the client was
+        # interrupted waiting for tpc_vote() to return.  When the next
+        # transaction committed, it tried to do something with the
+        # bogus _tbuf entries.  The exaplanation is wrong/incomplete,
+        # because tpc_begin() should clear the _tbuf.
+
+        # 2003-01-15T15:44:19 ERROR(200) ZODB A storage error occurred
+        # in the last phase of a two-phase commit.  This shouldn't happen.
+
+        # Traceback (innermost last):
+        # Module ZODB.Transaction, line 359, in _finish_one
+        # Module ZODB.Connection, line 691, in tpc_finish
+        # Module ZEO.ClientStorage, line 679, in tpc_finish
+        # Module ZEO.ClientStorage, line 709, in _update_cache
+        # KeyError: ...
+
+    def checkBasicPersistence(self):
+        # Verify cached data persists across client storage instances.
+
+        # To verify that the cache is being used, the test closes the
+        # server and then starts a new client with the server down.
+        # When the server is down, a load() gets the data from its cache.
+
+        self._storage = self.openClientStorage('test', 100000)
+        oid = self._storage.new_oid()
+        obj = MinPO(12)
+        revid1 = self._dostore(oid, data=obj)
+        self._storage.close()
+        self.shutdownServer()
+        self._storage = self.openClientStorage('test', 100000, wait=0)
+        data, revid2 = self._storage.load(oid, '')
+        self.assertEqual(zodb_unpickle(data), MinPO(12))
+        self.assertEqual(revid1, revid2)
+        self._storage.close()
+
+    def checkRollover(self):
+        # Check that the cache works when the files are swapped.
+
+        # In this case, only one object fits in a cache file.  When the
+        # cache files swap, the first object is effectively uncached.
+
+        self._storage = self.openClientStorage('test', 1000)
+        oid1 = self._storage.new_oid()
+        obj1 = MinPO("1" * 500)
+        self._dostore(oid1, data=obj1)
+        oid2 = self._storage.new_oid()
+        obj2 = MinPO("2" * 500)
+        self._dostore(oid2, data=obj2)
+        self._storage.close()
+        self.shutdownServer()
+        self._storage = self.openClientStorage('test', 1000, wait=0)
+        self._storage.load(oid1, '')
+        self._storage.load(oid2, '')
+        self._storage.close()
+
+    def checkReconnection(self):
+        # Check that the client reconnects when a server restarts.
+
+        # XXX Seem to get occasional errors that look like this:
+        # File ZEO/zrpc2.py, line 217, in handle_request
+        # File ZEO/StorageServer.py, line 325, in storea
+        # File ZEO/StorageServer.py, line 209, in _check_tid
+        # StorageTransactionError: (None, <tid>)
+        # could system reconnect and continue old transaction?
+
+        self._storage = self.openClientStorage()
+        oid = self._storage.new_oid()
+        obj = MinPO(12)
+        self._dostore(oid, data=obj)
+        logging.info("checkReconnection(): About to shutdown server")
+        self.shutdownServer()
+        logging.info("checkReconnection(): About to restart server")
+        self.startServer(create=0)
+        oid = self._storage.new_oid()
+        obj = MinPO(12)
+        while 1:
+            try:
+                self._dostore(oid, data=obj)
+                break
+            except ClientDisconnected:
+                # Maybe the exception mess is better now
+                logging.info("checkReconnection(): Error after"
+                             " server restart; retrying.", exc_info=True)
+                transaction.abort()
+            # Give the other thread a chance to run.
+            time.sleep(0.1)
+        logging.info("checkReconnection(): finished")
+        self._storage.close()
+
+    def checkBadMessage1(self):
+        # not even close to a real message
+        self._bad_message("salty")
+
+    def checkBadMessage2(self):
+        # just like a real message, but with an unpicklable argument
+        global Hack
+        class Hack:
+            pass
+
+        msg = Marshaller().encode(1, 0, "foo", (Hack(),))
+        self._bad_message(msg)
+        del Hack
+
+    def _bad_message(self, msg):
+        # Establish a connection, then send the server an ill-formatted
+        # request.  Verify that the connection is closed and that it is
+        # possible to establish a new connection.
+
+        self._storage = self.openClientStorage()
+        self._dostore()
+
+        # break into the internals to send a bogus message
+        zrpc_conn = self._storage._server.rpc
+        zrpc_conn.message_output(msg)
+
+        try:
+            self._dostore()
+        except ClientDisconnected:
+            pass
+        else:
+            self._storage.close()
+            self.fail("Server did not disconnect after bogus message")
+        self._storage.close()
+
+        self._storage = self.openClientStorage()
+        self._dostore()
+        self._storage.close()
+
+    # Test case for multiple storages participating in a single
+    # transaction.  This is not really a connection test, but it needs
+    # about the same infrastructure (several storage servers).
+
+    # XXX WARNING: with the current ZEO code, this occasionally fails.
+    # That's the point of this test. :-)
+
+    def NOcheckMultiStorageTransaction(self):
+        # Configuration parameters (larger values mean more likely deadlocks)
+        N = 2
+        # These don't *have* to be all the same, but it's convenient this way
+        self.nservers = N
+        self.nthreads = N
+        self.ntrans = N
+        self.nobj = N
+
+        # Start extra servers
+        for i in range(1, self.nservers):
+            self._newAddr()
+            self.startServer(index=i)
+
+        # Spawn threads that each do some transactions on all storages
+        threads = []
+        try:
+            for i in range(self.nthreads):
+                t = MSTThread(self, "T%d" % i)
+                threads.append(t)
+                t.start()
+            # Wait for all threads to finish
+            for t in threads:
+                t.join(60)
+                self.failIf(t.isAlive(), "%s didn't die" % t.getName())
+        finally:
+            for t in threads:
+                t.closeclients()
+
+    def checkCrossDBInvalidations(self):
+        db1 = DB(self.openClientStorage())
+        c1 = db1.open()
+        r1 = c1.root()
+
+        r1["a"] = MinPO("a")
+        transaction.commit()
+
+        db2 = DB(self.openClientStorage())
+        r2 = db2.open().root()
+
+        self.assertEqual(r2["a"].value, "a")
+
+        r2["b"] = MinPO("b")
+        transaction.commit()
+
+        # make sure the invalidation is received in the other client
+        for i in range(10):
+            c1._storage.sync()
+            if c1._invalidated.has_key(r1._p_oid):
+                break
+            time.sleep(0.1)
+        self.assert_(c1._invalidated.has_key(r1._p_oid))
+
+        # force the invalidations to be applied...
+        c1.sync()
+        r1.keys() # unghostify
+        self.assertEqual(r1._p_serial, r2._p_serial)
+
+        db2.close()
+        db1.close()
+
+class InvqTests(CommonSetupTearDown):
+    invq = 3
+
+    def checkQuickVerificationWith2Clients(self):
+        perstorage = self.openClientStorage(cache="test")
+        self.assertEqual(perstorage.verify_result, "full verification")
+
+        self._storage = self.openClientStorage()
+        oid = self._storage.new_oid()
+        oid2 = self._storage.new_oid()
+        # When we create a new storage, it should always do a full
+        # verification
+        self.assertEqual(self._storage.verify_result, "full verification")
+        # do two storages of the object to make sure an invalidation
+        # message is generated
+        revid = self._dostore(oid)
+        revid = self._dostore(oid, revid)
+        # Create a second object and revision to guarantee it doesn't
+        # show up in the list of invalidations sent when perstore restarts.
+        revid2 = self._dostore(oid2)
+        revid2 = self._dostore(oid2, revid2)
+
+        # sync() is needed to prevent invalidation for oid from arriving
+        # in the middle of the load() call.
+        perstorage.sync()
+        perstorage.load(oid, '')
+        perstorage.close()
+
+        revid = self._dostore(oid, revid)
+        perstorage = self.openClientStorage(cache="test")
+        self.assertEqual(perstorage.verify_result, "quick verification")
+        self.assertEqual(perstorage._server._last_invals,
+                         (revid, [(oid, '')]))
+
+        self.assertEqual(perstorage.load(oid, ''),
+                         self._storage.load(oid, ''))
+        perstorage.close()
+
+    def checkVerificationWith2ClientsInvqOverflow(self):
+        perstorage = self.openClientStorage(cache="test")
+        self.assertEqual(perstorage.verify_result, "full verification")
+
+        self._storage = self.openClientStorage()
+        oid = self._storage.new_oid()
+        # When we create a new storage, it should always do a full
+        # verification
+        self.assertEqual(self._storage.verify_result, "full verification")
+        # do two storages of the object to make sure an invalidation
+        # message is generated
+        revid = self._dostore(oid)
+        revid = self._dostore(oid, revid)
+
+        perstorage.load(oid, '')
+        perstorage.close()
+
+        # the test code sets invq bound to 2
+        for i in range(5):
+            revid = self._dostore(oid, revid)
+
+        perstorage = self.openClientStorage(cache="test")
+        self.assertEqual(perstorage.verify_result, "full verification")
+        t = time.time() + 30
+        while not perstorage.end_verify.isSet():
+            perstorage.sync()
+            if time.time() > t:
+                self.fail("timed out waiting for endVerify")
+
+        self.assertEqual(self._storage.load(oid, '')[1], revid)
+        self.assertEqual(perstorage.load(oid, ''),
+                         self._storage.load(oid, ''))
+
+        perstorage.close()
+
+class ReconnectionTests(CommonSetupTearDown):
+    # The setUp() starts a server automatically.  In order for its
+    # state to persist, we set the class variable keep to 1.  In
+    # order for its state to be cleaned up, the last startServer()
+    # call in the test must pass keep=0.
+    keep = 1
+    invq = 2
+
+    def checkReadOnlyStorage(self):
+        # Open a read-only client to a read-only *storage*; stores fail
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        # Start a read-only server
+        self.startServer(create=0, index=0, read_only=1, keep=0)
+        # Start a read-only client
+        self._storage = self.openClientStorage(read_only=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+    def checkReadOnlyFallbackReadOnlyStorage(self):
+        # Open a fallback client to a read-only *storage*; stores fail
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        # Start a read-only server
+        self.startServer(create=0, index=0, read_only=1, keep=0)
+        # Start a read-only-fallback client
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+    def checkReconnectReadOnly(self):
+        # A read-only client reconnects from a read-write to a
+        # read-only server
+
+        # Start a client
+        self._storage = self.openClientStorage(read_only=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+        # Shut down the server
+        self.shutdownServer()
+        self._servers = []
+        # Poll until the client disconnects
+        self.pollDown()
+        # Stores should still fail
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+        # Restart the server
+        self.startServer(create=0, read_only=1, keep=0)
+        # Poll until the client connects
+        self.pollUp()
+        # Stores should still fail
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+    def checkReconnectFallback(self):
+        # A fallback client reconnects from a read-write to a
+        # read-only server
+
+        # Start a client in fallback mode
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should succeed here
+        self._dostore()
+
+        # Shut down the server
+        self.shutdownServer()
+        self._servers = []
+        # Poll until the client disconnects
+        self.pollDown()
+        # Stores should fail now
+        self.assertRaises(ClientDisconnected, self._dostore)
+
+        # Restart the server
+        self.startServer(create=0, read_only=1, keep=0)
+        # Poll until the client connects
+        self.pollUp()
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+    def checkReconnectUpgrade(self):
+        # A fallback client reconnects from a read-only to a
+        # read-write server
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        # Start a read-only server
+        self.startServer(create=0, read_only=1)
+        # Start a client in fallback mode
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+        # Shut down the server
+        self.shutdownServer()
+        self._servers = []
+        # Poll until the client disconnects
+        self.pollDown()
+        # Stores should fail now
+        self.assertRaises(ClientDisconnected, self._dostore)
+
+        # Restart the server, this time read-write
+        self.startServer(create=0, keep=0)
+        # Poll until the client sconnects
+        self.pollUp()
+        # Stores should now succeed
+        self._dostore()
+
+    def checkReconnectSwitch(self):
+        # A fallback client initially connects to a read-only server,
+        # then discovers a read-write server and switches to that
+
+        # We don't want the read-write server created by setUp()
+        self.shutdownServer()
+        self._servers = []
+        # Allocate a second address (for the second server)
+        self._newAddr()
+
+        # Start a read-only server
+        self.startServer(create=0, index=0, read_only=1, keep=0)
+        # Start a client in fallback mode
+        self._storage = self.openClientStorage(read_only_fallback=1)
+        # Stores should fail here
+        self.assertRaises(ReadOnlyError, self._dostore)
+
+        # Start a read-write server
+        self.startServer(index=1, read_only=0, keep=0)
+        # After a while, stores should work
+        for i in range(300): # Try for 30 seconds
+            try:
+                self._dostore()
+                break
+            except (ClientDisconnected, ReadOnlyError):
+                # If the client isn't connected at all, sync() returns
+                # quickly and the test fails because it doesn't wait
+                # long enough for the client.
+                time.sleep(0.1)
+        else:
+            self.fail("Couldn't store after starting a read-write server")
+
+    def checkNoVerificationOnServerRestart(self):
+        self._storage = self.openClientStorage()
+        # When we create a new storage, it should always do a full
+        # verification
+        self.assertEqual(self._storage.verify_result, "full verification")
+        self._dostore()
+        self.shutdownServer()
+        self.pollDown()
+        self._storage.verify_result = None
+        self.startServer(create=0, keep=0)
+        self.pollUp()
+        # There were no transactions committed, so no verification
+        # should be needed.
+        self.assertEqual(self._storage.verify_result, "no verification")
+
+    def checkNoVerificationOnServerRestartWith2Clients(self):
+        perstorage = self.openClientStorage(cache="test")
+        self.assertEqual(perstorage.verify_result, "full verification")
+
+        self._storage = self.openClientStorage()
+        oid = self._storage.new_oid()
+        # When we create a new storage, it should always do a full
+        # verification
+        self.assertEqual(self._storage.verify_result, "full verification")
+        # do two storages of the object to make sure an invalidation
+        # message is generated
+        revid = self._dostore(oid)
+        self._dostore(oid, revid)
+
+        perstorage.load(oid, '')
+
+        self.shutdownServer()
+
+        self.pollDown()
+        self._storage.verify_result = None
+        perstorage.verify_result = None
+        logging.info('2ALLBEEF')
+        self.startServer(create=0, keep=0)
+        self.pollUp()
+        self.pollUp(storage=perstorage)
+        # There were no transactions committed, so no verification
+        # should be needed.
+        self.assertEqual(self._storage.verify_result, "no verification")
+        self.assertEqual(perstorage.verify_result, "no verification")
+        perstorage.close()
+        self._storage.close()
+
+class TimeoutTests(CommonSetupTearDown):
+    timeout = 1
+
+    def checkTimeout(self):
+        storage = self.openClientStorage()
+        txn = Transaction()
+        storage.tpc_begin(txn)
+        storage.tpc_vote(txn)
+        time.sleep(2)
+        self.assertRaises(ClientDisconnected, storage.tpc_finish, txn)
+        storage.close()
+
+    def checkTimeoutOnAbort(self):
+        storage = self.openClientStorage()
+        txn = Transaction()
+        storage.tpc_begin(txn)
+        storage.tpc_vote(txn)
+        storage.tpc_abort(txn)
+        storage.close()
+
+    def checkTimeoutOnAbortNoLock(self):
+        storage = self.openClientStorage()
+        txn = Transaction()
+        storage.tpc_begin(txn)
+        storage.tpc_abort(txn)
+        storage.close()
+
+    def checkTimeoutAfterVote(self):
+        raises = self.assertRaises
+        unless = self.failUnless
+        self._storage = storage = self.openClientStorage()
+        # Assert that the zeo cache is empty
+        unless(not list(storage._cache.contents()))
+        # Create the object
+        oid = storage.new_oid()
+        obj = MinPO(7)
+        # Now do a store, sleeping before the finish so as to cause a timeout
+        t = Transaction()
+        storage.tpc_begin(t)
+        revid1 = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
+        storage.tpc_vote(t)
+        # Now sleep long enough for the storage to time out
+        time.sleep(3)
+        storage.sync()
+        unless(not storage.is_connected())
+        storage._wait()
+        unless(storage.is_connected())
+        # We expect finish to fail
+        raises(ClientDisconnected, storage.tpc_finish, t)
+        # The cache should still be empty
+        unless(not list(storage._cache.contents()))
+        # Load should fail since the object should not be in either the cache
+        # or the server.
+        raises(KeyError, storage.load, oid, '')
+
+    def checkTimeoutProvokingConflicts(self):
+        eq = self.assertEqual
+        raises = self.assertRaises
+        unless = self.failUnless
+        self._storage = storage = self.openClientStorage()
+        # Assert that the zeo cache is empty
+        unless(not list(storage._cache.contents()))
+        # Create the object
+        oid = storage.new_oid()
+        obj = MinPO(7)
+        # We need to successfully commit an object now so we have something to
+        # conflict about.
+        t = Transaction()
+        storage.tpc_begin(t)
+        revid1a = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
+        revid1b = storage.tpc_vote(t)
+        revid1 = handle_serials(oid, revid1a, revid1b)
+        storage.tpc_finish(t)
+        # Now do a store, sleeping before the finish so as to cause a timeout
+        obj.value = 8
+        t = Transaction()
+        storage.tpc_begin(t)
+        revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
+        revid2b = storage.tpc_vote(t)
+        revid2 = handle_serials(oid, revid2a, revid2b)
+        # Now sleep long enough for the storage to time out
+        time.sleep(3)
+        storage.sync()
+        unless(not storage.is_connected())
+        storage._wait()
+        unless(storage.is_connected())
+        # We expect finish to fail
+        raises(ClientDisconnected, storage.tpc_finish, t)
+        # Now we think we've committed the second transaction, but we really
+        # haven't.  A third one should produce a POSKeyError on the server,
+        # which manifests as a ConflictError on the client.
+        obj.value = 9
+        t = Transaction()
+        storage.tpc_begin(t)
+        storage.store(oid, revid2, zodb_pickle(obj), '', t)
+        raises(ConflictError, storage.tpc_vote, t)
+        # Even aborting won't help
+        storage.tpc_abort(t)
+        storage.tpc_finish(t)
+        # Try again
+        obj.value = 10
+        t = Transaction()
+        storage.tpc_begin(t)
+        storage.store(oid, revid2, zodb_pickle(obj), '', t)
+        # Even aborting won't help
+        raises(ConflictError, storage.tpc_vote, t)
+        # Abort this one and try a transaction that should succeed
+        storage.tpc_abort(t)
+        storage.tpc_finish(t)
+        # Now do a store, sleeping before the finish so as to cause a timeout
+        obj.value = 11
+        t = Transaction()
+        storage.tpc_begin(t)
+        revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
+        revid2b = storage.tpc_vote(t)
+        revid2 = handle_serials(oid, revid2a, revid2b)
+        storage.tpc_finish(t)
+        # Now load the object and verify that it has a value of 11
+        data, revid = storage.load(oid, '')
+        eq(zodb_unpickle(data), MinPO(11))
+        eq(revid, revid2)
+
+class MSTThread(threading.Thread):
+
+    __super_init = threading.Thread.__init__
+
+    def __init__(self, testcase, name):
+        self.__super_init(name=name)
+        self.testcase = testcase
+        self.clients = []
+
+    def run(self):
+        tname = self.getName()
+        testcase = self.testcase
+
+        # Create client connections to each server
+        clients = self.clients
+        for i in range(len(testcase.addr)):
+            c = testcase.openClientStorage(addr=testcase.addr[i])
+            c.__name = "C%d" % i
+            clients.append(c)
+
+        for i in range(testcase.ntrans):
+            # Because we want a transaction spanning all storages,
+            # we can't use _dostore().  This is several _dostore() calls
+            # expanded in-line (mostly).
+
+            # Create oid->serial mappings
+            for c in clients:
+                c.__oids = []
+                c.__serials = {}
+
+            # Begin a transaction
+            t = Transaction()
+            for c in clients:
+                #print "%s.%s.%s begin\n" % (tname, c.__name, i),
+                c.tpc_begin(t)
+
+            for j in range(testcase.nobj):
+                for c in clients:
+                    # Create and store a new object on each server
+                    oid = c.new_oid()
+                    c.__oids.append(oid)
+                    data = MinPO("%s.%s.t%d.o%d" % (tname, c.__name, i, j))
+                    #print data.value
+                    data = zodb_pickle(data)
+                    s = c.store(oid, ZERO, data, '', t)
+                    c.__serials.update(handle_all_serials(oid, s))
+
+            # Vote on all servers and handle serials
+            for c in clients:
+                #print "%s.%s.%s vote\n" % (tname, c.__name, i),
+                s = c.tpc_vote(t)
+                c.__serials.update(handle_all_serials(None, s))
+
+            # Finish on all servers
+            for c in clients:
+                #print "%s.%s.%s finish\n" % (tname, c.__name, i),
+                c.tpc_finish(t)
+
+            for c in clients:
+                # Check that we got serials for all oids
+                for oid in c.__oids:
+                    testcase.failUnless(c.__serials.has_key(oid))
+                # Check that we got serials for no other oids
+                for oid in c.__serials.keys():
+                    testcase.failUnless(oid in c.__oids)
+
+    def closeclients(self):
+        # Close clients opened by run()
+        for c in self.clients:
+            try:
+                c.close()
+            except:
+                pass

lib/python/ZEO/tests/InvalidationTests.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import threading
+import time
+from random import Random
+
+import transaction
+
+from BTrees.check import check, display
+from BTrees.OOBTree import OOBTree
+
+from ZEO.tests.TestThread import TestThread
+
+from ZODB.DB import DB
+from ZODB.POSException \
+     import ReadConflictError, ConflictError, VersionLockError
+
+# The tests here let several threads have a go at one or more database
+# instances simultaneously.  Each thread appends a disjoint (from the
+# other threads) sequence of increasing integers to an OOBTree, one at
+# at time (per thread).  This provokes lots of conflicts, and BTrees
+# work hard at conflict resolution too.  An OOBTree is used because
+# that flavor has the smallest maximum bucket size, and so splits buckets
+# more often than other BTree flavors.
+#
+# When these tests were first written, they provoked an amazing number
+# of obscure timing-related bugs in cache consistency logic, revealed
+# by failure of the BTree to pass internal consistency checks at the end,
+# and/or by failure of the BTree to contain all the keys the threads
+# thought they added (i.e., the keys for which transaction.commit()
+# did not raise any exception).
+
+class FailableThread(TestThread):
+
+    # mixin class
+    # subclass must provide
+    # - self.stop attribute (an event)
+    # - self._testrun() method
+
+    def testrun(self):
+        try:
+            self._testrun()
+        except:
+            # Report the failure here to all the other threads, so
+            # that they stop quickly.
+            self.stop.set()
+            raise
+
+
+class StressTask:
+    # Append integers startnum, startnum + step, startnum + 2*step, ...
+    # to 'tree'.  If sleep is given, sleep
+    # that long after each append.  At the end, instance var .added_keys
+    # is a list of the ints the thread believes it added successfully.
+    def __init__(self, testcase, db, threadnum, startnum,
+                 step=2, sleep=None):
+        self.db = db
+        self.threadnum = threadnum
+        self.startnum = startnum
+        self.step = step
+        self.sleep = sleep
+        self.added_keys = []
+        self.tm = transaction.TransactionManager()
+        self.cn = self.db.open(txn_mgr=self.tm)
+        self.cn.sync()
+
+    def doStep(self):
+        tree = self.cn.root()["tree"]
+        key = self.startnum
+        tree[key] = self.threadnum
+
+    def commit(self):
+        cn = self.cn
+        key = self.startnum
+        self.tm.get().note("add key %s" % key)
+        try:
+            self.tm.get().commit()
+        except ConflictError, msg:
+            self.tm.get().abort()
+            cn.sync()
+        else:
+            if self.sleep:
+                time.sleep(self.sleep)
+            self.added_keys.append(key)
+        self.startnum += self.step
+
+    def cleanup(self):
+        self.tm.get().abort()
+        self.cn.close()
+
+def _runTasks(rounds, *tasks):
+    '''run *task* interleaved for *rounds* rounds.'''
+    def commit(run, actions):
+        actions.append(':')
+        for t in run:
+            t.commit()
+        del run[:]
+    r = Random()
+    r.seed(1064589285) # make it deterministic
+    run = []
+    actions = []
+    try:
+        for i in range(rounds):
+            t = r.choice(tasks)
+            if t in run:
+                commit(run, actions)
+            run.append(t)
+            t.doStep()
+            actions.append(`t.startnum`)
+        commit(run,actions)
+        # stderr.write(' '.join(actions)+'\n')
+    finally:
+        for t in tasks:
+            t.cleanup()
+
+
+class StressThread(FailableThread):
+
+    # Append integers startnum, startnum + step, startnum + 2*step, ...
+    # to 'tree' until Event stop is set.  If sleep is given, sleep
+    # that long after each append.  At the end, instance var .added_keys
+    # is a list of the ints the thread believes it added successfully.
+    def __init__(self, testcase, db, stop, threadnum, commitdict,
+                 startnum, step=2, sleep=None):
+        TestThread.__init__(self, testcase)
+        self.db = db
+        self.stop = stop
+        self.threadnum = threadnum
+        self.startnum = startnum
+        self.step = step
+        self.sleep = sleep
+        self.added_keys = []
+        self.commitdict = commitdict
+
+    def _testrun(self):
+        cn = self.db.open()
+        while not self.stop.isSet():
+            try:
+                tree = cn.root()["tree"]
+                break
+            except (ConflictError, KeyError):
+                transaction.abort()
+                cn.sync()
+        key = self.startnum
+        while not self.stop.isSet():
+            try:
+                tree[key] = self.threadnum
+                transaction.get().note("add key %s" % key)
+                transaction.commit()
+                self.commitdict[self] = 1
+                if self.sleep:
+                    time.sleep(self.sleep)
+            except (ReadConflictError, ConflictError), msg:
+                transaction.abort()
+                # sync() is necessary here to process invalidations
+                # if we get a read conflict.  In the read conflict case,
+                # no objects were modified so cn never got registered
+                # with the transaction.
+                cn.sync()
+            else:
+                self.added_keys.append(key)
+            key += self.step
+        cn.close()
+
+class LargeUpdatesThread(FailableThread):
+
+    # A thread that performs a lot of updates.  It attempts to modify
+    # more than 25 objects so that it can test code that runs vote
+    # in a separate thread when it modifies more than 25 objects.
+
+    def __init__(self, testcase, db, stop, threadnum, commitdict, startnum,
+                 step=2, sleep=None):
+        TestThread.__init__(self, testcase)
+        self.db = db
+        self.stop = stop
+        self.threadnum = threadnum
+        self.startnum = startnum
+        self.step = step
+        self.sleep = sleep
+        self.added_keys = []
+        self.commitdict = commitdict
+
+    def testrun(self):
+        try:
+            self._testrun()
+        except:
+            # Report the failure here to all the other threads, so
+            # that they stop quickly.
+            self.stop.set()
+            raise
+
+    def _testrun(self):
+        cn = self.db.open()
+        while not self.stop.isSet():
+            try:
+                tree = cn.root()["tree"]
+                break
+            except (ConflictError, KeyError):
+                # print "%d getting tree abort" % self.threadnum
+                transaction.abort()
+                cn.sync()
+
+        keys_added = {} # set of keys we commit
+        tkeys = []
+        while not self.stop.isSet():
+
+            # The test picks 50 keys spread across many buckets.
+            # self.startnum and self.step ensure that all threads use
+            # disjoint key sets, to minimize conflict errors.
+
+            nkeys = len(tkeys)
+            if nkeys < 50:
+                tkeys = range(self.startnum, 3000, self.step)
+                nkeys = len(tkeys)
+            step = max(int(nkeys / 50), 1)
+            keys = [tkeys[i] for i in range(0, nkeys, step)]
+            for key in keys:
+                try:
+                    tree[key] = self.threadnum
+                except (ReadConflictError, ConflictError), msg:
+                    # print "%d setting key %s" % (self.threadnum, msg)
+                    transaction.abort()
+                    cn.sync()
+                    break
+            else:
+                # print "%d set #%d" % (self.threadnum, len(keys))
+                transaction.get().note("keys %s" % ", ".join(map(str, keys)))
+                try:
+                    transaction.commit()
+                    self.commitdict[self] = 1
+                    if self.sleep:
+                        time.sleep(self.sleep)
+                except ConflictError, msg:
+                    # print "%d commit %s" % (self.threadnum, msg)
+                    transaction.abort()
+                    cn.sync()
+                    continue
+                for k in keys:
+                    tkeys.remove(k)
+                    keys_added[k] = 1
+                # sync() is necessary here to process invalidations
+                # if we get a read conflict.  In the read conflict case,
+                # no objects were modified so cn never got registered
+                # with the transaction.
+                cn.sync()
+        self.added_keys = keys_added.keys()
+        cn.close()
+
+class VersionStressThread(FailableThread):
+
+    def __init__(self, testcase, db, stop, threadnum, commitdict, startnum,
+                 step=2, sleep=None):
+        TestThread.__init__(self, testcase)
+        self.db = db
+        self.stop = stop
+        self.threadnum = threadnum
+        self.startnum = startnum
+        self.step = step
+        self.sleep = sleep
+        self.added_keys = []
+        self.commitdict = commitdict
+
+    def testrun(self):
+        try:
+            self._testrun()
+        except:
+            # Report the failure here to all the other threads, so
+            # that they stop quickly.
+            self.stop.set()
+            raise
+
+    def _testrun(self):
+        commit = 0
+        key = self.startnum
+        while not self.stop.isSet():
+            version = "%s:%s" % (self.threadnum, key)
+            commit = not commit
+            if self.oneupdate(version, key, commit):
+                self.added_keys.append(key)
+                self.commitdict[self] = 1
+            key += self.step
+
+    def oneupdate(self, version, key, commit=1):
+        # The mess of sleeps below were added to reduce the number
+        # of VersionLockErrors, based on empirical observation.
+        # It looks like the threads don't switch enough without
+        # the sleeps.
+
+        cn = self.db.open(version)
+        while not self.stop.isSet():
+            try:
+                tree = cn.root()["tree"]
+                break
+            except (ConflictError, KeyError):
+                transaction.abort()
+                cn.sync()
+        while not self.stop.isSet():
+            try:
+                tree[key] = self.threadnum
+                transaction.commit()
+                if self.sleep:
+                    time.sleep(self.sleep)
+                break
+            except (VersionLockError, ReadConflictError, ConflictError), msg:
+                transaction.abort()
+                # sync() is necessary here to process invalidations
+                # if we get a read conflict.  In the read conflict case,
+                # no objects were modified so cn never got registered
+                # with the transaction.
+                cn.sync()
+                if self.sleep:
+                    time.sleep(self.sleep)
+        try:
+            while not self.stop.isSet():
+                try:
+                    if commit:
+                        self.db.commitVersion(version)
+                        transaction.get().note("commit version %s" % version)
+                    else:
+                        self.db.abortVersion(version)
+                        transaction.get().note("abort version %s" % version)
+                    transaction.commit()
+                    if self.sleep:
+                        time.sleep(self.sleep)
+                    return commit
+                except ConflictError, msg:
+                    transaction.abort()
+                    cn.sync()
+        finally:
+            cn.close()
+        return 0
+
+class InvalidationTests:
+
+    level = 2
+
+    # Minimum # of seconds the main thread lets the workers run.  The
+    # test stops as soon as this much time has elapsed, and all threads
+    # have managed to commit a change.
+    MINTIME = 10
+
+    # Maximum # of seconds the main thread lets the workers run.  We
+    # stop after this long has elapsed regardless of whether all threads
+    # have managed to commit a change.
+    MAXTIME = 300
+
+    StressThread = StressThread
+
+    def _check_tree(self, cn, tree):
+        # Make sure the BTree is sane at the C level.
+        retries = 3
+        while retries:
+            retries -= 1
+            try:
+                check(tree)
+                tree._check()
+            except ReadConflictError:
+                if retries:
+                    transaction.abort()
+                    cn.sync()
+                else:
+                    raise
+            except:
+                display(tree)
+                raise
+
+    def _check_threads(self, tree, *threads):
+        # Make sure the thread's view of the world is consistent with
+        # the actual database state.
+        expected_keys = []
+        errormsgs = []
+        err = errormsgs.append
+        for t in threads:
+            if not t.added_keys:
+                err("thread %d didn't add any keys" % t.threadnum)
+            expected_keys.extend(t.added_keys)
+        expected_keys.sort()
+        actual_keys = list(tree.keys())
+        if expected_keys != actual_keys:
+            err("expected keys != actual keys")
+            for k in expected_keys:
+                if k not in actual_keys:
+                    err("key %s expected but not in tree" % k)
+            for k in actual_keys:
+                if k not in expected_keys:
+                    err("key %s in tree but not expected" % k)
+        if errormsgs:
+            display(tree)
+            self.fail('\n'.join(errormsgs))
+
+    def go(self, stop, commitdict, *threads):
+        # Run the threads
+        for t in threads:
+            t.start()
+        delay = self.MINTIME
+        start = time.time()
+        while time.time() - start <= self.MAXTIME:
+            stop.wait(delay)
+            if stop.isSet():
+                # Some thread failed.  Stop right now.
+                break
+            delay = 2.0
+            if len(commitdict) >= len(threads):
+                break
+            # Some thread still hasn't managed to commit anything.
+        stop.set()
+        for t in threads:
+            t.cleanup()
+
+    def checkConcurrentUpdates2Storages_emulated(self):
+        self._storage = storage1 = self.openClientStorage()
+        storage2 = self.openClientStorage()
+        db1 = DB(storage1)
+        db2 = DB(storage2)
+
+        cn = db1.open()
+        tree = cn.root()["tree"] = OOBTree()
+        transaction.commit()
+        # DM: allow time for invalidations to come in and process them
+        time.sleep(0.1)
+
+        # Run two threads that update the BTree
+        t1 = StressTask(self, db1, 1, 1,)
+        t2 = StressTask(self, db2, 2, 2,)
+        _runTasks(100, t1, t2)
+
+        cn.sync()
+        self._check_tree(cn, tree)
+        self._check_threads(tree, t1, t2)
+
+        cn.close()
+        db1.close()
+        db2.close()
+
+    def checkConcurrentUpdates2Storages(self):
+        self._storage = storage1 = self.openClientStorage()
+        storage2 = self.openClientStorage()
+        db1 = DB(storage1)
+        db2 = DB(storage2)
+        stop = threading.Event()
+
+        cn = db1.open()
+        tree = cn.root()["tree"] = OOBTree()
+        transaction.commit()
+        cn.close()
+
+        # Run two threads that update the BTree
+        cd = {}
+        t1 = self.StressThread(self, db1, stop, 1, cd, 1)
+        t2 = self.StressThread(self, db2, stop, 2, cd, 2)
+        self.go(stop, cd, t1, t2)
+
+        while db1.lastTransaction() != db2.lastTransaction():
+            db1._storage.sync()
+            db2._storage.sync()
+
+        cn = db1.open()
+        tree = cn.root()["tree"]
+        self._check_tree(cn, tree)
+        self._check_threads(tree, t1, t2)
+
+        cn.close()
+        db1.close()
+        db2.close()
+
+    def checkConcurrentUpdates1Storage(self):
+        self._storage = storage1 = self.openClientStorage()
+        db1 = DB(storage1)
+        stop = threading.Event()
+
+        cn = db1.open()
+        tree = cn.root()["tree"] = OOBTree()
+        transaction.commit()
+        cn.close()
+
+        # Run two threads that update the BTree
+        cd = {}
+        t1 = self.StressThread(self, db1, stop, 1, cd, 1, sleep=0.01)
+        t2 = self.StressThread(self, db1, stop, 2, cd, 2, sleep=0.01)
+        self.go(stop, cd, t1, t2)
+
+        cn = db1.open()
+        tree = cn.root()["tree"]
+        self._check_tree(cn, tree)
+        self._check_threads(tree, t1, t2)
+
+        cn.close()
+        db1.close()
+
+    def checkConcurrentUpdates2StoragesMT(self):
+        self._storage = storage1 = self.openClientStorage()
+        db1 = DB(storage1)
+        db2 = DB(self.openClientStorage())
+        stop = threading.Event()
+
+        cn = db1.open()
+        tree = cn.root()["tree"] = OOBTree()
+        transaction.commit()
+        cn.close()
+
+        # Run three threads that update the BTree.
+        # Two of the threads share a single storage so that it
+        # is possible for both threads to read the same object
+        # at the same time.
+
+        cd = {}
+        t1 = self.StressThread(self, db1, stop, 1, cd, 1, 3)
+        t2 = self.StressThread(self, db2, stop, 2, cd, 2, 3, 0.01)
+        t3 = self.StressThread(self, db2, stop, 3, cd, 3, 3, 0.01)
+        self.go(stop, cd, t1, t2, t3)
+
+        while db1.lastTransaction() != db2.lastTransaction():
+            db1._storage.sync()
+            db2._storage.sync()
+
+
+        cn = db1.open()
+        tree = cn.root()["tree"]
+        self._check_tree(cn, tree)
+        self._check_threads(tree, t1, t2, t3)
+
+        cn.close()
+        db1.close()
+        db2.close()
+
+    def checkConcurrentUpdatesInVersions(self):
+        self._storage = storage1 = self.openClientStorage()
+        db1 = DB(storage1)
+        db2 = DB(self.openClientStorage())
+        stop = threading.Event()
+
+        cn = db1.open()
+        tree = cn.root()["tree"] = OOBTree()
+        transaction.commit()
+        cn.close()
+
+        # Run three threads that update the BTree.
+        # Two of the threads share a single storage so that it
+        # is possible for both threads to read the same object
+        # at the same time.
+
+        cd = {}
+        t1 = VersionStressThread(self, db1, stop, 1, cd, 1, 3)
+        t2 = VersionStressThread(self, db2, stop, 2, cd, 2, 3, 0.01)
+        t3 = VersionStressThread(self, db2, stop, 3, cd, 3, 3, 0.01)
+        self.go(stop, cd, t1, t2, t3)
+
+        while db1.lastTransaction() != db2.lastTransaction():
+            db1._storage.sync()
+            db2._storage.sync()
+
+
+        cn = db1.open()
+        tree = cn.root()["tree"]
+        self._check_tree(cn, tree)
+        self._check_threads(tree, t1, t2, t3)
+
+        cn.close()
+        db1.close()
+        db2.close()
+
+    def checkConcurrentLargeUpdates(self):
+        # Use 3 threads like the 2StorageMT test above.
+        self._storage = storage1 = self.openClientStorage()
+        db1 = DB(storage1)
+        db2 = DB(self.openClientStorage())
+        stop = threading.Event()
+
+        cn = db1.open()
+        tree = cn.root()["tree"] = OOBTree()
+        for i in range(0, 3000, 2):
+            tree[i] = 0
+        transaction.commit()
+        cn.close()
+
+        # Run three threads that update the BTree.
+        # Two of the threads share a single storage so that it
+        # is possible for both threads to read the same object
+        # at the same time.
+
+        cd = {}
+        t1 = LargeUpdatesThread(self, db1, stop, 1, cd, 1, 3, 0.02)
+        t2 = LargeUpdatesThread(self, db2, stop, 2, cd, 2, 3, 0.01)
+        t3 = LargeUpdatesThread(self, db2, stop, 3, cd, 3, 3, 0.01)
+        self.go(stop, cd, t1, t2, t3)
+
+        while db1.lastTransaction() != db2.lastTransaction():
+            db1._storage.sync()
+            db2._storage.sync()
+
+        cn = db1.open()
+        tree = cn.root()["tree"]
+        self._check_tree(cn, tree)
+
+        # Purge the tree of the dummy entries mapping to 0.
+        losers = [k for k, v in tree.items() if v == 0]
+        for k in losers:
+            del tree[k]
+        transaction.commit()
+
+        self._check_threads(tree, t1, t2, t3)
+
+        cn.close()
+        db1.close()
+        db2.close()

lib/python/ZEO/tests/TestThread.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""A Thread base class for use with unittest."""
+
+from cStringIO import StringIO
+import threading
+import traceback
+
+class TestThread(threading.Thread):
+    __super_init = threading.Thread.__init__
+    __super_run = threading.Thread.run
+
+    def __init__(self, testcase, group=None, target=None, name=None,
+                 args=(), kwargs={}, verbose=None):
+        self.__super_init(group, target, name, args, kwargs, verbose)
+        self.setDaemon(1)
+        self._testcase = testcase
+
+    def run(self):
+        try:
+            self.testrun()
+        except Exception:
+            s = StringIO()
+            traceback.print_exc(file=s)
+            self._testcase.fail("Exception in thread %s:\n%s\n" %
+                                (self, s.getvalue()))
+
+    def cleanup(self, timeout=15):
+        self.join(timeout)
+        if self.isAlive():
+            self._testcase.fail("Thread did not finish: %s" % self)

lib/python/ZEO/tests/ThreadTests.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Compromising positions involving threads."""
+
+import threading
+
+import transaction
+from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
+import ZEO.ClientStorage
+
+ZERO = '\0'*8
+
+class BasicThread(threading.Thread):
+    def __init__(self, storage, doNextEvent, threadStartedEvent):
+        self.storage = storage
+        self.trans = transaction.Transaction()
+        self.doNextEvent = doNextEvent
+        self.threadStartedEvent = threadStartedEvent
+        self.gotValueError = 0
+        self.gotDisconnected = 0
+        threading.Thread.__init__(self)
+        self.setDaemon(1)
+
+    def join(self):
+        threading.Thread.join(self, 10)
+        assert not self.isAlive()
+
+
+class GetsThroughVoteThread(BasicThread):
+    # This thread gets partially through a transaction before it turns
+    # execution over to another thread.  We're trying to establish that a
+    # tpc_finish() after a storage has been closed by another thread will get
+    # a ClientStorageError error.
+    #
+    # This class gets does a tpc_begin(), store(), tpc_vote() and is waiting
+    # to do the tpc_finish() when the other thread closes the storage.
+    def run(self):
+        self.storage.tpc_begin(self.trans)
+        oid = self.storage.new_oid()
+        self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans)
+        self.storage.tpc_vote(self.trans)
+        self.threadStartedEvent.set()
+        self.doNextEvent.wait(10)
+        try:
+            self.storage.tpc_finish(self.trans)
+        except ZEO.ClientStorage.ClientStorageError:
+            self.gotValueError = 1
+            self.storage.tpc_abort(self.trans)
+
+
+class GetsThroughBeginThread(BasicThread):
+    # This class is like the above except that it is intended to be run when
+    # another thread is already in a tpc_begin().  Thus, this thread will
+    # block in the tpc_begin until another thread closes the storage.  When
+    # that happens, this one will get disconnected too.
+    def run(self):
+        try:
+            self.storage.tpc_begin(self.trans)
+        except ZEO.ClientStorage.ClientStorageError:
+            self.gotValueError = 1
+
+
+class ThreadTests:
+    # Thread 1 should start a transaction, but not get all the way through it.
+    # Main thread should close the connection.  Thread 1 should then get
+    # disconnected.
+    def checkDisconnectedOnThread2Close(self):
+        doNextEvent = threading.Event()
+        threadStartedEvent = threading.Event()
+        thread1 = GetsThroughVoteThread(self._storage,
+                                        doNextEvent, threadStartedEvent)
+        thread1.start()
+        threadStartedEvent.wait(10)
+        self._storage.close()
+        doNextEvent.set()
+        thread1.join()
+        self.assertEqual(thread1.gotValueError, 1)
+
+    # Thread 1 should start a transaction, but not get all the way through
+    # it.  While thread 1 is in the middle of the transaction, a second thread
+    # should start a transaction, and it will block in the tcp_begin() --
+    # because thread 1 has acquired the lock in its tpc_begin().  Now the main
+    # thread closes the storage and both sub-threads should get disconnected.
+    def checkSecondBeginFails(self):
+        doNextEvent = threading.Event()
+        threadStartedEvent = threading.Event()
+        thread1 = GetsThroughVoteThread(self._storage,
+                                        doNextEvent, threadStartedEvent)
+        thread2 = GetsThroughBeginThread(self._storage,
+                                         doNextEvent, threadStartedEvent)
+        thread1.start()
+        threadStartedEvent.wait(1)
+        thread2.start()
+        self._storage.close()
+        doNextEvent.set()
+        thread1.join()
+        thread2.join()
+        self.assertEqual(thread1.gotValueError, 1)
+        self.assertEqual(thread2.gotValueError, 1)
+
+    # Run a bunch of threads doing small and large stores in parallel
+    def checkMTStores(self):
+        threads = []
+        for i in range(5):
+            t = threading.Thread(target=self.mtstorehelper)
+            threads.append(t)
+            t.start()
+        for t in threads:
+            t.join(30)
+        for i in threads:
+            self.failUnless(not t.isAlive())
+
+    # Helper for checkMTStores
+    def mtstorehelper(self):
+        name = threading.currentThread().getName()
+        objs = []
+        for i in range(10):
+            objs.append(MinPO("X" * 200000))
+            objs.append(MinPO("X"))
+        for obj in objs:
+            self._dostore(data=obj)

lib/python/ZEO/tests/__init__.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################

lib/python/ZEO/tests/auth_plaintext.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Implements plaintext password authentication. The password is stored in
+an SHA hash in the Database. The client sends over the plaintext
+password, and the SHA hashing is done on the server side.
+
+This mechanism offers *no network security at all*; the only security
+is provided by not storing plaintext passwords on disk.
+"""
+
+import sha
+
+from ZEO.StorageServer import ZEOStorage
+from ZEO.auth import register_module
+from ZEO.auth.base import Client, Database
+
+def session_key(username, realm, password):
+    return sha.new("%s:%s:%s" % (username, realm, password)).hexdigest()
+
+class StorageClass(ZEOStorage):
+
+    def auth(self, username, password):
+        try:
+            dbpw = self.database.get_password(username)
+        except LookupError:
+            return 0
+
+        password_dig = sha.new(password).hexdigest()
+        if dbpw == password_dig:
+            self.connection.setSessionKey(session_key(username,
+                                                      self.database.realm,
+                                                      password))
+        return self.finish_auth(dbpw == password_dig)
+
+class PlaintextClient(Client):
+    extensions = ["auth"]
+
+    def start(self, username, realm, password):
+        if self.stub.auth(username, password):
+            return session_key(username, realm, password)
+        else:
+            return None
+
+register_module("plaintext", StorageClass, PlaintextClient, Database)

lib/python/ZEO/tests/deadlock.py

+import ZODB
+from ZODB.POSException import ConflictError
+from ZEO.ClientStorage import ClientStorage, ClientDisconnected
+from ZEO.zrpc.error import DisconnectedError
+
+import os
+import random
+import time
+
+L = range(1, 100)
+
+def main():
+    z1 = ClientStorage(('localhost', 2001), wait=1)
+    z2 = ClientStorage(('localhost', 2002), wait=2)
+    db1 = ZODB.DB(z1)
+    db2 = ZODB.DB(z2)
+    c1 = db1.open()
+    c2 = db2.open()
+    r1 = c1.root()
+    r2 = c2.root()
+
+    while 1:
+        try:
+            try:
+                update(r1, r2)
+            except ConflictError, msg:
+                print msg
+                transaction.abort()
+                c1.sync()
+                c2.sync()
+        except (ClientDisconnected, DisconnectedError), err:
+            print "disconnected", err
+            time.sleep(2)
+
+def update(r1, r2):
+    k1 = random.choice(L)
+    k2 = random.choice(L)
+
+    updates = [(k1, r1),
+               (k2, r2)]
+    random.shuffle(updates)
+    for key, root in updates:
+        root[key] = time.time()
+    transaction.commit()
+    print os.getpid(), k1, k2
+
+if __name__ == "__main__":
+    main()

lib/python/ZEO/tests/forker.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Library for forking storage server and connecting client storage"""
+
+import os
+import sys
+import time
+import errno
+import socket
+import logging
+import StringIO
+import tempfile
+import logging
+
+logger = logging.getLogger('ZEO.tests.forker')
+
+class ZEOConfig:
+    """Class to generate ZEO configuration file. """
+
+    def __init__(self, addr):
+        self.address = addr
+        self.read_only = None
+        self.invalidation_queue_size = None
+        self.monitor_address = None
+        self.transaction_timeout = None
+        self.authentication_protocol = None
+        self.authentication_database = None
+        self.authentication_realm = None
+
+    def dump(self, f):
+        print >> f, "<zeo>"
+        print >> f, "address %s:%s" % self.address
+        if self.read_only is not None:
+            print >> f, "read-only", self.read_only and "true" or "false"
+        if self.invalidation_queue_size is not None:
+            print >> f, "invalidation-queue-size", self.invalidation_queue_size
+        if self.monitor_address is not None:
+            print >> f, "monitor-address %s:%s" % self.monitor_address
+        if self.transaction_timeout is not None:
+            print >> f, "transaction-timeout", self.transaction_timeout
+        if self.authentication_protocol is not None:
+            print >> f, "authentication-protocol", self.authentication_protocol
+        if self.authentication_database is not None:
+            print >> f, "authentication-database", self.authentication_database
+        if self.authentication_realm is not None:
+            print >> f, "authentication-realm", self.authentication_realm
+        print >> f, "</zeo>"
+
+        logger = logging.getLogger()
+        print >> f
+        print >> f, "<eventlog>"
+        print >> f, "level", logger.level
+        for handler in logger.handlers:
+            if isinstance(handler, logging.FileHandler):
+                path = handler.baseFilename
+            elif isinstance(handler, logging.StreamHandler):
+                stream = handler.stream
+                if stream.name == "<stdout>":
+                    path = "STDOUT"
+                elif stream.name == "<stderr>":
+                    path = "STDERR"
+                else:
+                    # just drop it on the floor; unlikely an issue when testing
+                    continue
+            else:
+                # just drop it on the floor; unlikely an issue when testing
+                continue
+            # This doesn't convert the level values to names, so the
+            # generated configuration isn't as nice as it could be,
+            # but it doesn't really need to be.
+            print >> f, "<logfile>"
+            print >> f, "level", handler.level
+            print >> f, "path ", path
+            if handler.formatter:
+                formatter = handler.formatter
+                if formatter._fmt:
+                    print >> f, "format", encode_format(formatter._fmt)
+                if formatter.datefmt:
+                    print >> f, "dateformat", encode_format(formatter.datefmt)
+            print >> f, "</logfile>"
+        print >> f, "</eventlog>"
+
+    def __str__(self):
+        f = StringIO.StringIO()
+        self.dump(f)
+        return f.getvalue()
+
+
+def encode_format(fmt):
+    # The list of replacements mirrors
+    # ZConfig.components.logger.handlers._control_char_rewrites
+    for xform in (("\n", r"\n"), ("\t", r"\t"), ("\b", r"\b"),
+                  ("\f", r"\f"), ("\r", r"\r")):
+        fmt = fmt.replace(*xform)
+    return fmt
+
+
+def start_zeo_server(storage_conf, zeo_conf, port, keep=0):
+    """Start a ZEO server in a separate process.
+
+    Takes two positional arguments a string containing the storage conf
+    and a ZEOConfig object.
+
+    Returns the ZEO port, the test server port, the pid, and the path
+    to the config file.
+    """
+
+    # Store the config info in a temp file.
+    tmpfile = tempfile.mktemp(".conf")
+    fp = open(tmpfile, 'w')
+    zeo_conf.dump(fp)
+    fp.write(storage_conf)
+    fp.close()
+
+    # Find the zeoserver script
+    import ZEO.tests.zeoserver
+    script = ZEO.tests.zeoserver.__file__
+    if script.endswith('.pyc'):
+        script = script[:-1]
+
+    # Create a list of arguments, which we'll tuplify below
+    qa = _quote_arg
+    args = [qa(sys.executable), qa(script), '-C', qa(tmpfile)]
+    if keep:
+        args.append("-k")
+    d = os.environ.copy()
+    d['PYTHONPATH'] = os.pathsep.join(sys.path)
+    pid = os.spawnve(os.P_NOWAIT, sys.executable, tuple(args), d)
+    adminaddr = ('localhost', port + 1)
+    # We need to wait until the server starts, but not forever.
+    # 30 seconds is a somewhat arbitrary upper bound.  A BDBStorage
+    # takes a long time to open -- more than 10 seconds on occasion.
+    for i in range(120):
+        time.sleep(0.25)
+        try:
+            logger.debug('connect %s', i)
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.connect(adminaddr)
+            ack = s.recv(1024)
+            s.close()
+            logging.debug('acked: %s' % ack)
+            break
+        except socket.error, e:
+            if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
+                raise
+            s.close()
+    else:
+        logging.debug('boo hoo')
+        raise
+    return ('localhost', port), adminaddr, pid, tmpfile
+
+
+if sys.platform[:3].lower() == "win":
+    def _quote_arg(s):
+        return '"%s"' % s
+else:
+    def _quote_arg(s):
+        return s
+
+
+def shutdown_zeo_server(adminaddr):
+    # Do this in a loop to guard against the possibility that the
+    # client failed to connect to the adminaddr earlier.  That really
+    # only requires two iterations, but do a third for pure
+    # superstition.
+    for i in range(3):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.connect(adminaddr)
+        except socket.error, e:
+            if e[0] == errno.ECONNREFUSED and i > 0:
+                break
+            raise
+        try:
+            ack = s.recv(1024)
+        except socket.error, e:
+            if e[0] == errno.ECONNRESET:
+                raise
+            ack = 'no ack received'
+        logger.debug('shutdown_zeo_server(): acked: %s' % ack)
+        s.close()

lib/python/ZEO/tests/multi.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""A multi-client test of the ZEO storage server"""
+# XXX This code is currently broken.
+
+import ZODB, ZODB.DB, ZODB.FileStorage, ZODB.POSException
+import persistent
+import persistent.mapping
+import transaction
+from ZEO.tests import forker
+
+import os
+import tempfile
+import time
+import types
+
+VERBOSE = 1
+CLIENTS = 4
+RECORDS_PER_CLIENT = 100
+CONFLICT_DELAY = 0.1
+CONNECT_DELAY = 0.1
+CLIENT_CACHE = '' # use temporary cache
+
+class Record(persistent.Persistent):
+    def __init__(self, client=None, value=None):
+        self.client = client
+        self.value = None
+        self.next = None
+
+    def set_next(self, next):
+        self.next = next
+
+class Stats(persistent.Persistent):
+    def __init__(self):
+        self.begin = time.time()
+        self.end = None
+
+    def done(self):
+        self.end = time.time()
+
+def init_storage():
+    path = tempfile.mktemp()
+    if VERBOSE:
+        print "FileStorage path:", path
+    fs = ZODB.FileStorage.FileStorage(path)
+
+    db = ZODB.DB(fs)
+    root = db.open().root()
+    root["multi"] = persistent.mapping.PersistentMapping()
+    transaction.commit()
+
+    return fs
+
+def start_server(addr):
+    storage = init_storage()
+    pid, exit = forker.start_zeo_server(storage, addr)
+    return pid, exit
+
+def start_client(addr, client_func=None):
+    pid = os.fork()
+    if pid == 0:
+        try:
+            import ZEO.ClientStorage
+            if VERBOSE:
+                print "Client process started:", os.getpid()
+            cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE)
+            if client_func is None:
+                run(cli)
+            else:
+                client_func(cli)
+            cli.close()
+        finally:
+            os._exit(0)
+    else:
+        return pid
+
+def run(storage):
+    if hasattr(storage, 'is_connected'):
+        while not storage.is_connected():
+            time.sleep(CONNECT_DELAY)
+    pid = os.getpid()
+    print "Client process connected:", pid, storage
+    db = ZODB.DB(storage)
+    root = db.open().root()
+    while 1:
+        try:
+            s = root[pid] = Stats()
+            transaction.commit()
+        except ZODB.POSException.ConflictError:
+            transaction.abort()
+            time.sleep(CONFLICT_DELAY)
+        else:
+            break
+
+    dict = root["multi"]
+    prev = None
+    i = 0
+    while i < RECORDS_PER_CLIENT:
+        try:
+            size = len(dict)
+            r = dict[size] = Record(pid, size)
+            if prev:
+                prev.set_next(r)
+            transaction.commit()
+        except ZODB.POSException.ConflictError, err:
+            transaction.abort()
+            time.sleep(CONFLICT_DELAY)
+        else:
+            i = i + 1
+            if VERBOSE and (i < 5 or i % 10 == 0):
+                print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT)
+    s.done()
+    transaction.commit()
+
+    print "Client completed:", pid
+
+def main(client_func=None):
+    if VERBOSE:
+        print "Main process:", os.getpid()
+    addr = tempfile.mktemp()
+    t0 = time.time()
+    server_pid, server = start_server(addr)
+    t1 = time.time()
+    pids = []
+    for i in range(CLIENTS):
+        pids.append(start_client(addr, client_func))
+    for pid in pids:
+        assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \
+               (repr(pid), type(pid))
+        try:
+            if VERBOSE:
+                print "waitpid(%s)" % repr(pid)
+            os.waitpid(pid, 0)
+        except os.error, err:
+            print "waitpid(%s) failed: %s" % (repr(pid), err)
+    t2 = time.time()
+    server.close()
+    os.waitpid(server_pid, 0)
+
+    # XXX Should check that the results are consistent!
+
+    print "Total time:", t2 - t0
+    print "Server start time", t1 - t0
+    print "Client time:", t2 - t1
+
+if __name__ == "__main__":
+    main()

lib/python/ZEO/tests/speed.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+usage="""Test speed of a ZODB storage
+
+Options:
+
+    -d file    The data file to use as input.
+               The default is this script.
+
+    -n n       The number of repititions
+
+    -s module  A module that defines a 'Storage'
+               attribute, which is an open storage.
+               If not specified, a FileStorage will ne
+               used.
+
+    -z         Test compressing data
+
+    -D         Run in debug mode
+
+    -L         Test loads as well as stores by minimizing
+               the cache after eachrun
+
+    -M         Output means only
+
+    -C         Run with a persistent client cache
+
+    -U         Run ZEO using a Unix domain socket
+
+    -t n       Number of concurrent threads to run.
+"""
+
+import asyncore
+import sys, os, getopt, time
+##sys.path.insert(0, os.getcwd())
+
+import persistent
+import transaction
+import ZODB
+from ZODB.POSException import ConflictError
+from ZEO.tests import forker
+
+class P(persistent.Persistent):
+    pass
+
+fs_name = "zeo-speed.fs"
+
+class ZEOExit(asyncore.file_dispatcher):
+    """Used to exit ZEO.StorageServer when run is done"""
+    def writable(self):
+        return 0
+    def readable(self):
+        return 1
+    def handle_read(self):
+        buf = self.recv(4)
+        assert buf == "done"
+        self.delete_fs()
+        os._exit(0)
+    def handle_close(self):
+        print "Parent process exited unexpectedly"
+        self.delete_fs()
+        os._exit(0)
+    def delete_fs(self):
+        os.unlink(fs_name)
+        os.unlink(fs_name + ".lock")
+        os.unlink(fs_name + ".tmp")
+
+def work(db, results, nrep, compress, data, detailed, minimize, threadno=None):
+    for j in range(nrep):
+        for r in 1, 10, 100, 1000:
+            t = time.time()
+            conflicts = 0
+
+            jar = db.open()
+            while 1:
+                try:
+                    transaction.begin()
+                    rt = jar.root()
+                    key = 's%s' % r
+                    if rt.has_key(key):
+                        p = rt[key]
+                    else:
+                        rt[key] = p =P()
+                    for i in range(r):
+                        v = getattr(p, str(i), P())
+                        if compress is not None:
+                            v.d = compress(data)
+                        else:
+                            v.d = data
+                        setattr(p, str(i), v)
+                    transaction.commit()
+                except ConflictError:
+                    conflicts = conflicts + 1
+                else:
+                    break
+            jar.close()
+
+            t = time.time() - t
+            if detailed:
+                if threadno is None:
+                    print "%s\t%s\t%.4f\t%d" % (j, r, t, conflicts)
+                else:
+                    print "%s\t%s\t%.4f\t%d\t%d" % (j, r, t, conflicts,
+                                                    threadno)
+            results[r].append((t, conflicts))
+            rt=d=p=v=None # release all references
+            if minimize:
+                time.sleep(3)
+                jar.cacheMinimize()
+
+def main(args):
+    opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U')
+    s = None
+    compress = None
+    data=sys.argv[0]
+    nrep=5
+    minimize=0
+    detailed=1
+    cache = None
+    domain = 'AF_INET'
+    threads = 1
+    for o, v in opts:
+        if o=='-n': nrep = int(v)
+        elif o=='-d': data = v
+        elif o=='-s': s = v
+        elif o=='-z':
+            import zlib
+            compress = zlib.compress
+        elif o=='-L':
+            minimize=1
+        elif o=='-M':
+            detailed=0
+        elif o=='-D':
+            global debug
+            os.environ['STUPID_LOG_FILE']=''
+            os.environ['STUPID_LOG_SEVERITY']='-999'
+            debug = 1
+        elif o == '-C':
+            cache = 'speed'
+        elif o == '-U':
+            domain = 'AF_UNIX'
+        elif o == '-t':
+            threads = int(v)
+
+    zeo_pipe = None
+    if s:
+        s = __import__(s, globals(), globals(), ('__doc__',))
+        s = s.Storage
+        server = None
+    else:
+        s, server, pid = forker.start_zeo("FileStorage",
+                                          (fs_name, 1), domain=domain)
+
+    data=open(data).read()
+    db=ZODB.DB(s,
+               # disable cache deactivation
+               cache_size=4000,
+               cache_deactivate_after=6000,)
+
+    print "Beginning work..."
+    results={1:[], 10:[], 100:[], 1000:[]}
+    if threads > 1:
+        import threading
+        l = []
+        for i in range(threads):
+            t = threading.Thread(target=work,
+                                 args=(db, results, nrep, compress, data,
+                                       detailed, minimize, i))
+            l.append(t)
+        for t in l:
+            t.start()
+        for t in l:
+            t.join()
+
+    else:
+        work(db, results, nrep, compress, data, detailed, minimize)
+
+    if server is not None:
+        server.close()
+        os.waitpid(pid, 0)
+
+    if detailed:
+        print '-'*24
+    print "num\tmean\tmin\tmax"
+    for r in 1, 10, 100, 1000:
+        times = []
+        for time, conf in results[r]:
+            times.append(time)
+        t = mean(times)
+        print "%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times))
+
+def mean(l):
+    tot = 0
+    for v in l:
+        tot = tot + v
+    return tot / len(l)
+
+##def compress(s):
+##    c = zlib.compressobj()
+##    o = c.compress(s)
+##    return o + c.flush()
+
+if __name__=='__main__':
+    main(sys.argv[1:])

lib/python/ZEO/tests/stress.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""A ZEO client-server stress test to look for leaks.
+
+The stress test should run in an infinite loop and should involve
+multiple connections.
+"""
+# XXX This code is currently broken.
+
+import transaction
+import ZODB
+from ZODB.MappingStorage import MappingStorage
+from ZODB.tests import MinPO
+from ZEO.ClientStorage import ClientStorage
+from ZEO.tests import forker
+
+import os
+import random
+import types
+
+NUM_TRANSACTIONS_PER_CONN = 10
+NUM_CONNECTIONS = 10
+NUM_ROOTS = 20
+MAX_DEPTH = 20
+MIN_OBJSIZE = 128
+MAX_OBJSIZE = 2048
+
+def an_object():
+    """Return an object suitable for a PersistentMapping key"""
+    size = random.randrange(MIN_OBJSIZE, MAX_OBJSIZE)
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        buf = f.read(size)
+        f.close()
+        return buf
+    else:
+        f = open(MinPO.__file__)
+        l = list(f.read(size))
+        f.close()
+        random.shuffle(l)
+        return "".join(l)
+
+def setup(cn):
+    """Initialize the database with some objects"""
+    root = cn.root()
+    for i in range(NUM_ROOTS):
+        prev = an_object()
+        for j in range(random.randrange(1, MAX_DEPTH)):
+            o = MinPO.MinPO(prev)
+            prev = o
+        root[an_object()] = o
+        transaction.commit()
+    cn.close()
+
+def work(cn):
+    """Do some work with a transaction"""
+    cn.sync()
+    root = cn.root()
+    obj = random.choice(root.values())
+    # walk down to the bottom
+    while not isinstance(obj.value, types.StringType):
+        obj = obj.value
+    obj.value = an_object()
+    transaction.commit()
+
+def main():
+    # Yuck!  Need to cleanup forker so that the API is consistent
+    # across Unix and Windows, at least if that's possible.
+    if os.name == "nt":
+        zaddr, tport, pid = forker.start_zeo_server('MappingStorage', ())
+        def exitserver():
+            import socket
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.connect(tport)
+            s.close()
+    else:
+        zaddr = '', random.randrange(20000, 30000)
+        pid, exitobj = forker.start_zeo_server(MappingStorage(), zaddr)
+        def exitserver():
+            exitobj.close()
+
+    while 1:
+        pid = start_child(zaddr)
+        print "started", pid
+        os.waitpid(pid, 0)
+
+    exitserver()
+
+def start_child(zaddr):
+
+    pid = os.fork()
+    if pid != 0:
+        return pid
+    try:
+        _start_child(zaddr)
+    finally:
+        os._exit(0)
+
+def _start_child(zaddr):
+    storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1)
+    db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS)
+    setup(db.open())
+    conns = []
+    conn_count = 0
+
+    for i in range(NUM_CONNECTIONS):
+        c = db.open()
+        c.__count = 0
+        conns.append(c)
+        conn_count += 1
+
+    while conn_count < 25:
+        c = random.choice(conns)
+        if c.__count > NUM_TRANSACTIONS_PER_CONN:
+            conns.remove(c)
+            c.close()
+            conn_count += 1
+            c = db.open()
+            c.__count = 0
+            conns.append(c)
+        else:
+            c.__count += 1
+        work(c)
+
+if __name__ == "__main__":
+    main()

lib/python/ZEO/tests/testAuth.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Test suite for AuthZEO."""
+
+import os
+import tempfile
+import time
+import unittest
+
+from ZEO import zeopasswd
+from ZEO.Exceptions import ClientDisconnected
+from ZEO.tests.ConnectionTests import CommonSetupTearDown
+
+class AuthTest(CommonSetupTearDown):
+    __super_getServerConfig = CommonSetupTearDown.getServerConfig
+    __super_setUp = CommonSetupTearDown.setUp
+    __super_tearDown = CommonSetupTearDown.tearDown
+
+    realm = None
+
+    def setUp(self):
+        self.pwfile = tempfile.mktemp()
+        if self.realm:
+            self.pwdb = self.dbclass(self.pwfile, self.realm)
+        else:
+            self.pwdb = self.dbclass(self.pwfile)
+        self.pwdb.add_user("foo", "bar")
+        self.pwdb.save()
+        self._checkZEOpasswd()
+        self.__super_setUp()
+
+    def _checkZEOpasswd(self):
+        args = ["-f", self.pwfile, "-p", self.protocol]
+        if self.protocol == "plaintext":
+            from ZEO.auth.base import Database
+            zeopasswd.main(args + ["-d", "foo"], Database)
+            zeopasswd.main(args + ["foo", "bar"], Database)
+        else:
+            zeopasswd.main(args + ["-d", "foo"])
+            zeopasswd.main(args + ["foo", "bar"])
+
+    def tearDown(self):
+        self.__super_tearDown()
+        os.remove(self.pwfile)
+
+    def getConfig(self, path, create, read_only):
+        return "<mappingstorage 1/>"
+
+    def getServerConfig(self, addr, ro_svr):
+        zconf = self.__super_getServerConfig(addr, ro_svr)
+        zconf.authentication_protocol = self.protocol
+        zconf.authentication_database = self.pwfile
+        zconf.authentication_realm = self.realm
+        return zconf
+
+    def wait(self):
+        for i in range(25):
+            time.sleep(0.1)
+            if self._storage.test_connection:
+                return
+        self.fail("Timed out waiting for client to authenticate")
+
+    def testOK(self):
+        # Sleep for 0.2 seconds to give the server some time to start up
+        # seems to be needed before and after creating the storage
+        self._storage = self.openClientStorage(wait=0, username="foo",
+                                              password="bar", realm=self.realm)
+        self.wait()
+
+        self.assert_(self._storage._connection)
+        self._storage._connection.poll()
+        self.assert_(self._storage.is_connected())
+        # Make a call to make sure the mechanism is working
+        self._storage.versions()
+
+    def testNOK(self):
+        self._storage = self.openClientStorage(wait=0, username="foo",
+                                              password="noogie",
+                                              realm=self.realm)
+        self.wait()
+        # If the test established a connection, then it failed.
+        self.failIf(self._storage._connection)
+
+    def testUnauthenticatedMessage(self):
+        # Test that an unauthenticated message is rejected by the server
+        # if it was sent after the connection was authenticated.
+        # Sleep for 0.2 seconds to give the server some time to start up
+        # seems to be needed before and after creating the storage
+        self._storage = self.openClientStorage(wait=0, username="foo",
+                                              password="bar", realm=self.realm)
+        self.wait()
+        self._storage.versions()
+        # Manually clear the state of the hmac connection
+        self._storage._connection._SizedMessageAsyncConnection__hmac_send = None
+        # Once the client stops using the hmac, it should be disconnected.
+        self.assertRaises(ClientDisconnected, self._storage.versions)
+
+
+class PlainTextAuth(AuthTest):
+    import ZEO.tests.auth_plaintext
+    protocol = "plaintext"
+    database = "authdb.sha"
+    dbclass = ZEO.tests.auth_plaintext.Database
+    realm = "Plaintext Realm"
+
+class DigestAuth(AuthTest):
+    import ZEO.auth.auth_digest
+    protocol = "digest"
+    database = "authdb.digest"
+    dbclass = ZEO.auth.auth_digest.DigestDatabase
+    realm = "Digest Realm"
+
+test_classes = [PlainTextAuth, DigestAuth]
+
+def test_suite():
+    suite = unittest.TestSuite()
+    for klass in test_classes:
+        sub = unittest.makeSuite(klass)
+        suite.addTest(sub)
+    return suite
+
+if __name__ == "__main__":
+    unittest.main(defaultTest='test_suite')

lib/python/ZEO/tests/testConnection.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Test setup for ZEO connection logic.
+
+The actual tests are in ConnectionTests.py; this file provides the
+platform-dependent scaffolding.
+"""
+
+# System imports
+import unittest
+# Import the actual test class
+from ZEO.tests import ConnectionTests, InvalidationTests
+
+
+class FileStorageConfig:
+    def getConfig(self, path, create, read_only):
+        return """\
+        <filestorage 1>
+        path %s
+        create %s
+        read-only %s
+        </filestorage>""" % (path,
+                             create and 'yes' or 'no',
+                             read_only and 'yes' or 'no')
+
+class BerkeleyStorageConfig:
+    def getConfig(self, path, create, read_only):
+        return """\
+        <fullstorage 1>
+        envdir %s
+        read-only %s
+        </fullstorage>""" % (path, read_only and "yes" or "no")
+
+class MappingStorageConfig:
+    def getConfig(self, path, create, read_only):
+        return """<mappingstorage 1/>"""
+
+
+class FileStorageConnectionTests(
+    FileStorageConfig,
+    ConnectionTests.ConnectionTests,
+    InvalidationTests.InvalidationTests
+    ):
+    """FileStorage-specific connection tests."""
+    level = 2
+
+class FileStorageReconnectionTests(
+    FileStorageConfig,
+    ConnectionTests.ReconnectionTests,
+    ):
+    """FileStorage-specific re-connection tests."""
+    # Run this at level 1 because MappingStorage can't do reconnection tests
+    level = 1
+
+class FileStorageInvqTests(
+    FileStorageConfig,
+    ConnectionTests.InvqTests
+    ):
+    """FileStorage-specific invalidation queue tests."""
+    level = 1
+
+class FileStorageTimeoutTests(
+    FileStorageConfig,
+    ConnectionTests.TimeoutTests
+    ):
+    level = 2
+
+class BDBConnectionTests(
+    BerkeleyStorageConfig,
+    ConnectionTests.ConnectionTests,
+    InvalidationTests.InvalidationTests
+    ):
+    """Berkeley storage connection tests."""
+    level = 2
+
+class BDBReconnectionTests(
+    BerkeleyStorageConfig,
+    ConnectionTests.ReconnectionTests
+    ):
+    """Berkeley storage re-connection tests."""
+    level = 2
+
+class BDBInvqTests(
+    BerkeleyStorageConfig,
+    ConnectionTests.InvqTests
+    ):
+    """Berkeley storage invalidation queue tests."""
+    level = 2
+
+class BDBTimeoutTests(
+    BerkeleyStorageConfig,
+    ConnectionTests.TimeoutTests
+    ):
+    level = 2
+
+
+class MappingStorageConnectionTests(
+    MappingStorageConfig,
+    ConnectionTests.ConnectionTests
+    ):
+    """Mapping storage connection tests."""
+    level = 1
+
+# The ReconnectionTests can't work with MappingStorage because it's only an
+# in-memory storage and has no persistent state.
+
+class MappingStorageTimeoutTests(
+    MappingStorageConfig,
+    ConnectionTests.TimeoutTests
+    ):
+    level = 1
+
+
+
+test_classes = [FileStorageConnectionTests,
+                FileStorageReconnectionTests,
+                FileStorageInvqTests,
+                FileStorageTimeoutTests,
+                MappingStorageConnectionTests,
+                MappingStorageTimeoutTests]
+
+def test_suite():
+    suite = unittest.TestSuite()
+    for klass in test_classes:
+        sub = unittest.makeSuite(klass, 'check')
+        suite.addTest(sub)
+    return suite
+
+
+if __name__ == "__main__":
+    unittest.main(defaultTest='test_suite')

lib/python/ZEO/tests/testMonitor.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Test that the monitor produce sensible results.
+
+$Id: testMonitor.py,v 1.9 2004/02/27 00:31:52 faassen Exp $
+"""
+
+import socket
+import unittest
+
+from ZEO.tests.ConnectionTests import CommonSetupTearDown
+from ZEO.monitor import StorageStats
+
+class MonitorTests(CommonSetupTearDown):
+
+    monitor = 1
+
+    def get_monitor_output(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.connect(('localhost', 42000))
+        L = []
+        while 1:
+            buf = s.recv(8192)
+            if buf:
+                L.append(buf)
+            else:
+                break
+        s.close()
+        return "".join(L)
+
+    def parse(self, s):
+        # Return a list of StorageStats, one for each storage.
+        lines = s.split("\n")
+        self.assert_(lines[0].startswith("ZEO monitor server"))
+        # lines[1] is a date
+
+        # Break up rest of lines into sections starting with Storage:
+        # and ending with a blank line.
+        sections = []
+        cur = None
+        for line in lines[2:]:
+            if line.startswith("Storage:"):
+                cur = [line]
+            elif line:
+                cur.append(line)
+            else:
+                if cur is not None:
+                    sections.append(cur)
+                    cur = None
+        assert cur is None # bug in the test code if this fails
+
+        d = {}
+        for sect in sections:
+            hdr = sect[0]
+            key, value = hdr.split(":")
+            storage = int(value)
+            s = d[storage] = StorageStats()
+            s.parse("\n".join(sect[1:]))
+
+        return d
+
+    def getConfig(self, path, create, read_only):
+        return """<mappingstorage 1/>"""
+
+    def testMonitor(self):
+        # just open a client to know that the server is up and running
+        # XXX should put this in setUp
+        self.storage = self.openClientStorage()
+        s = self.get_monitor_output()
+        self.storage.close()
+        self.assert_(s.find("monitor") != -1)
+        d = self.parse(s)
+        stats = d[1]
+        self.assertEqual(stats.clients, 1)
+        self.assertEqual(stats.commits, 0)
+
+def test_suite():
+    return unittest.makeSuite(MonitorTests)

lib/python/ZEO/tests/testTransactionBuffer.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import random
+import unittest
+
+from ZEO.TransactionBuffer import TransactionBuffer
+
+def random_string(size):
+    """Return a random string of size size."""
+    l = [chr(random.randrange(256)) for i in range(size)]
+    return "".join(l)
+
+def new_store_data():
+    """Return arbitrary data to use as argument to store() method."""
+    return random_string(8), '', random_string(random.randrange(1000))
+
+def new_invalidate_data():
+    """Return arbitrary data to use as argument to invalidate() method."""
+    return random_string(8), ''
+
+class TransBufTests(unittest.TestCase):
+
+    def checkTypicalUsage(self):
+        tbuf = TransactionBuffer()
+        tbuf.store(*new_store_data())
+        tbuf.invalidate(*new_invalidate_data())
+        for o in tbuf:
+            pass
+
+    def doUpdates(self, tbuf):
+        data = []
+        for i in range(10):
+            d = new_store_data()
+            tbuf.store(*d)
+            data.append(d)
+            d = new_invalidate_data()
+            tbuf.invalidate(*d)
+            data.append(d)
+
+        for i, x in enumerate(tbuf):
+            if x[2] is None:
+                # the tbuf add a dummy None to invalidates
+                x = x[:2]
+            self.assertEqual(x, data[i])
+
+    def checkOrderPreserved(self):
+        tbuf = TransactionBuffer()
+        self.doUpdates(tbuf)
+
+    def checkReusable(self):
+        tbuf = TransactionBuffer()
+        self.doUpdates(tbuf)
+        tbuf.clear()
+        self.doUpdates(tbuf)
+        tbuf.clear()
+        self.doUpdates(tbuf)
+
+def test_suite():
+    return unittest.makeSuite(TransBufTests, 'check')

lib/python/ZEO/tests/testZEO.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Test suite for ZEO based on ZODB.tests."""
+
+# System imports
+import os
+import random
+import socket
+import asyncore
+import tempfile
+import unittest
+import logging
+
+# ZODB test support
+import ZODB
+from ZODB.tests.MinPO import MinPO
+from ZODB.tests.StorageTestBase import zodb_unpickle
+
+# ZODB test mixin classes
+from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \
+     TransactionalUndoStorage, TransactionalUndoVersionStorage, \
+     PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \
+     MTStorage, ReadOnlyStorage
+
+from ZEO.ClientStorage import ClientStorage
+from ZEO.tests import forker, Cache, CommitLockTests, ThreadTests
+
+logger = logging.getLogger('ZEO.tests.testZEO')
+
+class DummyDB:
+    def invalidate(self, *args):
+        pass
+
+
+class MiscZEOTests:
+    """ZEO tests that don't fit in elsewhere."""
+
+    def checkLargeUpdate(self):
+        obj = MinPO("X" * (10 * 128 * 1024))
+        self._dostore(data=obj)
+
+    def checkZEOInvalidation(self):
+        addr = self._storage._addr
+        storage2 = ClientStorage(addr, wait=1, min_disconnect_poll=0.1)
+        try:
+            oid = self._storage.new_oid()
+            ob = MinPO('first')
+            revid1 = self._dostore(oid, data=ob)
+            data, serial = storage2.load(oid, '')
+            self.assertEqual(zodb_unpickle(data), MinPO('first'))
+            self.assertEqual(serial, revid1)
+            revid2 = self._dostore(oid, data=MinPO('second'), revid=revid1)
+            for n in range(3):
+                # Let the server and client talk for a moment.
+                # Is there a better way to do this?
+                asyncore.poll(0.1)
+            data, serial = storage2.load(oid, '')
+            self.assertEqual(zodb_unpickle(data), MinPO('second'),
+                             'Invalidation message was not sent!')
+            self.assertEqual(serial, revid2)
+        finally:
+            storage2.close()
+
+def get_port():
+    """Return a port that is not in use.
+
+    Checks if a port is in use by trying to connect to it.  Assumes it
+    is not in use if connect raises an exception.
+
+    Raises RuntimeError after 10 tries.
+    """
+    for i in range(10):
+        port = random.randrange(20000, 30000)
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            try:
+                s.connect(('localhost', port))
+            except socket.error:
+                # XXX check value of error?
+                return port
+        finally:
+            s.close()
+    raise RuntimeError, "Can't find port"
+
+class GenericTests(
+    # Base class for all ZODB tests
+    StorageTestBase.StorageTestBase,
+    # ZODB test mixin classes (in the same order as imported)
+    BasicStorage.BasicStorage,
+    PackableStorage.PackableStorage,
+    Synchronization.SynchronizedStorage,
+    MTStorage.MTStorage,
+    ReadOnlyStorage.ReadOnlyStorage,
+    # ZEO test mixin classes (in the same order as imported)
+    CommitLockTests.CommitLockVoteTests,
+    ThreadTests.ThreadTests,
+    # Locally defined (see above)
+    MiscZEOTests
+    ):
+
+    """Combine tests from various origins in one class."""
+
+    def setUp(self):
+        logger.info("setUp() %s", self.id()) # XXX is this really needed?
+        port = get_port()
+        zconf = forker.ZEOConfig(('', port))
+        zport, adminaddr, pid, path = forker.start_zeo_server(self.getConfig(),
+                                                              zconf, port)
+        self._pids = [pid]
+        self._servers = [adminaddr]
+        self._conf_path = path
+        self._storage = ClientStorage(zport, '1', cache_size=20000000,
+                                      min_disconnect_poll=0.5, wait=1,
+                                      wait_timeout=60)
+        self._storage.registerDB(DummyDB(), None)
+
+    def tearDown(self):
+        self._storage.close()
+        os.remove(self._conf_path)
+        for server in self._servers:
+            forker.shutdown_zeo_server(server)
+        if hasattr(os, 'waitpid'):
+            # Not in Windows Python until 2.3
+            for pid in self._pids:
+                os.waitpid(pid, 0)
+
+    def open(self, read_only=0):
+        # XXX Needed to support ReadOnlyStorage tests.  Ought to be a
+        # cleaner way.
+        addr = self._storage._addr
+        self._storage.close()
+        self._storage = ClientStorage(addr, read_only=read_only, wait=1)
+
+    def checkWriteMethods(self):
+        # ReadOnlyStorage defines checkWriteMethods.  The decision
+        # about where to raise the read-only error was changed after
+        # Zope 2.5 was released.  So this test needs to detect Zope
+        # of the 2.5 vintage and skip the test.
+
+        # The __version__ attribute was not present in Zope 2.5.
+        if hasattr(ZODB, "__version__"):
+            ReadOnlyStorage.ReadOnlyStorage.checkWriteMethods(self)
+
+    def checkSortKey(self):
+        key = '%s:%s' % (self._storage._storage, self._storage._server_addr)
+        self.assertEqual(self._storage.sortKey(), key)
+
+class FullGenericTests(
+    GenericTests,
+    Cache.StorageWithCache,
+    Cache.TransUndoStorageWithCache,
+    CommitLockTests.CommitLockUndoTests,
+    ConflictResolution.ConflictResolvingStorage,
+    ConflictResolution.ConflictResolvingTransUndoStorage,
+    PackableStorage.PackableUndoStorage,
+    RevisionStorage.RevisionStorage,
+    TransactionalUndoStorage.TransactionalUndoStorage,
+    TransactionalUndoVersionStorage.TransactionalUndoVersionStorage,
+    VersionStorage.VersionStorage,
+    ):
+    """Extend GenericTests with tests that MappingStorage can't pass."""
+
+class FileStorageTests(FullGenericTests):
+    """Test ZEO backed by a FileStorage."""
+    level = 2
+
+    def getConfig(self):
+        filename = self.__fs_base = tempfile.mktemp()
+        return """\
+        <filestorage 1>
+        path %s
+        </filestorage>
+        """ % filename
+
+class BDBTests(FullGenericTests):
+    """ZEO backed by a Berkeley full storage."""
+    level = 2
+
+    def getConfig(self):
+        self._envdir = tempfile.mktemp()
+        return """\
+        <fullstorage 1>
+        envdir %s
+        </fullstorage>
+        """ % self._envdir
+
+class MappingStorageTests(GenericTests):
+    """ZEO backed by a Mapping storage."""
+
+    def getConfig(self):
+        return """<mappingstorage 1/>"""
+
+    # XXX There are still a bunch of tests that fail.  Are there
+    # still test classes in GenericTests that shouldn't be there?
+    # XXX Is the above comment still relevant?
+
+test_classes = [FileStorageTests, MappingStorageTests]
+
+def test_suite():
+    suite = unittest.TestSuite()
+    for klass in test_classes:
+        sub = unittest.makeSuite(klass, "check")
+        suite.addTest(sub)
+    return suite
+
+
+if __name__ == "__main__":
+    unittest.main(defaultTest="test_suite")

lib/python/ZEO/tests/testZEOOptions.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+"""Test suite for ZEO.runzeo.ZEOOptions."""
+
+import os
+import sys
+import tempfile
+import unittest
+
+import ZODB.config
+
+from ZEO.runzeo import ZEOOptions
+from zdaemon.tests.testzdoptions import TestZDOptions
+
+# When a hostname isn't specified in an address, ZConfig supplies a
+# platform-dependent default value.
+DEFAULT_HOSTNAME = ''
+if sys.platform in ['win32',]:
+    DEFAULT_HOSTNAME = 'localhost'
+
+class TestZEOOptions(TestZDOptions):
+
+    OptionsClass = ZEOOptions
+
+    input_args = ["-f", "Data.fs", "-a", "5555"]
+    output_opts = [("-f", "Data.fs"), ("-a", "5555")]
+    output_args = []
+
+    configdata = """
+        <zeo>
+          address 5555
+        </zeo>
+        <filestorage fs>
+          path Data.fs
+        </filestorage>
+        """
+
+    def setUp(self):
+        self.tempfilename = tempfile.mktemp()
+        f = open(self.tempfilename, "w")
+        f.write(self.configdata)
+        f.close()
+
+    def tearDown(self):
+        try:
+            os.remove(self.tempfilename)
+        except os.error:
+            pass
+
+    def test_configure(self):
+        # Hide the base class test_configure
+        pass
+
+    def test_defaults_with_schema(self):
+        options = self.OptionsClass()
+        options.realize(["-C", self.tempfilename])
+        self.assertEqual(options.address, (DEFAULT_HOSTNAME, 5555))
+        self.assertEqual(len(options.storages), 1)
+        opener = options.storages[0]
+        self.assertEqual(opener.name, "fs")
+        self.assertEqual(opener.__class__, ZODB.config.FileStorage)
+        self.assertEqual(options.read_only, 0)
+        self.assertEqual(options.transaction_timeout, None)
+        self.assertEqual(options.invalidation_queue_size, 100)
+
+    def test_defaults_without_schema(self):
+        options = self.OptionsClass()
+        options.realize(["-a", "5555", "-f", "Data.fs"])
+        self.assertEqual(options.address, (DEFAULT_HOSTNAME, 5555))
+        self.assertEqual(len(options.storages), 1)
+        opener = options.storages[0]
+        self.assertEqual(opener.name, "1")
+        self.assertEqual(opener.__class__, ZODB.config.FileStorage)
+        self.assertEqual(opener.config.path, "Data.fs")
+        self.assertEqual(options.read_only, 0)
+        self.assertEqual(options.transaction_timeout, None)
+        self.assertEqual(options.invalidation_queue_size, 100)
+
+    def test_commandline_overrides(self):
+        options = self.OptionsClass()
+        options.realize(["-C", self.tempfilename,
+                         "-a", "6666", "-f", "Wisdom.fs"])
+        self.assertEqual(options.address, (DEFAULT_HOSTNAME, 6666))
+        self.assertEqual(len(options.storages), 1)
+        opener = options.storages[0]
+        self.assertEqual(opener.__class__, ZODB.config.FileStorage)
+        self.assertEqual(opener.config.path, "Wisdom.fs")
+        self.assertEqual(options.read_only, 0)
+        self.assertEqual(options.transaction_timeout, None)
+        self.assertEqual(options.invalidation_queue_size, 100)
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    for cls in [TestZEOOptions]:
+        suite.addTest(unittest.makeSuite(cls))
+    return suite
+
+if __name__ == "__main__":
+    unittest.main(defaultTest='test_suite')

lib/python/ZEO/tests/test_cache.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Basic unit tests for a multi-version client cache."""
+
+import os
+import tempfile
+import unittest
+
+import ZEO.cache
+from ZODB.utils import p64
+
+n1 = p64(1)
+n2 = p64(2)
+n3 = p64(3)
+n4 = p64(4)
+n5 = p64(5)
+
+class CacheTests(unittest.TestCase):
+
+    def setUp(self):
+        self.cache = ZEO.cache.ClientCache()
+        self.cache.open()
+
+    def tearDown(self):
+        if self.cache.path:
+            os.remove(self.cache.path)
+
+    def testLastTid(self):
+        self.assertEqual(self.cache.getLastTid(), None)
+        self.cache.setLastTid(n2)
+        self.assertEqual(self.cache.getLastTid(), n2)
+        self.cache.invalidate(None, "", n1)
+        self.assertEqual(self.cache.getLastTid(), n2)
+        self.cache.invalidate(None, "", n3)
+        self.assertEqual(self.cache.getLastTid(), n3)
+        self.assertRaises(ValueError, self.cache.setLastTid, n2)
+
+    def testLoad(self):
+        data1 = "data for n1"
+        self.assertEqual(self.cache.load(n1, ""), None)
+        self.assertEqual(self.cache.load(n1, "version"), None)
+        self.cache.store(n1, "", n3, None, data1)
+        self.assertEqual(self.cache.load(n1, ""), (data1, n3, ""))
+        # The cache doesn't know whether version exists, because it
+        # only has non-version data.
+        self.assertEqual(self.cache.load(n1, "version"), None)
+        self.assertEqual(self.cache.modifiedInVersion(n1), None)
+
+    def testInvalidate(self):
+        data1 = "data for n1"
+        self.cache.store(n1, "", n3, None, data1)
+        self.cache.invalidate(n1, "", n4)
+        self.cache.invalidate(n2, "", n2)
+        self.assertEqual(self.cache.load(n1, ""), None)
+        self.assertEqual(self.cache.loadBefore(n1, n4),
+                         (data1, n3, n4))
+
+    def testVersion(self):
+        data1 = "data for n1"
+        data1v = "data for n1 in version"
+        self.cache.store(n1, "version", n3, None, data1v)
+        self.assertEqual(self.cache.load(n1, ""), None)
+        self.assertEqual(self.cache.load(n1, "version"),
+                         (data1v, n3, "version"))
+        self.assertEqual(self.cache.load(n1, "random"), None)
+        self.assertEqual(self.cache.modifiedInVersion(n1), "version")
+        self.cache.invalidate(n1, "version", n4)
+        self.assertEqual(self.cache.load(n1, "version"), None)
+
+    def testNonCurrent(self):
+        data1 = "data for n1"
+        data2 = "data for n2"
+        self.cache.store(n1, "", n4, None, data1)
+        self.cache.store(n1, "", n2, n3, data2)
+        # can't say anything about state before n2
+        self.assertEqual(self.cache.loadBefore(n1, n2), None)
+        # n3 is the upper bound of non-current record n2
+        self.assertEqual(self.cache.loadBefore(n1, n3), (data2, n2, n3))
+        # no data for between n2 and n3
+        self.assertEqual(self.cache.loadBefore(n1, n4), None)
+        self.cache.invalidate(n1, "", n5)
+        self.assertEqual(self.cache.loadBefore(n1, n5), (data1, n4, n5))
+        self.assertEqual(self.cache.loadBefore(n2, n4), None)
+
+    def testException(self):
+        self.assertRaises(ValueError,
+                          self.cache.store,
+                          n1, "version", n2, n3, "data")
+        self.cache.store(n1, "", n2, None, "data")
+        self.assertRaises(ValueError,
+                          self.cache.store,
+                          n1, "", n3, None, "data")
+
+    def testEviction(self):
+        # Manually override the current maxsize
+        maxsize = self.cache.size = self.cache.fc.maxsize = 3395 # 1245
+        self.cache.fc = ZEO.cache.FileCache(3395, None, self.cache)
+
+        # Trivial test of eviction code.  Doesn't test non-current
+        # eviction.
+        data = ["z" * i for i in range(100)]
+        for i in range(50):
+            n = p64(i)
+            self.cache.store(n, "", n, None, data[i])
+            self.assertEquals(len(self.cache), i + 1)
+            self.assert_(self.cache.fc.currentsize < maxsize)
+        # The cache now uses 1225 bytes.  The next insert
+        # should delete some objects.
+        n = p64(50)
+        self.cache.store(n, "", n, None, data[51])
+        self.assert_(len(self.cache) < 51)
+        self.assert_(self.cache.fc.currentsize <= maxsize)
+
+        # XXX Need to make sure eviction of non-current data
+        # and of version data are handled correctly.
+
+    def testSerialization(self):
+        self.cache.store(n1, "", n2, None, "data for n1")
+        self.cache.store(n2, "version", n2, None, "version data for n2")
+        self.cache.store(n3, "", n3, n4, "non-current data for n3")
+        self.cache.store(n3, "", n4, n5, "more non-current data for n3")
+
+        path = tempfile.mktemp()
+        # Copy data from self.cache into path, reaching into the cache
+        # guts to make the copy.
+        dst = open(path, "wb+")
+        src = self.cache.fc.f
+        src.seek(0)
+        dst.write(src.read(self.cache.fc.maxsize))
+        dst.close()
+        copy = ZEO.cache.ClientCache(path)
+        copy.open()
+
+        # Verify that internals of both objects are the same.
+        # Could also test that external API produces the same results.
+        eq = self.assertEqual
+        eq(copy.tid, self.cache.tid)
+        eq(len(copy), len(self.cache))
+        eq(copy.version, self.cache.version)
+        eq(copy.current, self.cache.current)
+        eq(copy.noncurrent, self.cache.noncurrent)
+
+def test_suite():
+    return unittest.makeSuite(CacheTests)

lib/python/ZEO/tests/zeoserver.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Helper file used to launch a ZEO server cross platform"""
+
+import os
+import sys
+import time
+import errno
+import getopt
+import socket
+import signal
+import asyncore
+import threading
+import logging
+
+import ThreadedAsync.LoopCallback
+
+from ZEO.StorageServer import StorageServer
+from ZEO.runzeo import ZEOOptions
+
+
+def cleanup(storage):
+    # FileStorage and the Berkeley storages have this method, which deletes
+    # all files and directories used by the storage.  This prevents @-files
+    # from clogging up /tmp
+    try:
+        storage.cleanup()
+    except AttributeError:
+        pass
+
+logger = logging.getLogger('ZEO.tests.zeoserver')
+
+def log(label, msg, *args):
+    message = "(%s) %s" % (label, msg)
+    logger.debug(message, args)
+
+
+class ZEOTestServer(asyncore.dispatcher):
+    """A server for killing the whole process at the end of a test.
+
+    The first time we connect to this server, we write an ack character down
+    the socket.  The other end should block on a recv() of the socket so it
+    can guarantee the server has started up before continuing on.
+
+    The second connect to the port immediately exits the process, via
+    os._exit(), without writing data on the socket.  It does close and clean
+    up the storage first.  The other end will get the empty string from its
+    recv() which will be enough to tell it that the server has exited.
+
+    I think this should prevent us from ever getting a legitimate addr-in-use
+    error.
+    """
+    __super_init = asyncore.dispatcher.__init__
+
+    def __init__(self, addr, server, keep):
+        self.__super_init()
+        self._server = server
+        self._sockets = [self]
+        self._keep = keep
+        # Count down to zero, the number of connects
+        self._count = 1
+        self._label ='%d @ %s' % (os.getpid(), addr)
+        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        # Some ZEO tests attempt a quick start of the server using the same
+        # port so we have to set the reuse flag.
+        self.set_reuse_addr()
+        try:
+            self.bind(addr)
+        except:
+            # We really want to see these exceptions
+            import traceback
+            traceback.print_exc()
+            raise
+        self.listen(5)
+        self.log('bound and listening')
+
+    def log(self, msg, *args):
+        log(self._label, msg, *args)
+
+    def handle_accept(self):
+        sock, addr = self.accept()
+        self.log('in handle_accept()')
+        # When we're done with everything, close the storage.  Do not write
+        # the ack character until the storage is finished closing.
+        if self._count <= 0:
+            self.log('closing the storage')
+            self._server.close_server()
+            if not self._keep:
+                for storage in self._server.storages.values():
+                    cleanup(storage)
+            self.log('exiting')
+            # Close all the other sockets so that we don't have to wait
+            # for os._exit() to get to it before starting the next
+            # server process.
+            for s in self._sockets:
+                s.close()
+            # Now explicitly close the socket returned from accept(),
+            # since it didn't go through the wrapper.
+            sock.close()
+            os._exit(0)
+        self.log('continuing')
+        sock.send('X')
+        self._count -= 1
+
+    def register_socket(self, sock):
+        # Register a socket to be closed when server shutsdown.
+        self._sockets.append(sock)
+
+class Suicide(threading.Thread):
+    def __init__(self, addr):
+        threading.Thread.__init__(self)
+        self._adminaddr = addr
+
+    def run(self):
+        # If this process doesn't exit in 330 seconds, commit suicide.
+        # The client threads in the ConcurrentUpdate tests will run for
+        # as long as 300 seconds.  Set this timeout to 330 to minimize
+        # chance that the server gives up before the clients.
+        time.sleep(330)
+        log(str(os.getpid()), "suicide thread invoking shutdown")
+
+        # If the server hasn't shut down yet, the client may not be
+        # able to connect to it.  If so, try to kill the process to
+        # force it to shutdown.
+        if hasattr(os, "kill"):
+            os.kill(pid, signal.SIGTERM)
+            time.sleep(5)
+            os.kill(pid, signal.SIGKILL)
+        else:
+            from ZEO.tests.forker import shutdown_zeo_server
+            # XXX If the -k option was given to zeoserver, then the
+            # process will go away but the temp files won't get
+            # cleaned up.
+            shutdown_zeo_server(self._adminaddr)
+
+
+def main():
+    global pid
+    pid = os.getpid()
+    label = str(pid)
+    log(label, "starting")
+
+    # We don't do much sanity checking of the arguments, since if we get it
+    # wrong, it's a bug in the test suite.
+    keep = 0
+    configfile = None
+    # Parse the arguments and let getopt.error percolate
+    opts, args = getopt.getopt(sys.argv[1:], 'kC:')
+    for opt, arg in opts:
+        if opt == '-k':
+            keep = 1
+        elif opt == '-C':
+            configfile = arg
+
+    zo = ZEOOptions()
+    zo.realize(["-C", configfile])
+    zeo_port = int(zo.address[1])
+
+    # XXX a hack
+    if zo.auth_protocol == "plaintext":
+        import ZEO.tests.auth_plaintext
+
+    # Open the config file and let ZConfig parse the data there.  Then remove
+    # the config file, otherwise we'll leave turds.
+    # The rest of the args are hostname, portnum
+    test_port = zeo_port + 1
+    test_addr = ('localhost', test_port)
+    addr = ('localhost', zeo_port)
+    log(label, 'creating the storage server')
+    storage = zo.storages[0].open()
+    mon_addr = None
+    if zo.monitor_address:
+        mon_addr = zo.monitor_address
+    server = StorageServer(
+        zo.address,
+        {"1": storage},
+        read_only=zo.read_only,
+        invalidation_queue_size=zo.invalidation_queue_size,
+        transaction_timeout=zo.transaction_timeout,
+        monitor_address=mon_addr,
+        auth_protocol=zo.auth_protocol,
+        auth_database=zo.auth_database,
+        auth_realm=zo.auth_realm)
+
+    try:
+        log(label, 'creating the test server, keep: %s', keep)
+        t = ZEOTestServer(test_addr, server, keep)
+    except socket.error, e:
+        if e[0] <> errno.EADDRINUSE: raise
+        log(label, 'addr in use, closing and exiting')
+        storage.close()
+        cleanup(storage)
+        sys.exit(2)
+
+    t.register_socket(server.dispatcher)
+    # Create daemon suicide thread
+    d = Suicide(test_addr)
+    d.setDaemon(1)
+    d.start()
+    # Loop for socket events
+    log(label, 'entering ThreadedAsync loop')
+    ThreadedAsync.LoopCallback.loop()
+
+
+if __name__ == '__main__':
+    main()

lib/python/ZEO/util.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Utilities for setting up the server environment."""
+
+import os
+
+def parentdir(p, n=1):
+    """Return the ancestor of p from n levels up."""
+    d = p
+    while n:
+        d = os.path.dirname(d)
+        if not d or d == '.':
+            d = os.getcwd()
+        n -= 1
+    return d
+
+class Environment:
+    """Determine location of the Data.fs & ZEO_SERVER.pid files.
+
+    Pass the argv[0] used to start ZEO to the constructor.
+
+    Use the zeo_pid and fs attributes to get the filenames.
+    """
+
+    def __init__(self, argv0):
+        v = os.environ.get("INSTANCE_HOME")
+        if v is None:
+            # looking for a Zope/var directory assuming that this code
+            # is installed in Zope/lib/python/ZEO
+            p = parentdir(argv0, 4)
+            if os.path.isdir(os.path.join(p, "var")):
+                v = p
+            else:
+                v = os.getcwd()
+        self.home = v
+        self.var = os.path.join(v, "var")
+        if not os.path.isdir(self.var):
+            self.var = self.home
+
+        pid = os.environ.get("ZEO_SERVER_PID")
+        if pid is None:
+            pid = os.path.join(self.var, "ZEO_SERVER.pid")
+
+        self.zeo_pid = pid
+        self.fs = os.path.join(self.var, "Data.fs")

lib/python/ZEO/version.txt

+2.3a3

lib/python/ZEO/zeoctl.py

+"""Wrapper script for zdctl.py that causes it to use the ZEO schema."""
+
+import os
+
+import ZEO
+import zdaemon.zdctl
+
+
+# Main program
+def main(args=None):
+    options = zdaemon.zdctl.ZDCtlOptions()
+    options.schemadir = os.path.dirname(ZEO.__file__)
+    options.schemafile = "zeoctl.xml"
+    zdaemon.zdctl.main(args, options)
+
+
+if __name__ == "__main__":
+    main()

lib/python/ZEO/zeoctl.xml

+<schema>
+
+  <description>
+    This schema describes the configuration of the ZEO storage server
+    controller.  It differs from the schema for the storage server
+    only in that the "runner" section is required.
+  </description>
+
+  <!-- Use the storage types defined by ZODB. -->
+  <import package="ZODB"/>
+
+  <!-- Use the ZEO server information structure. -->
+  <import package="ZEO"/>
+
+  <import package="ZConfig.components.logger"/>
+
+  <!-- runner control -->
+  <import package="zdaemon"/>
+
+
+  <section type="zeo" name="*" required="yes" attribute="zeo" />
+
+  <section type="runner" name="*" required="yes" attribute="runner" />
+
+  <multisection name="+" type="ZODB.storage"
+                attribute="storages"
+                required="yes" />
+
+  <section name="*" type="eventlog" attribute="eventlog" required="no" />
+
+</schema>

lib/python/ZEO/zeopasswd.py

+#!python
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Update a user's authentication tokens for a ZEO server.
+
+usage: python zeopasswd.py [options] username [password]
+
+Specify either a configuration file:
+
+    -C/--configuration -- ZConfig configuration file
+
+or the individual options:
+
+    -f/--filename -- authentication database filename
+    -p/--protocol -- authentication protocol name
+    -r/--realm -- authentication database realm
+
+Additional options:
+
+    -d/--delete -- delete user instead of updating password
+"""
+
+import getopt
+import getpass
+import sys
+import os
+
+import ZConfig
+import ZEO
+
+def usage(msg):
+    print __doc__
+    print msg
+    sys.exit(2)
+
+def options(args):
+    """Password-specific options loaded from regular ZEO config file."""
+    try:
+        opts, args = getopt.getopt(args, "dr:p:f:C:", ["configure=",
+                                                          "protocol=",
+                                                          "filename=",
+                                                          "realm"])
+    except getopt.error, msg:
+        usage(msg)
+    config = None
+    delete = 0
+    auth_protocol = None
+    auth_db = ""
+    auth_realm = None
+    for k, v in opts:
+        if k == '-C' or k == '--configure':
+            schemafile = os.path.join(os.path.dirname(ZEO.__file__),
+                                                     "schema.xml")
+            schema = ZConfig.loadSchema(schemafile)
+            config, nil = ZConfig.loadConfig(schema, v)
+        if k == '-d' or k == '--delete':
+            delete = 1
+        if k == '-p' or k == '--protocol':
+            auth_protocol = v
+        if k == '-f' or k == '--filename':
+            auth_db = v
+        if k == '-r' or k == '--realm':
+            auth_realm = v
+
+    if config is not None:
+        if auth_protocol or auth_db:
+            usage("Error: Conflicting options; use either -C *or* -p and -f")
+        auth_protocol = config.zeo.authentication_protocol
+        auth_db = config.zeo.authentication_database
+        auth_realm = config.zeo.authentication_realm
+    elif not (auth_protocol and auth_db):
+        usage("Error: Must specifiy configuration file or protocol and database")
+
+    password = None
+    if delete:
+        if not args:
+            usage("Error: Must specify a username to delete")
+        elif len(args) > 1:
+            usage("Error: Too many arguments")
+        username = args[0]
+    else:
+        if not args:
+            usage("Error: Must specify a username")
+        elif len(args) > 2:
+            usage("Error: Too many arguments")
+        elif len(args) == 1:
+            username = args[0]
+        else:
+            username, password = args
+
+    return auth_protocol, auth_db, auth_realm, delete, username, password
+
+def main(args=None, dbclass=None):
+    p, auth_db, auth_realm, delete, username, password = options(args)
+    if p is None:
+        usage("Error: configuration does not specify auth protocol")
+    if p == "digest":
+        from ZEO.auth.auth_digest import DigestDatabase as Database
+    elif p == "srp":
+        from ZEO.auth.auth_srp import SRPDatabase as Database
+    elif dbclass:
+        # dbclass is used for testing tests.auth_plaintext, see testAuth.py
+        Database = dbclass
+    else:
+        raise ValueError, "Unknown database type %r" % p
+    if auth_db is None:
+        usage("Error: configuration does not specify auth database")
+    db = Database(auth_db, auth_realm)
+    if delete:
+        db.del_user(username)
+    else:
+        if password is None:
+            password = getpass.getpass("Enter password: ")
+        db.add_user(username, password)
+    db.save()
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

lib/python/ZEO/zrpc/__init__.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+# zrpc is a package with the following modules
+# client -- manages connection creation to remote server
+# connection -- object dispatcher
+# log -- logging helper
+# error -- exceptions raised by zrpc
+# marshal -- internal, handles basic protocol issues
+# server -- manages incoming connections from remote clients
+# smac -- sized message async connections
+# trigger -- medusa's trigger
+
+# zrpc is not an advertised subpackage of ZEO; its interfaces are internal

lib/python/ZEO/zrpc/_hmac.py

+# This file is a slightly modified copy of Python 2.3's Lib/hmac.py.
+# This file is under the Python Software Foundation (PSF) license.
+
+"""HMAC (Keyed-Hashing for Message Authentication) Python module.
+
+Implements the HMAC algorithm as described by RFC 2104.
+"""
+
+def _strxor(s1, s2):
+    """Utility method. XOR the two strings s1 and s2 (must have same length).
+    """
+    return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
+
+# The size of the digests returned by HMAC depends on the underlying
+# hashing module used.
+digest_size = None
+
+class HMAC:
+    """RFC2104 HMAC class.
+
+    This supports the API for Cryptographic Hash Functions (PEP 247).
+    """
+
+    def __init__(self, key, msg = None, digestmod = None):
+        """Create a new HMAC object.
+
+        key:       key for the keyed hash object.
+        msg:       Initial input for the hash, if provided.
+        digestmod: A module supporting PEP 247. Defaults to the md5 module.
+        """
+        if digestmod is None:
+            import md5
+            digestmod = md5
+
+        self.digestmod = digestmod
+        self.outer = digestmod.new()
+        self.inner = digestmod.new()
+        # Python 2.1 and 2.2 differ about the correct spelling
+        try:
+            self.digest_size = digestmod.digestsize
+        except AttributeError:
+            self.digest_size = digestmod.digest_size
+
+        blocksize = 64
+        ipad = "\x36" * blocksize
+        opad = "\x5C" * blocksize
+
+        if len(key) > blocksize:
+            key = digestmod.new(key).digest()
+
+        key = key + chr(0) * (blocksize - len(key))
+        self.outer.update(_strxor(key, opad))
+        self.inner.update(_strxor(key, ipad))
+        if msg is not None:
+            self.update(msg)
+
+##    def clear(self):
+##        raise NotImplementedError, "clear() method not available in HMAC."
+
+    def update(self, msg):
+        """Update this hashing object with the string msg.
+        """
+        self.inner.update(msg)
+
+    def copy(self):
+        """Return a separate copy of this hashing object.
+
+        An update to this copy won't affect the original object.
+        """
+        other = HMAC("")
+        other.digestmod = self.digestmod
+        other.inner = self.inner.copy()
+        other.outer = self.outer.copy()
+        return other
+
+    def digest(self):
+        """Return the hash value of this hashing object.
+
+        This returns a string containing 8-bit data.  The object is
+        not altered in any way by this function; you can continue
+        updating the object after calling this function.
+        """
+        h = self.outer.copy()
+        h.update(self.inner.digest())
+        return h.digest()
+
+    def hexdigest(self):
+        """Like digest(), but returns a string of hexadecimal digits instead.
+        """
+        return "".join([hex(ord(x))[2:].zfill(2)
+                        for x in tuple(self.digest())])
+
+def new(key, msg = None, digestmod = None):
+    """Create a new hashing object and return it.
+
+    key: The starting key for the hash.
+    msg: if available, will immediately be hashed into the object's starting
+    state.
+
+    You can now feed arbitrary strings into the object using its update()
+    method, and can ask for the hash value at any time by calling its digest()
+    method.
+    """
+    return HMAC(key, msg, digestmod)

lib/python/ZEO/zrpc/client.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import errno
+import select
+import socket
+import sys
+import threading
+import time
+import types
+import logging
+
+import ThreadedAsync
+
+from ZODB.POSException import ReadOnlyError
+from ZODB.loglevels import BLATHER
+
+from ZEO.zrpc.log import log
+from ZEO.zrpc.trigger import trigger
+from ZEO.zrpc.connection import ManagedConnection
+
+class ConnectionManager(object):
+    """Keeps a connection up over time"""
+
+    def __init__(self, addrs, client, tmin=1, tmax=180):
+        self.addrlist = self._parse_addrs(addrs)
+        self.client = client
+        self.tmin = tmin
+        self.tmax = tmax
+        self.cond = threading.Condition(threading.Lock())
+        self.connection = None # Protected by self.cond
+        self.closed = 0
+        # If thread is not None, then there is a helper thread
+        # attempting to connect.
+        self.thread = None # Protected by self.cond
+        self.trigger = None
+        self.thr_async = 0
+        ThreadedAsync.register_loop_callback(self.set_async)
+
+    def __repr__(self):
+        return "<%s for %s>" % (self.__class__.__name__, self.addrlist)
+
+    def _parse_addrs(self, addrs):
+        # Return a list of (addr_type, addr) pairs.
+
+        # For backwards compatibility (and simplicity?) the
+        # constructor accepts a single address in the addrs argument --
+        # a string for a Unix domain socket or a 2-tuple with a
+        # hostname and port.  It can also accept a list of such addresses.
+
+        addr_type = self._guess_type(addrs)
+        if addr_type is not None:
+            return [(addr_type, addrs)]
+        else:
+            addrlist = []
+            for addr in addrs:
+                addr_type = self._guess_type(addr)
+                if addr_type is None:
+                    raise ValueError, (
+                        "unknown address in list: %s" % repr(addr))
+                addrlist.append((addr_type, addr))
+            return addrlist
+
+    def _guess_type(self, addr):
+        if isinstance(addr, types.StringType):
+            return socket.AF_UNIX
+
+        if (len(addr) == 2
+            and isinstance(addr[0], types.StringType)
+            and isinstance(addr[1], types.IntType)):
+            return socket.AF_INET
+
+        # not anything I know about
+        return None
+
+    def close(self):
+        """Prevent ConnectionManager from opening new connections"""
+        self.closed = 1
+        ThreadedAsync.remove_loop_callback(self.set_async)
+        self.cond.acquire()
+        try:
+            t = self.thread
+            self.thread = None
+            conn = self.connection
+        finally:
+            self.cond.release()
+        if t is not None:
+            log("CM.close(): stopping and joining thread")
+            t.stop()
+            t.join(30)
+            if t.isAlive():
+                log("CM.close(): self.thread.join() timed out",
+                    level=logging.WARNING)
+        if conn is not None:
+            # This will call close_conn() below which clears self.connection
+            conn.close()
+        if self.trigger is not None:
+            self.trigger.close()
+            self.trigger = None
+        ThreadedAsync.remove_loop_callback(self.set_async)
+
+    def set_async(self, map):
+        # This is the callback registered with ThreadedAsync.  The
+        # callback might be called multiple times, so it shouldn't
+        # create a trigger every time and should never do anything
+        # after it's closed.
+
+        # It may be that the only case where it is called multiple
+        # times is in the test suite, where ThreadedAsync's loop can
+        # be started in a child process after a fork.  Regardless,
+        # it's good to be defensive.
+
+        # XXX need each connection started with async==0 to have a
+        # callback
+        log("CM.set_async(%s)" % repr(map), level=logging.DEBUG)
+        if not self.closed and self.trigger is None:
+            log("CM.set_async(): first call")
+            self.trigger = trigger()
+            self.thr_async = 1 # XXX needs to be set on the Connection
+
+    def attempt_connect(self):
+        """Attempt a connection to the server without blocking too long.
+
+        There isn't a crisp definition for too long.  When a
+        ClientStorage is created, it attempts to connect to the
+        server.  If the server isn't immediately available, it can
+        operate from the cache.  This method will start the background
+        connection thread and wait a little while to see if it
+        finishes quickly.
+        """
+
+        # XXX Will a single attempt take too long?
+        # XXX Answer: it depends -- normally, you'll connect or get a
+        # connection refused error very quickly.  Packet-eating
+        # firewalls and other mishaps may cause the connect to take a
+        # long time to time out though.  It's also possible that you
+        # connect quickly to a slow server, and the attempt includes
+        # at least one roundtrip to the server (the register() call).
+        # But that's as fast as you can expect it to be.
+        self.connect()
+        self.cond.acquire()
+        try:
+            t = self.thread
+            conn = self.connection
+        finally:
+            self.cond.release()
+        if t is not None and conn is None:
+            event = t.one_attempt
+            event.wait()
+            self.cond.acquire()
+            try:
+                conn = self.connection
+            finally:
+                self.cond.release()
+        return conn is not None
+
+    def connect(self, sync=0):
+        self.cond.acquire()
+        try:
+            if self.connection is not None:
+                return
+            t = self.thread
+            if t is None:
+                log("CM.connect(): starting ConnectThread")
+                self.thread = t = ConnectThread(self, self.client,
+                                                self.addrlist,
+                                                self.tmin, self.tmax)
+                t.setDaemon(1)
+                t.start()
+            if sync:
+                while self.connection is None:
+                    self.cond.wait(30)
+                    if self.connection is None:
+                        log("CM.connect(sync=1): still waiting...")
+        finally:
+            self.cond.release()
+        if sync:
+            assert self.connection is not None
+
+    def connect_done(self, conn, preferred):
+        # Called by ConnectWrapper.notify_client() after notifying the client
+        log("CM.connect_done(preferred=%s)" % preferred)
+        self.cond.acquire()
+        try:
+            self.connection = conn
+            if preferred:
+                self.thread = None
+            self.cond.notifyAll() # Wake up connect(sync=1)
+        finally:
+            self.cond.release()
+
+    def close_conn(self, conn):
+        # Called by the connection when it is closed
+        self.cond.acquire()
+        try:
+            if conn is not self.connection:
+                # Closing a non-current connection
+                log("CM.close_conn() non-current", level=BLATHER)
+                return
+            log("CM.close_conn()")
+            self.connection = None
+        finally:
+            self.cond.release()
+        self.client.notifyDisconnected()
+        if not self.closed:
+            self.connect()
+
+    def is_connected(self):
+        self.cond.acquire()
+        try:
+            return self.connection is not None
+        finally:
+            self.cond.release()
+
+# When trying to do a connect on a non-blocking socket, some outcomes
+# are expected.  Set _CONNECT_IN_PROGRESS to the errno value(s) expected
+# when an initial connect can't complete immediately.  Set _CONNECT_OK
+# to the errno value(s) expected if the connect succeeds *or* if it's
+# already connected (our code can attempt redundant connects).
+if hasattr(errno, "WSAEWOULDBLOCK"):    # Windows
+    # XXX The official Winsock docs claim that WSAEALREADY should be
+    # treated as yet another "in progress" indicator, but we've never
+    # seen this.
+    _CONNECT_IN_PROGRESS = (errno.WSAEWOULDBLOCK,)
+    # Win98: WSAEISCONN; Win2K: WSAEINVAL
+    _CONNECT_OK          = (0, errno.WSAEISCONN, errno.WSAEINVAL)
+else:                                   # Unix
+    _CONNECT_IN_PROGRESS = (errno.EINPROGRESS,)
+    _CONNECT_OK          = (0, errno.EISCONN)
+
+class ConnectThread(threading.Thread):
+    """Thread that tries to connect to server given one or more addresses.
+
+    The thread is passed a ConnectionManager and the manager's client
+    as arguments.  It calls testConnection() on the client when a
+    socket connects; that should return 1 or 0 indicating whether this
+    is a preferred or a fallback connection.  It may also raise an
+    exception, in which case the connection is abandoned.
+
+    The thread will continue to run, attempting connections, until a
+    preferred connection is seen and successfully handed over to the
+    manager and client.
+
+    As soon as testConnection() finds a preferred connection, or after
+    all sockets have been tried and at least one fallback connection
+    has been seen, notifyConnected(connection) is called on the client
+    and connect_done() on the manager.  If this was a preferred
+    connection, the thread then exits; otherwise, it keeps trying
+    until it gets a preferred connection, and then reconnects the
+    client using that connection.
+
+    """
+
+    __super_init = threading.Thread.__init__
+
+    # We don't expect clients to call any methods of this Thread other
+    # than close() and those defined by the Thread API.
+
+    def __init__(self, mgr, client, addrlist, tmin, tmax):
+        self.__super_init(name="Connect(%s)" % addrlist)
+        self.mgr = mgr
+        self.client = client
+        self.addrlist = addrlist
+        self.tmin = tmin
+        self.tmax = tmax
+        self.stopped = 0
+        self.one_attempt = threading.Event()
+        # A ConnectThread keeps track of whether it has finished a
+        # call to try_connecting().  This allows the ConnectionManager
+        # to make an attempt to connect right away, but not block for
+        # too long if the server isn't immediately available.
+
+    def stop(self):
+        self.stopped = 1
+
+    def run(self):
+        delay = self.tmin
+        success = 0
+        # Don't wait too long the first time.
+        # XXX make timeout configurable?
+        attempt_timeout = 5
+        while not self.stopped:
+            success = self.try_connecting(attempt_timeout)
+            if not self.one_attempt.isSet():
+                self.one_attempt.set()
+                attempt_timeout = 75
+            if success > 0:
+                break
+            time.sleep(delay)
+            if self.mgr.is_connected():
+                log("CT: still trying to replace fallback connection",
+                    level=logging.INFO)
+            delay = min(delay*2, self.tmax)
+        log("CT: exiting thread: %s" % self.getName())
+
+    def try_connecting(self, timeout):
+        """Try connecting to all self.addrlist addresses.
+
+        Return 1 if a preferred connection was found; 0 if no
+        connection was found; and -1 if a fallback connection was
+        found.
+
+        If no connection is found within timeout seconds, return 0.
+        """
+        log("CT: attempting to connect on %d sockets" % len(self.addrlist))
+        deadline = time.time() + timeout
+        wrappers = self._create_wrappers()
+        for wrap in wrappers.keys():
+            if wrap.state == "notified":
+                return 1
+        try:
+            if time.time() > deadline:
+                return 0
+            r = self._connect_wrappers(wrappers, deadline)
+            if r is not None:
+                return r
+            if time.time() > deadline:
+                return 0
+            r = self._fallback_wrappers(wrappers, deadline)
+            if r is not None:
+                return r
+            # Alas, no luck.
+            assert not wrappers
+        finally:
+            for wrap in wrappers.keys():
+                wrap.close()
+            del wrappers
+        return 0
+
+    def _create_wrappers(self):
+        # Create socket wrappers
+        wrappers = {}  # keys are active wrappers
+        for domain, addr in self.addrlist:
+            wrap = ConnectWrapper(domain, addr, self.mgr, self.client)
+            wrap.connect_procedure()
+            if wrap.state == "notified":
+                for w in wrappers.keys():
+                    w.close()
+                return {wrap: wrap}
+            if wrap.state != "closed":
+                wrappers[wrap] = wrap
+        return wrappers
+
+    def _connect_wrappers(self, wrappers, deadline):
+        # Next wait until they all actually connect (or fail)
+        # The deadline is necessary, because we'd wait forever if a
+        # sockets never connects or fails.
+        while wrappers:
+            if self.stopped:
+                for wrap in wrappers.keys():
+                    wrap.close()
+                return 0
+            # Select connecting wrappers
+            connecting = [wrap
+                          for wrap in wrappers.keys()
+                          if wrap.state == "connecting"]
+            if not connecting:
+                break
+            if time.time() > deadline:
+                break
+            try:
+                r, w, x = select.select([], connecting, connecting, 1.0)
+                log("CT: select() %d, %d, %d" % tuple(map(len, (r,w,x))))
+            except select.error, msg:
+                log("CT: select failed; msg=%s" % str(msg),
+                    level=logging.WARNING) # XXX Is this the right level?
+                continue
+            # Exceptable wrappers are in trouble; close these suckers
+            for wrap in x:
+                log("CT: closing troubled socket %s" % str(wrap.addr))
+                del wrappers[wrap]
+                wrap.close()
+            # Writable sockets are connected
+            for wrap in w:
+                wrap.connect_procedure()
+                if wrap.state == "notified":
+                    del wrappers[wrap] # Don't close this one
+                    for wrap in wrappers.keys():
+                        wrap.close()
+                    return 1
+                if wrap.state == "closed":
+                    del wrappers[wrap]
+
+    def _fallback_wrappers(self, wrappers, deadline):
+        # If we've got wrappers left at this point, they're fallback
+        # connections.  Try notifying them until one succeeds.
+        for wrap in wrappers.keys():
+            assert wrap.state == "tested" and wrap.preferred == 0
+            if self.mgr.is_connected():
+                wrap.close()
+            else:
+                wrap.notify_client()
+                if wrap.state == "notified":
+                    del wrappers[wrap] # Don't close this one
+                    for wrap in wrappers.keys():
+                        wrap.close()
+                    return -1
+            assert wrap.state == "closed"
+            del wrappers[wrap]
+
+            # XXX should check deadline
+
+
+class ConnectWrapper:
+    """An object that handles the connection procedure for one socket.
+
+    This is a little state machine with states:
+        closed
+        opened
+        connecting
+        connected
+        tested
+        notified
+    """
+
+    def __init__(self, domain, addr, mgr, client):
+        """Store arguments and create non-blocking socket."""
+        self.domain = domain
+        self.addr = addr
+        self.mgr = mgr
+        self.client = client
+        # These attributes are part of the interface
+        self.state = "closed"
+        self.sock = None
+        self.conn = None
+        self.preferred = 0
+        log("CW: attempt to connect to %s" % repr(addr))
+        try:
+            self.sock = socket.socket(domain, socket.SOCK_STREAM)
+        except socket.error, err:
+            log("CW: can't create socket, domain=%s: %s" % (domain, err),
+                level=logging.ERROR)
+            self.close()
+            return
+        self.sock.setblocking(0)
+        self.state = "opened"
+
+    def connect_procedure(self):
+        """Call sock.connect_ex(addr) and interpret result."""
+        if self.state in ("opened", "connecting"):
+            try:
+                err = self.sock.connect_ex(self.addr)
+            except socket.error, msg:
+                log("CW: connect_ex(%r) failed: %s" % (self.addr, msg),
+                    level=logging.ERROR)
+                self.close()
+                return
+            log("CW: connect_ex(%s) returned %s" %
+                (self.addr, errno.errorcode.get(err) or str(err)))
+            if err in _CONNECT_IN_PROGRESS:
+                self.state = "connecting"
+                return
+            if err not in _CONNECT_OK:
+                log("CW: error connecting to %s: %s" %
+                    (self.addr, errno.errorcode.get(err) or str(err)),
+                    level=logging.WARNING)
+                self.close()
+                return
+            self.state = "connected"
+        if self.state == "connected":
+            self.test_connection()
+
+    def test_connection(self):
+        """Establish and test a connection at the zrpc level.
+
+        Call the client's testConnection(), giving the client a chance
+        to do app-level check of the connection.
+        """
+        self.conn = ManagedConnection(self.sock, self.addr,
+                                      self.client, self.mgr)
+        self.sock = None # The socket is now owned by the connection
+        try:
+            self.preferred = self.client.testConnection(self.conn)
+            self.state = "tested"
+        except ReadOnlyError:
+            log("CW: ReadOnlyError in testConnection (%s)" % repr(self.addr))
+            self.close()
+            return
+        except:
+            log("CW: error in testConnection (%s)" % repr(self.addr),
+                level=logging.ERROR, exc_info=True)
+            self.close()
+            return
+        if self.preferred:
+            self.notify_client()
+
+    def notify_client(self):
+        """Call the client's notifyConnected().
+
+        If this succeeds, call the manager's connect_done().
+
+        If the client is already connected, we assume it's a fallback
+        connection, and the new connection must be a preferred
+        connection.  The client will close the old connection.
+        """
+        try:
+            self.client.notifyConnected(self.conn)
+        except:
+            log("CW: error in notifyConnected (%s)" % repr(self.addr),
+                level=logging.ERROR, exc_info=True)
+            self.close()
+            return
+        self.state = "notified"
+        self.mgr.connect_done(self.conn, self.preferred)
+
+    def close(self):
+        """Close the socket and reset everything."""
+        self.state = "closed"
+        self.mgr = self.client = None
+        self.preferred = 0
+        if self.conn is not None:
+            # Closing the ZRPC connection will eventually close the
+            # socket, somewhere in asyncore.
+            # XXX Why do we care? --Guido
+            self.conn.close()
+            self.conn = None
+        if self.sock is not None:
+            self.sock.close()
+            self.sock = None
+
+    def fileno(self):
+        return self.sock.fileno()

lib/python/ZEO/zrpc/connection.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import asyncore
+import errno
+import select
+import sys
+import threading
+import types
+import logging
+
+import ThreadedAsync
+from ZEO.zrpc import smac
+from ZEO.zrpc.error import ZRPCError, DisconnectedError
+from ZEO.zrpc.marshal import Marshaller
+from ZEO.zrpc.trigger import trigger
+from ZEO.zrpc.log import short_repr, log
+from ZODB.loglevels import BLATHER, TRACE
+
+REPLY = ".reply" # message name used for replies
+ASYNC = 1
+
+class Delay:
+    """Used to delay response to client for synchronous calls.
+
+    When a synchronous call is made and the original handler returns
+    without handling the call, it returns a Delay object that prevents
+    the mainloop from sending a response.
+    """
+
+    def set_sender(self, msgid, send_reply, return_error):
+        self.msgid = msgid
+        self.send_reply = send_reply
+        self.return_error = return_error
+
+    def reply(self, obj):
+        self.send_reply(self.msgid, obj)
+
+    def error(self, exc_info):
+        log("Error raised in delayed method", logging.ERROR, exc_info=True)
+        self.return_error(self.msgid, 0, *exc_info[:2])
+
+class MTDelay(Delay):
+
+    def __init__(self):
+        self.ready = threading.Event()
+
+    def set_sender(self, msgid, send_reply, return_error):
+        Delay.set_sender(self, msgid, send_reply, return_error)
+        self.ready.set()
+
+    def reply(self, obj):
+        self.ready.wait()
+        Delay.reply(self, obj)
+
+    def error(self, exc_info):
+        self.ready.wait()
+        Delay.error(self, exc_info)
+
+class Connection(smac.SizedMessageAsyncConnection, object):
+    """Dispatcher for RPC on object on both sides of socket.
+
+    The connection supports synchronous calls, which expect a return,
+    and asynchronous calls, which do not.
+
+    It uses the Marshaller class to handle encoding and decoding of
+    method calls and arguments.  Marshaller uses pickle to encode
+    arbitrary Python objects.  The code here doesn't ever see the wire
+    format.
+
+    A Connection is designed for use in a multithreaded application,
+    where a synchronous call must block until a response is ready.
+
+    A socket connection between a client and a server allows either
+    side to invoke methods on the other side.  The processes on each
+    end of the socket use a Connection object to manage communication.
+
+    The Connection deals with decoded RPC messages.  They are
+    represented as four-tuples containing: msgid, flags, method name,
+    and a tuple of method arguments.
+
+    The msgid starts at zero and is incremented by one each time a
+    method call message is sent.  Each side of the connection has a
+    separate msgid state.
+
+    When one side of the connection (the client) calls a method, it
+    sends a message with a new msgid.  The other side (the server),
+    replies with a message that has the same msgid, the string
+    ".reply" (the global variable REPLY) as the method name, and the
+    actual return value in the args position.  Note that each side of
+    the Connection can initiate a call, in which case it will be the
+    client for that particular call.
+
+    The protocol also supports asynchronous calls.  The client does
+    not wait for a return value for an asynchronous call.  The only
+    defined flag is ASYNC.  If a method call message has the ASYNC
+    flag set, the server will raise an exception.
+
+    If a method call raises an Exception, the exception is propagated
+    back to the client via the REPLY message.  The client side will
+    raise any exception it receives instead of returning the value to
+    the caller.
+    """
+
+    __super_init = smac.SizedMessageAsyncConnection.__init__
+    __super_close = smac.SizedMessageAsyncConnection.close
+    __super_setSessionKey = smac.SizedMessageAsyncConnection.setSessionKey
+
+    # Protocol variables:
+    #
+    # oldest_protocol_version -- the oldest protocol version we support
+    # protocol_version -- the newest protocol version we support; preferred
+
+    oldest_protocol_version = "Z200"
+    protocol_version = "Z201"
+
+    # Protocol history:
+    #
+    # Z200 -- Original ZEO 2.0 protocol
+    #
+    # Z201 -- Added invalidateTransaction() to client.
+    #         Renamed several client methods.
+    #         Added several sever methods:
+    #             lastTransaction()
+    #             getAuthProtocol() and scheme-specific authentication methods
+    #             getExtensionMethods().
+    #             getInvalidations().
+
+    def __init__(self, sock, addr, obj=None):
+        self.obj = None
+        self.marshal = Marshaller()
+        self.closed = False
+        self.msgid = 0
+        self.peer_protocol_version = None # Set in recv_handshake()
+        self.logger = logging.getLogger('ZEO.zrpc.Connection')
+        if isinstance(addr, types.TupleType):
+            self.log_label = "(%s:%d) " % addr
+        else:
+            self.log_label = "(%s) " % addr
+        self.__super_init(sock, addr)
+        # A Connection either uses asyncore directly or relies on an
+        # asyncore mainloop running in a separate thread.  If
+        # thr_async is true, then the mainloop is running in a
+        # separate thread.  If thr_async is true, then the asyncore
+        # trigger (self.trigger) is used to notify that thread of
+        # activity on the current thread.
+        self.thr_async = False
+        self.trigger = None
+        self._prepare_async()
+        # The singleton dict is used in synchronous mode when a method
+        # needs to call into asyncore to try to force some I/O to occur.
+        # The singleton dict is a socket map containing only this object.
+        self._singleton = {self._fileno: self}
+        # msgid_lock guards access to msgid
+        self.msgid_lock = threading.Lock()
+        # replies_cond is used to block when a synchronous call is
+        # waiting for a response
+        self.replies_cond = threading.Condition()
+        self.replies = {}
+        # waiting_for_reply is used internally to indicate whether
+        # a call is in progress.  setting a session key is deferred
+        # until after the call returns.
+        self.waiting_for_reply = False
+        self.delay_sesskey = None
+        self.register_object(obj)
+        self.handshake()
+
+    def __repr__(self):
+        return "<%s %s>" % (self.__class__.__name__, self.addr)
+
+    __str__ = __repr__ # Defeat asyncore's dreaded __getattr__
+
+    def log(self, message, level=BLATHER, exc_info=False):
+        self.logger.log(level, self.log_label + message, exc_info=exc_info)
+
+    def close(self):
+        if self.closed:
+            return
+        self._singleton.clear()
+        self.closed = True
+        self.close_trigger()
+        self.__super_close()
+
+    def close_trigger(self):
+        # Overridden by ManagedConnection
+        if self.trigger is not None:
+            self.trigger.close()
+
+    def register_object(self, obj):
+        """Register obj as the true object to invoke methods on."""
+        self.obj = obj
+
+    def handshake(self, proto=None):
+        # Overridden by ManagedConnection
+
+        # When a connection is created the first message sent is a
+        # 4-byte protocol version.  This mechanism should allow the
+        # protocol to evolve over time, and let servers handle clients
+        # using multiple versions of the protocol.
+
+        # The mechanism replaces the message_input() method for the
+        # first message received.
+
+        # The client sends the protocol version it is using.
+        self.message_input = self.recv_handshake
+        self.message_output(proto or self.protocol_version)
+
+    def recv_handshake(self, proto):
+        # Extended by ManagedConnection
+        del self.message_input
+        self.peer_protocol_version = proto
+        if self.oldest_protocol_version <= proto <= self.protocol_version:
+            self.log("received handshake %r" % proto, level=logging.INFO)
+        else:
+            self.log("bad handshake %s" % short_repr(proto),
+                     level=logging.ERROR)
+            raise ZRPCError("bad handshake %r" % proto)
+
+    def message_input(self, message):
+        """Decoding an incoming message and dispatch it"""
+        # If something goes wrong during decoding, the marshaller
+        # will raise an exception.  The exception will ultimately
+        # result in asycnore calling handle_error(), which will
+        # close the connection.
+        msgid, flags, name, args = self.marshal.decode(message)
+
+        if __debug__:
+            self.log("recv msg: %s, %s, %s, %s" % (msgid, flags, name,
+                                                   short_repr(args)),
+                     level=TRACE)
+        if name == REPLY:
+            self.handle_reply(msgid, flags, args)
+        else:
+            self.handle_request(msgid, flags, name, args)
+
+    def handle_reply(self, msgid, flags, args):
+        if __debug__:
+            self.log("recv reply: %s, %s, %s"
+                     % (msgid, flags, short_repr(args)), level=TRACE)
+        self.replies_cond.acquire()
+        try:
+            self.replies[msgid] = flags, args
+            self.replies_cond.notifyAll()
+        finally:
+            self.replies_cond.release()
+
+    def handle_request(self, msgid, flags, name, args):
+        if not self.check_method(name):
+            msg = "Invalid method name: %s on %s" % (name, repr(self.obj))
+            raise ZRPCError(msg)
+        if __debug__:
+            self.log("calling %s%s" % (name, short_repr(args)),
+                     level=logging.DEBUG)
+
+        meth = getattr(self.obj, name)
+        try:
+            self.waiting_for_reply = True
+            try:
+                ret = meth(*args)
+            finally:
+                self.waiting_for_reply = False
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except Exception, msg:
+            self.log("%s() raised exception: %s" % (name, msg), logging.INFO,
+                     exc_info=True)
+            error = sys.exc_info()[:2]
+            return self.return_error(msgid, flags, *error)
+
+        if flags & ASYNC:
+            if ret is not None:
+                raise ZRPCError("async method %s returned value %s" %
+                                (name, short_repr(ret)))
+        else:
+            if __debug__:
+                self.log("%s returns %s" % (name, short_repr(ret)),
+                         logging.DEBUG)
+            if isinstance(ret, Delay):
+                ret.set_sender(msgid, self.send_reply, self.return_error)
+            else:
+                self.send_reply(msgid, ret)
+
+        if self.delay_sesskey:
+            self.__super_setSessionKey(self.delay_sesskey)
+            self.delay_sesskey = None
+
+    def handle_error(self):
+        if sys.exc_info()[0] == SystemExit:
+            raise sys.exc_info()
+        self.log("Error caught in asyncore",
+                 level=logging.ERROR, exc_info=True)
+        self.close()
+
+    def check_method(self, name):
+        # XXX Is this sufficient "security" for now?
+        if name.startswith('_'):
+            return None
+        return hasattr(self.obj, name)
+
+    def send_reply(self, msgid, ret):
+        try:
+            msg = self.marshal.encode(msgid, 0, REPLY, ret)
+        except self.marshal.errors:
+            try:
+                r = short_repr(ret)
+            except:
+                r = "<unreprable>"
+            err = ZRPCError("Couldn't pickle return %.100s" % r)
+            msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err))
+        self.message_output(msg)
+        self.poll()
+
+    def return_error(self, msgid, flags, err_type, err_value):
+        if flags & ASYNC:
+            self.log("Asynchronous call raised exception: %s" % self,
+                     level=logging.ERROR, exc_info=True)
+            return
+        if type(err_value) is not types.InstanceType:
+            err_value = err_type, err_value
+
+        try:
+            msg = self.marshal.encode(msgid, 0, REPLY, (err_type, err_value))
+        except self.marshal.errors:
+            try:
+                r = short_repr(err_value)
+            except:
+                r = "<unreprable>"
+            err = ZRPCError("Couldn't pickle error %.100s" % r)
+            msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err))
+        self.message_output(msg)
+        self.poll()
+
+    def setSessionKey(self, key):
+        if self.waiting_for_reply:
+            self.delay_sesskey = key
+        else:
+            self.__super_setSessionKey(key)
+
+    # The next two public methods (call and callAsync) are used by
+    # clients to invoke methods on remote objects
+
+    def send_call(self, method, args, flags):
+        # send a message and return its msgid
+        self.msgid_lock.acquire()
+        try:
+            msgid = self.msgid
+            self.msgid = self.msgid + 1
+        finally:
+            self.msgid_lock.release()
+        if __debug__:
+            self.log("send msg: %d, %d, %s, ..." % (msgid, flags, method),
+                     level=TRACE)
+        buf = self.marshal.encode(msgid, flags, method, args)
+        self.message_output(buf)
+        return msgid
+
+    def call(self, method, *args):
+        if self.closed:
+            raise DisconnectedError()
+        msgid = self.send_call(method, args, 0)
+        r_flags, r_args = self.wait(msgid)
+        if (isinstance(r_args, types.TupleType) and len(r_args) > 1
+            and type(r_args[0]) == types.ClassType
+            and issubclass(r_args[0], Exception)):
+            inst = r_args[1]
+            raise inst # error raised by server
+        else:
+            return r_args
+
+    # For testing purposes, it is useful to begin a synchronous call
+    # but not block waiting for its response.  Since these methods are
+    # used for testing they can assume they are not in async mode and
+    # call asyncore.poll() directly to get the message out without
+    # also waiting for the reply.
+
+    def _deferred_call(self, method, *args):
+        if self.closed:
+            raise DisconnectedError()
+        msgid = self.send_call(method, args, 0)
+        asyncore.poll(0.01, self._singleton)
+        return msgid
+
+    def _deferred_wait(self, msgid):
+        r_flags, r_args = self.wait(msgid)
+        if (isinstance(r_args, types.TupleType)
+            and type(r_args[0]) == types.ClassType
+            and issubclass(r_args[0], Exception)):
+            inst = r_args[1]
+            raise inst # error raised by server
+        else:
+            return r_args
+
+    def callAsync(self, method, *args):
+        if self.closed:
+            raise DisconnectedError()
+        self.send_call(method, args, ASYNC)
+        self.poll()
+
+    def callAsyncNoPoll(self, method, *args):
+        # Like CallAsync but doesn't poll.  This exists so that we can
+        # send invalidations atomically to all clients without
+        # allowing any client to sneak in a load request.
+        if self.closed:
+            raise DisconnectedError()
+        self.send_call(method, args, ASYNC)
+
+    # handle IO, possibly in async mode
+
+    def _prepare_async(self):
+        self.thr_async = False
+        ThreadedAsync.register_loop_callback(self.set_async)
+        # XXX If we are not in async mode, this will cause dead
+        # Connections to be leaked.
+
+    def set_async(self, map):
+        self.trigger = trigger()
+        self.thr_async = True
+
+    def is_async(self):
+        # Overridden by ManagedConnection
+        if self.thr_async:
+            return 1
+        else:
+            return 0
+
+    def _pull_trigger(self, tryagain=10):
+        try:
+            self.trigger.pull_trigger()
+        except OSError:
+            self.trigger.close()
+            self.trigger = trigger()
+            if tryagain > 0:
+                self._pull_trigger(tryagain=tryagain-1)
+
+    def wait(self, msgid):
+        """Invoke asyncore mainloop and wait for reply."""
+        if __debug__:
+            self.log("wait(%d), async=%d" % (msgid, self.is_async()),
+                     level=TRACE)
+        if self.is_async():
+            self._pull_trigger()
+
+        # Delay used when we call asyncore.poll() directly.
+        # Start with a 1 msec delay, double until 1 sec.
+        delay = 0.001
+
+        self.replies_cond.acquire()
+        try:
+            while 1:
+                if self.closed:
+                    raise DisconnectedError()
+                reply = self.replies.get(msgid)
+                if reply is not None:
+                    del self.replies[msgid]
+                    if __debug__:
+                        self.log("wait(%d): reply=%s" %
+                                 (msgid, short_repr(reply)), level=TRACE)
+                    return reply
+                if self.is_async():
+                    self.replies_cond.wait(10.0)
+                else:
+                    self.replies_cond.release()
+                    try:
+                        try:
+                            if __debug__:
+                                self.log("wait(%d): asyncore.poll(%s)" %
+                                         (msgid, delay), level=TRACE)
+                            asyncore.poll(delay, self._singleton)
+                            if delay < 1.0:
+                                delay += delay
+                        except select.error, err:
+                            self.log("Closing.  asyncore.poll() raised %s."
+                                     % err, level=BLATHER)
+                            self.close()
+                    finally:
+                        self.replies_cond.acquire()
+        finally:
+            self.replies_cond.release()
+
+    def flush(self):
+        """Invoke poll() until the output buffer is empty."""
+        if __debug__:
+            self.log("flush")
+        while self.writable():
+            self.poll()
+
+    def poll(self):
+        """Invoke asyncore mainloop to get pending message out."""
+        if __debug__:
+            self.log("poll(), async=%d" % self.is_async(), level=TRACE)
+        if self.is_async():
+            self._pull_trigger()
+        else:
+            asyncore.poll(0.0, self._singleton)
+
+    def pending(self, timeout=0):
+        """Invoke mainloop until any pending messages are handled."""
+        if __debug__:
+            self.log("pending(), async=%d" % self.is_async(), level=TRACE)
+        if self.is_async():
+            return
+        # Inline the asyncore poll() function to know whether any input
+        # was actually read.  Repeat until no input is ready.
+
+        # Pending does reads and writes.  In the case of server
+        # startup, we may need to write out zeoVerify() messages.
+        # Always check for read status, but don't check for write status
+        # only there is output to do.  Only continue in this loop as
+        # long as there is data to read.
+        r = r_in = [self._fileno]
+        x_in = []
+        while r and not self.closed:
+            if self.writable():
+                w_in = [self._fileno]
+            else:
+                w_in = []
+            try:
+                r, w, x = select.select(r_in, w_in, x_in, timeout)
+            except select.error, err:
+                if err[0] == errno.EINTR:
+                    timeout = 0
+                    continue
+                else:
+                    raise
+            else:
+                # Make sure any subsequent select does not block.  The
+                # loop is only intended to make sure all incoming data is
+                # returned.
+
+                # XXX What if the server sends a lot of invalidations,
+                # such that pending never finishes?  Seems unlikely, but
+                # not impossible.
+                timeout = 0
+            if r:
+                try:
+                    self.handle_read_event()
+                except asyncore.ExitNow:
+                    raise
+                except:
+                    self.handle_error()
+            if w:
+                try:
+                    self.handle_write_event()
+                except asyncore.ExitNow:
+                    raise
+                except:
+                    self.handle_error()
+
+class ManagedServerConnection(Connection):
+    """Server-side Connection subclass."""
+    __super_init = Connection.__init__
+    __super_close = Connection.close
+
+    def __init__(self, sock, addr, obj, mgr):
+        self.mgr = mgr
+        self.__super_init(sock, addr, obj)
+        self.obj.notifyConnected(self)
+
+    def close(self):
+        self.obj.notifyDisconnected()
+        self.mgr.close_conn(self)
+        self.__super_close()
+
+class ManagedConnection(Connection):
+    """Client-side Connection subclass."""
+    __super_init = Connection.__init__
+    __super_close = Connection.close
+
+    def __init__(self, sock, addr, obj, mgr):
+        self.mgr = mgr
+        self.__super_init(sock, addr, obj)
+        self.check_mgr_async()
+
+    # PROTOCOL NEGOTIATION:
+    #
+    # The code implementing protocol version 2.0.0 (which is deployed
+    # in the field and cannot be changed) *only* talks to peers that
+    # send a handshake indicating protocol version 2.0.0.  In that
+    # version, both the client and the server immediately send out
+    # their protocol handshake when a connection is established,
+    # without waiting for their peer, and disconnect when a different
+    # handshake is receive.
+    #
+    # The new protocol uses this to enable new clients to talk to
+    # 2.0.0 servers: in the new protocol, the client waits until it
+    # receives the server's protocol handshake before sending its own
+    # handshake.  The client sends the lower of its own protocol
+    # version and the server protocol version, allowing it to talk to
+    # servers using later protocol versions (2.0.2 and higher) as
+    # well: the effective protocol used will be the lower of the
+    # client and server protocol.
+    #
+    # The ZEO modules ClientStorage and ServerStub have backwards
+    # compatibility code for dealing with the previous version of the
+    # protocol.  The client accept the old version of some messages,
+    # and will not send new messages when talking to an old server.
+    #
+    # As long as the client hasn't sent its handshake, it can't send
+    # anything else; output messages are queued during this time.
+    # (Output can happen because the connection testing machinery can
+    # start sending requests before the handshake is received.)
+    #
+    # UPGRADING FROM ZEO 2.0.0 TO NEWER VERSIONS:
+    #
+    # Because a new client can talk to an old server, but not vice
+    # versa, all clients should be upgraded before upgrading any
+    # servers.  Protocol upgrades beyond 2.0.1 will not have this
+    # restriction, because clients using protocol 2.0.1 or later can
+    # talk to both older and newer servers.
+    #
+    # No compatibility with protocol version 1 is provided.
+
+    def handshake(self):
+        self.message_input = self.recv_handshake
+        self.message_output = self.queue_output
+        self.output_queue = []
+        # The handshake is sent by recv_handshake() below
+
+    def queue_output(self, message):
+        self.output_queue.append(message)
+
+    def recv_handshake(self, proto):
+        del self.message_output
+        proto = min(proto, self.protocol_version)
+        Connection.recv_handshake(self, proto) # Raise error if wrong proto
+        self.message_output(proto)
+        queue = self.output_queue
+        del self.output_queue
+        for message in queue:
+            self.message_output(message)
+
+    # Defer the ThreadedAsync work to the manager.
+
+    def close_trigger(self):
+        # the manager should actually close the trigger
+        del self.trigger
+
+    def set_async(self, map):
+        pass
+
+    def _prepare_async(self):
+        # Don't do the register_loop_callback that the superclass does
+        pass
+
+    def check_mgr_async(self):
+        if not self.thr_async and self.mgr.thr_async:
+            assert self.mgr.trigger is not None, \
+                   "manager (%s) has no trigger" % self.mgr
+            self.thr_async = True
+            self.trigger = self.mgr.trigger
+            return 1
+        return 0
+
+    def is_async(self):
+        # XXX could the check_mgr_async() be avoided on each test?
+        if self.thr_async:
+            return 1
+        return self.check_mgr_async()
+
+    def close(self):
+        self.mgr.close_conn(self)
+        self.__super_close()

lib/python/ZEO/zrpc/error.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+from ZODB import POSException
+from ZEO.Exceptions import ClientDisconnected
+
+class ZRPCError(POSException.StorageError):
+    pass
+
+class DisconnectedError(ZRPCError, ClientDisconnected):
+    """The database storage is disconnected from the storage server.
+
+    The error occurred because a problem in the low-level RPC connection,
+    or because the connection was closed.
+    """
+
+    # This subclass is raised when zrpc catches the error.

lib/python/ZEO/zrpc/log.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import os
+import threading
+import logging
+
+from ZODB.loglevels import BLATHER
+
+LOG_THREAD_ID = 0 # Set this to 1 during heavy debugging
+
+logger = logging.getLogger('ZEO.zrpc')
+
+_label = "%s" % os.getpid()
+
+def new_label():
+    global _label
+    _label = str(os.getpid())
+
+def log(message, level=BLATHER, label=None, exc_info=False):
+    label = label or _label
+    if LOG_THREAD_ID:
+        label = label + ':' + threading.currentThread().getName()
+    logger.log(level, '(%s) %s' % (label, message), exc_info=exc_info)
+
+REPR_LIMIT = 60
+
+def short_repr(obj):
+    "Return an object repr limited to REPR_LIMIT bytes."
+
+    # Some of the objects being repr'd are large strings. A lot of memory
+    # would be wasted to repr them and then truncate, so they are treated
+    # specially in this function.
+    # Also handle short repr of a tuple containing a long string.
+
+    # This strategy works well for arguments to StorageServer methods.
+    # The oid is usually first and will get included in its entirety.
+    # The pickle is near the beginning, too, and you can often fit the
+    # module name in the pickle.
+
+    if isinstance(obj, str):
+        if len(obj) > REPR_LIMIT:
+            r = repr(obj[:REPR_LIMIT])
+        else:
+            r = repr(obj)
+        if len(r) > REPR_LIMIT:
+            r = r[:REPR_LIMIT-4] + '...' + r[-1]
+        return r
+    elif isinstance(obj, (list, tuple)):
+        elts = []
+        size = 0
+        for elt in obj:
+            r = short_repr(elt)
+            elts.append(r)
+            size += len(r)
+            if size > REPR_LIMIT:
+                break
+        if isinstance(obj, tuple):
+            r = "(%s)" % (", ".join(elts))
+        else:
+            r = "[%s]" % (", ".join(elts))
+    else:
+        r = repr(obj)
+    if len(r) > REPR_LIMIT:
+        return r[:REPR_LIMIT] + '...'
+    else:
+        return r

lib/python/ZEO/zrpc/marshal.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import cPickle
+from cStringIO import StringIO
+import types
+import logging
+
+from ZEO.zrpc.error import ZRPCError
+from ZEO.zrpc.log import log, short_repr
+
+class Marshaller:
+    """Marshal requests and replies to second across network"""
+
+    def encode(self, msgid, flags, name, args):
+        """Returns an encoded message"""
+        # (We used to have a global pickler, but that's not thread-safe. :-( )
+        pickler = cPickle.Pickler()
+        pickler.fast = 1
+        return pickler.dump((msgid, flags, name, args), 1)
+
+    def decode(self, msg):
+        """Decodes msg and returns its parts"""
+        unpickler = cPickle.Unpickler(StringIO(msg))
+        unpickler.find_global = find_global
+
+        try:
+            return unpickler.load() # msgid, flags, name, args
+        except:
+            log("can't decode message: %s" % short_repr(msg),
+                level=logging.ERROR)
+            raise
+
+_globals = globals()
+_silly = ('__doc__',)
+
+def find_global(module, name):
+    """Helper for message unpickler"""
+    try:
+        m = __import__(module, _globals, _globals, _silly)
+    except ImportError, msg:
+        raise ZRPCError("import error %s: %s" % (module, msg))
+
+    try:
+        r = getattr(m, name)
+    except AttributeError:
+        raise ZRPCError("module %s has no global %s" % (module, name))
+
+    safe = getattr(r, '__no_side_effects__', 0)
+    if safe:
+        return r
+
+    # XXX what's a better way to do this?  esp w/ 2.1 & 2.2
+    if type(r) == types.ClassType and issubclass(r, Exception):
+        return r
+
+    raise ZRPCError("Unsafe global: %s.%s" % (module, name))

lib/python/ZEO/zrpc/server.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import asyncore
+import socket
+import types
+
+from ZEO.zrpc.connection import Connection
+from ZEO.zrpc.log import log
+import logging
+import ThreadedAsync.LoopCallback
+
+# Export the main asyncore loop
+loop = ThreadedAsync.LoopCallback.loop
+
+class Dispatcher(asyncore.dispatcher):
+    """A server that accepts incoming RPC connections"""
+    __super_init = asyncore.dispatcher.__init__
+
+    def __init__(self, addr, factory=Connection):
+        self.__super_init()
+        self.addr = addr
+        self.factory = factory
+        self._open_socket()
+
+    def _open_socket(self):
+        if type(self.addr) == types.TupleType:
+            self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        else:
+            self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.set_reuse_addr()
+        log("listening on %s" % str(self.addr), logging.INFO)
+        self.bind(self.addr)
+        self.listen(5)
+
+    def writable(self):
+        return 0
+
+    def readable(self):
+        return 1
+
+    def handle_accept(self):
+        try:
+            sock, addr = self.accept()
+        except socket.error, msg:
+            log("accepted failed: %s" % msg)
+            return
+        c = self.factory(sock, addr)
+        log("connect from %s: %s" % (repr(addr), c))

lib/python/ZEO/zrpc/smac.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Sized Message Async Connections.
+
+This class extends the basic asyncore layer with a record-marking
+layer.  The message_output() method accepts an arbitrary sized string
+as its argument.  It sends over the wire the length of the string
+encoded using struct.pack('>I') and the string itself.  The receiver
+passes the original string to message_input().
+
+This layer also supports an optional message authentication code
+(MAC).  If a session key is present, it uses HMAC-SHA-1 to generate a
+20-byte MAC.  If a MAC is present, the high-order bit of the length
+is set to 1 and the MAC immediately follows the length.
+"""
+
+import asyncore
+import errno
+try:
+    import hmac
+except ImportError:
+    import _hmac as hmac
+import sha
+import socket
+import struct
+import threading
+import logging
+from types import StringType
+
+from ZODB.loglevels import TRACE
+
+from ZEO.zrpc.log import log, short_repr
+from ZEO.zrpc.error import DisconnectedError
+
+
+# Use the dictionary to make sure we get the minimum number of errno
+# entries.   We expect that EWOULDBLOCK == EAGAIN on most systems --
+# or that only one is actually used.
+
+tmp_dict = {errno.EWOULDBLOCK: 0,
+            errno.EAGAIN: 0,
+            errno.EINTR: 0,
+            }
+expected_socket_read_errors = tuple(tmp_dict.keys())
+
+tmp_dict = {errno.EAGAIN: 0,
+            errno.EWOULDBLOCK: 0,
+            errno.ENOBUFS: 0,
+            errno.EINTR: 0,
+            }
+expected_socket_write_errors = tuple(tmp_dict.keys())
+del tmp_dict
+
+# We chose 60000 as the socket limit by looking at the largest strings
+# that we could pass to send() without blocking.
+SEND_SIZE = 60000
+
+MAC_BIT = 0x80000000L
+
+class SizedMessageAsyncConnection(asyncore.dispatcher):
+    __super_init = asyncore.dispatcher.__init__
+    __super_close = asyncore.dispatcher.close
+
+    __closed = True # Marker indicating that we're closed
+
+    socket = None # to outwit Sam's getattr
+
+    def __init__(self, sock, addr, map=None, debug=None):
+        self.addr = addr
+        if debug is not None:
+            self._debug = debug
+        elif not hasattr(self, '_debug'):
+            self._debug = __debug__
+        # __input_lock protects __inp, __input_len, __state, __msg_size
+        self.__input_lock = threading.Lock()
+        self.__inp = None # None, a single String, or a list
+        self.__input_len = 0
+        # Instance variables __state, __msg_size and __has_mac work together:
+        #   when __state == 0:
+        #     __msg_size == 4, and the next thing read is a message size;
+        #     __has_mac is set according to the MAC_BIT in the header
+        #   when __state == 1:
+        #     __msg_size is variable, and the next thing read is a message.
+        #     __has_mac indicates if we're in MAC mode or not (and
+        #               therefore, if we need to check the mac header)
+        # The next thing read is always of length __msg_size.
+        # The state alternates between 0 and 1.
+        self.__state = 0
+        self.__has_mac = 0
+        self.__msg_size = 4
+        self.__output_lock = threading.Lock() # Protects __output
+        self.__output = []
+        self.__closed = False
+        # Each side of the connection sends and receives messages.  A
+        # MAC is generated for each message and depends on each
+        # previous MAC; the state of the MAC generator depends on the
+        # history of operations it has performed.  So the MACs must be
+        # generated in the same order they are verified.
+
+        # Each side is guaranteed to receive messages in the order
+        # they are sent, but there is no ordering constraint between
+        # message sends and receives.  If the two sides are A and B
+        # and message An indicates the nth message sent by A, then
+        # A1 A2 B1 B2 and A1 B1 B2 A2 are both legitimate total
+        # orderings of the messages.
+
+        # As a result, there must be seperate MAC generators for each
+        # side of the connection.  If not, the generator state would
+        # be different after A1 A2 B1 B2 than it would be after
+        # A1 B1 B2 A2; if the generator state was different, the MAC
+        # could not be verified.
+        self.__hmac_send = None
+        self.__hmac_recv = None
+
+        self.__super_init(sock, map)
+
+    def setSessionKey(self, sesskey):
+        log("set session key %r" % sesskey)
+        self.__hmac_send = hmac.HMAC(sesskey, digestmod=sha)
+        self.__hmac_recv = hmac.HMAC(sesskey, digestmod=sha)
+
+    def get_addr(self):
+        return self.addr
+
+    # XXX avoid expensive getattr calls?  Can't remember exactly what
+    # this comment was supposed to mean, but it has something to do
+    # with the way asyncore uses getattr and uses if sock:
+    def __nonzero__(self):
+        return 1
+
+    def handle_read(self):
+        self.__input_lock.acquire()
+        try:
+            # Use a single __inp buffer and integer indexes to make this fast.
+            try:
+                d = self.recv(8192)
+            except socket.error, err:
+                if err[0] in expected_socket_read_errors:
+                    return
+                raise
+            if not d:
+                return
+
+            input_len = self.__input_len + len(d)
+            msg_size = self.__msg_size
+            state = self.__state
+            has_mac = self.__has_mac
+
+            inp = self.__inp
+            if msg_size > input_len:
+                if inp is None:
+                    self.__inp = d
+                elif type(self.__inp) is StringType:
+                    self.__inp = [self.__inp, d]
+                else:
+                    self.__inp.append(d)
+                self.__input_len = input_len
+                return # keep waiting for more input
+
+            # load all previous input and d into single string inp
+            if isinstance(inp, StringType):
+                inp = inp + d
+            elif inp is None:
+                inp = d
+            else:
+                inp.append(d)
+                inp = "".join(inp)
+
+            offset = 0
+            while (offset + msg_size) <= input_len:
+                msg = inp[offset:offset + msg_size]
+                offset = offset + msg_size
+                if not state:
+                    msg_size = struct.unpack(">I", msg)[0]
+                    has_mac = msg_size & MAC_BIT
+                    if has_mac:
+                        msg_size ^= MAC_BIT
+                        msg_size += 20
+                    elif self.__hmac_send:
+                        raise ValueError("Received message without MAC")
+                    state = 1
+                else:
+                    msg_size = 4
+                    state = 0
+                    # XXX We call message_input() with __input_lock
+                    # held!!!  And message_input() may end up calling
+                    # message_output(), which has its own lock.  But
+                    # message_output() cannot call message_input(), so
+                    # the locking order is always consistent, which
+                    # prevents deadlock.  Also, message_input() may
+                    # take a long time, because it can cause an
+                    # incoming call to be handled.  During all this
+                    # time, the __input_lock is held.  That's a good
+                    # thing, because it serializes incoming calls.
+                    if has_mac:
+                        mac = msg[:20]
+                        msg = msg[20:]
+                        if self.__hmac_recv:
+                            self.__hmac_recv.update(msg)
+                            _mac = self.__hmac_recv.digest()
+                            if mac != _mac:
+                                raise ValueError("MAC failed: %r != %r"
+                                                 % (_mac, mac))
+                        else:
+                            log("Received MAC but no session key set")
+                    elif self.__hmac_send:
+                        raise ValueError("Received message without MAC")
+                    self.message_input(msg)
+
+            self.__state = state
+            self.__has_mac = has_mac
+            self.__msg_size = msg_size
+            self.__inp = inp[offset:]
+            self.__input_len = input_len - offset
+        finally:
+            self.__input_lock.release()
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        if len(self.__output) == 0:
+            return False
+        else:
+            return True
+
+    def handle_write(self):
+        self.__output_lock.acquire()
+        try:
+            output = self.__output
+            while output:
+                # Accumulate output into a single string so that we avoid
+                # multiple send() calls, but avoid accumulating too much
+                # data.  If we send a very small string and have more data
+                # to send, we will likely incur delays caused by the
+                # unfortunate interaction between the Nagle algorithm and
+                # delayed acks.  If we send a very large string, only a
+                # portion of it will actually be delivered at a time.
+
+                l = 0
+                for i in range(len(output)):
+                    l += len(output[i])
+                    if l > SEND_SIZE:
+                        break
+
+                i += 1
+                # It is very unlikely that i will be 1.
+                v = "".join(output[:i])
+                del output[:i]
+
+                try:
+                    n = self.send(v)
+                except socket.error, err:
+                    if err[0] in expected_socket_write_errors:
+                        break # we couldn't write anything
+                    raise
+                if n < len(v):
+                    output.insert(0, v[n:])
+                    break # we can't write any more
+        finally:
+            self.__output_lock.release()
+
+    def handle_close(self):
+        self.close()
+
+    def message_output(self, message):
+        if __debug__:
+            if self._debug:
+                log("message_output %d bytes: %s hmac=%d" %
+                    (len(message), short_repr(message),
+                    self.__hmac_send and 1 or 0),
+                    level=TRACE)
+
+        if self.__closed:
+            raise DisconnectedError(
+                "This action is temporarily unavailable.<p>")
+        self.__output_lock.acquire()
+        try:
+            # do two separate appends to avoid copying the message string
+            if self.__hmac_send:
+                self.__output.append(struct.pack(">I", len(message) | MAC_BIT))
+                self.__hmac_send.update(message)
+                self.__output.append(self.__hmac_send.digest())
+            else:
+                self.__output.append(struct.pack(">I", len(message)))
+            if len(message) <= SEND_SIZE:
+                self.__output.append(message)
+            else:
+                for i in range(0, len(message), SEND_SIZE):
+                    self.__output.append(message[i:i+SEND_SIZE])
+        finally:
+            self.__output_lock.release()
+
+    def close(self):
+        if not self.__closed:
+            self.__closed = True
+            self.__super_close()

lib/python/ZEO/zrpc/trigger.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import asyncore
+import os
+import socket
+import thread
+
+if os.name == 'posix':
+
+    class trigger(asyncore.file_dispatcher):
+
+        "Wake up a call to select() running in the main thread"
+
+        # This is useful in a context where you are using Medusa's I/O
+        # subsystem to deliver data, but the data is generated by another
+        # thread.  Normally, if Medusa is in the middle of a call to
+        # select(), new output data generated by another thread will have
+        # to sit until the call to select() either times out or returns.
+        # If the trigger is 'pulled' by another thread, it should immediately
+        # generate a READ event on the trigger object, which will force the
+        # select() invocation to return.
+
+        # A common use for this facility: letting Medusa manage I/O for a
+        # large number of connections; but routing each request through a
+        # thread chosen from a fixed-size thread pool.  When a thread is
+        # acquired, a transaction is performed, but output data is
+        # accumulated into buffers that will be emptied more efficiently
+        # by Medusa. [picture a server that can process database queries
+        # rapidly, but doesn't want to tie up threads waiting to send data
+        # to low-bandwidth connections]
+
+        # The other major feature provided by this class is the ability to
+        # move work back into the main thread: if you call pull_trigger()
+        # with a thunk argument, when select() wakes up and receives the
+        # event it will call your thunk from within that thread.  The main
+        # purpose of this is to remove the need to wrap thread locks around
+        # Medusa's data structures, which normally do not need them.  [To see
+        # why this is true, imagine this scenario: A thread tries to push some
+        # new data onto a channel's outgoing data queue at the same time that
+        # the main thread is trying to remove some]
+
+        def __init__(self):
+            r, w = self._fds = os.pipe()
+            self.trigger = w
+            asyncore.file_dispatcher.__init__(self, r)
+            self.lock = thread.allocate_lock()
+            self.thunks = []
+            self._closed = 0
+
+        # Override the asyncore close() method, because it seems that
+        # it would only close the r file descriptor and not w.  The
+        # constructor calls file_dispatcher.__init__ and passes r,
+        # which would get stored in a file_wrapper and get closed by
+        # the default close.  But that would leave w open...
+
+        def close(self):
+            if not self._closed:
+                self._closed = 1
+                self.del_channel()
+                for fd in self._fds:
+                    os.close(fd)
+                self._fds = []
+
+        def __repr__(self):
+            return '<select-trigger (pipe) at %x>' % id(self)
+
+        def readable(self):
+            return 1
+
+        def writable(self):
+            return 0
+
+        def handle_connect(self):
+            pass
+
+        def handle_close(self):
+            self.close()
+
+        def pull_trigger(self, thunk=None):
+            if thunk:
+                self.lock.acquire()
+                try:
+                    self.thunks.append(thunk)
+                finally:
+                    self.lock.release()
+            os.write(self.trigger, 'x')
+
+        def handle_read(self):
+            try:
+                self.recv(8192)
+            except socket.error:
+                return
+            self.lock.acquire()
+            try:
+                for thunk in self.thunks:
+                    try:
+                        thunk()
+                    except:
+                        nil, t, v, tbinfo = asyncore.compact_traceback()
+                        print ('exception in trigger thunk:'
+                               ' (%s:%s %s)' % (t, v, tbinfo))
+                self.thunks = []
+            finally:
+                self.lock.release()
+
+else:
+
+    # XXX Should define a base class that has the common methods and
+    # then put the platform-specific in a subclass named trigger.
+
+    # win32-safe version
+
+    HOST = '127.0.0.1'
+    MINPORT = 19950
+    NPORTS = 50
+
+    class trigger(asyncore.dispatcher):
+
+        portoffset = 0
+
+        def __init__(self):
+            a = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            w = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+            # set TCP_NODELAY to true to avoid buffering
+            w.setsockopt(socket.IPPROTO_TCP, 1, 1)
+
+            # tricky: get a pair of connected sockets
+            for i in range(NPORTS):
+                trigger.portoffset = (trigger.portoffset + 1) % NPORTS
+                port = MINPORT + trigger.portoffset
+                address = (HOST, port)
+                try:
+                    a.bind(address)
+                except socket.error:
+                    continue
+                else:
+                    break
+            else:
+                raise RuntimeError, 'Cannot bind trigger!'
+
+            a.listen(1)
+            w.setblocking(0)
+            try:
+                w.connect(address)
+            except:
+                pass
+            r, addr = a.accept()
+            a.close()
+            w.setblocking(1)
+            self.trigger = w
+
+            asyncore.dispatcher.__init__(self, r)
+            self.lock = thread.allocate_lock()
+            self.thunks = []
+            self._trigger_connected = 0
+            self._closed = 0
+
+        def close(self):
+            if not self._closed:
+                self._closed = 1
+                self.del_channel()
+                # self.socket is a, self.trigger is w from __init__
+                self.socket.close()
+                self.trigger.close()
+
+        def __repr__(self):
+            return '<select-trigger (loopback) at %x>' % id(self)
+
+        def readable(self):
+            return 1
+
+        def writable(self):
+            return 0
+
+        def handle_connect(self):
+            pass
+
+        def pull_trigger(self, thunk=None):
+            if thunk:
+                self.lock.acquire()
+                try:
+                    self.thunks.append(thunk)
+                finally:
+                    self.lock.release()
+            self.trigger.send('x')
+
+        def handle_read(self):
+            try:
+                self.recv(8192)
+            except socket.error:
+                return
+            self.lock.acquire()
+            try:
+                for thunk in self.thunks:
+                    try:
+                        thunk()
+                    except:
+                        nil, t, v, tbinfo = asyncore.compact_traceback()
+                        print ('exception in trigger thunk:'
+                               ' (%s:%s %s)' % (t, v, tbinfo))
+                self.thunks = []
+            finally:
+                self.lock.release()