Skip to content

Z
Zope
Commit 96172040 authored by root's avatar root
Browse files
Options

Use tagged version of src/ZEO

parent 291d1897
Hide whitespace changes
Inline Side-by-side
Showing with 12602 additions and 0 deletions
lib/python/ZEO/ClientStorage.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The ClientStorage class and the exceptions that it may raise.
Public contents of this module:
ClientStorage -- the main class, implementing the Storage API
"""
import cPickle
import os
import socket
import tempfile
import threading
import time
import types
import logging
from ZEO import ServerStub
from ZEO.cache import ClientCache
from ZEO.TransactionBuffer import TransactionBuffer
from ZEO.Exceptions import ClientStorageError, ClientDisconnected, AuthError
from ZEO.auth import get_module
from ZEO.zrpc.client import ConnectionManager
from ZODB import POSException
from ZODB.loglevels import BLATHER
from persistent.TimeStamp import TimeStamp
logger = logging.getLogger('ZEO.ClientStorage')
_pid = str(os.getpid())
def log2(msg, level=logging.INFO, subsys=_pid, exc_info=False):
message = "(%s) %s" % (subsys, msg)
logger.log(level, message, exc_info=exc_info)
try:
from ZODB.ConflictResolution import ResolvedSerial
except ImportError:
ResolvedSerial = 'rs'
def tid2time(tid):
return str(TimeStamp(tid))
def get_timestamp(prev_ts=None):
"""Internal helper to return a unique TimeStamp instance.
If the optional argument is not None, it must be a TimeStamp; the
return value is then guaranteed to be at least 1 microsecond later
the argument.
"""
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5] + (t % 60,))
if prev_ts is not None:
t = t.laterThan(prev_ts)
return t
class DisconnectedServerStub:
"""Internal helper class used as a faux RPC stub when disconnected.
This raises ClientDisconnected on all attribute accesses.
This is a singleton class -- there should be only one instance,
the global disconnected_stub, os it can be tested by identity.
"""
def __getattr__(self, attr):
raise ClientDisconnected()
# Singleton instance of DisconnectedServerStub
disconnected_stub = DisconnectedServerStub()
MB = 1024**2
class ClientStorage(object):
"""A Storage class that is a network client to a remote storage.
This is a faithful implementation of the Storage API.
This class is thread-safe; transactions are serialized in
tpc_begin().
"""
# Classes we instantiate. A subclass might override.
TransactionBufferClass = TransactionBuffer
ClientCacheClass = ClientCache
ConnectionManagerClass = ConnectionManager
StorageServerStubClass = ServerStub.StorageServer
def __init__(self, addr, storage='1', cache_size=20 * MB,
name='', client=None, debug=0, var=None,
min_disconnect_poll=5, max_disconnect_poll=300,
wait_for_server_on_startup=None, # deprecated alias for wait
wait=None, wait_timeout=None,
read_only=0, read_only_fallback=0,
username='', password='', realm=None):
"""ClientStorage constructor.
This is typically invoked from a custom_zodb.py file.
All arguments except addr should be keyword arguments.
Arguments:
addr -- The server address(es). This is either a list of
addresses or a single address. Each address can be a
(hostname, port) tuple to signify a TCP/IP connection or
a pathname string to signify a Unix domain socket
connection. A hostname may be a DNS name or a dotted IP
address. Required.
storage -- The storage name, defaulting to '1'. The name must
match one of the storage names supported by the server(s)
specified by the addr argument. The storage name is
displayed in the Zope control panel.
cache_size -- The disk cache size, defaulting to 20 megabytes.
This is passed to the ClientCache constructor.
name -- The storage name, defaulting to ''. If this is false,
str(addr) is used as the storage name.
client -- A name used to construct persistent cache filenames.
Defaults to None, in which case the cache is not persistent.
See ClientCache for more info.
debug -- Ignored. This is present only for backwards
compatibility with ZEO 1.
var -- When client is not None, this specifies the directory
where the persistent cache files are created. It defaults
to None, in whichcase the current directory is used.
min_disconnect_poll -- The minimum delay in seconds between
attempts to connect to the server, in seconds. Defaults
to 5 seconds.
max_disconnect_poll -- The maximum delay in seconds between
attempts to connect to the server, in seconds. Defaults
to 300 seconds.
wait_for_server_on_startup -- A backwards compatible alias for
the wait argument.
wait -- A flag indicating whether to wait until a connection
with a server is made, defaulting to true.
wait_timeout -- Maximum time to wait for a connection before
giving up. Only meaningful if wait is True.
read_only -- A flag indicating whether this should be a
read-only storage, defaulting to false (i.e. writing is
allowed by default).
read_only_fallback -- A flag indicating whether a read-only
remote storage should be acceptable as a fallback when no
writable storages are available. Defaults to false. At
most one of read_only and read_only_fallback should be
true.
username -- string with username to be used when authenticating.
These only need to be provided if you are connecting to an
authenticated server storage.
password -- string with plaintext password to be used
when authenticated.
Note that the authentication protocol is defined by the server
and is detected by the ClientStorage upon connecting (see
testConnection() and doAuth() for details).
"""
log2("%s (pid=%d) created %s/%s for storage: %r" %
(self.__class__.__name__,
os.getpid(),
read_only and "RO" or "RW",
read_only_fallback and "fallback" or "normal",
storage))
if debug:
log2("ClientStorage(): debug argument is no longer used")
# wait defaults to True, but wait_for_server_on_startup overrides
# if not None
if wait_for_server_on_startup is not None:
if wait is not None and wait != wait_for_server_on_startup:
log2("ClientStorage(): conflicting values for wait and "
"wait_for_server_on_startup; wait prevails",
level=logging.WARNING)
else:
log2("ClientStorage(): wait_for_server_on_startup "
"is deprecated; please use wait instead")
wait = wait_for_server_on_startup
elif wait is None:
wait = 1
self._addr = addr # For tests
# A ZEO client can run in disconnected mode, using data from
# its cache, or in connected mode. Several instance variables
# are related to whether the client is connected.
# _server: All method calls are invoked through the server
# stub. When not connect, set to disconnected_stub an
# object that raises ClientDisconnected errors.
# _ready: A threading Event that is set only if _server
# is set to a real stub.
# _connection: The current zrpc connection or None.
# _connection is set as soon as a connection is established,
# but _server is set only after cache verification has finished
# and clients can safely use the server. _pending_server holds
# a server stub while it is being verified.
self._server = disconnected_stub
self._connection = None
self._pending_server = None
self._ready = threading.Event()
# _is_read_only stores the constructor argument
self._is_read_only = read_only
# _conn_is_read_only stores the status of the current connection
self._conn_is_read_only = 0
self._storage = storage
self._read_only_fallback = read_only_fallback
self._username = username
self._password = password
self._realm = realm
# Flag tracking disconnections in the middle of a transaction. This
# is reset in tpc_begin() and set in notifyDisconnected().
self._midtxn_disconnect = 0
# _server_addr is used by sortKey()
self._server_addr = None
self._tfile = None
self._pickler = None
self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client',
'supportsUndo':0, 'supportsVersions': 0,
'supportsTransactionalUndo': 0}
self._tbuf = self.TransactionBufferClass()
self._db = None
self._ltid = None # the last committed transaction
# _serials: stores (oid, serialno) as returned by server
# _seriald: _check_serials() moves from _serials to _seriald,
# which maps oid to serialno
# XXX If serial number matches transaction id, then there is
# no need to have all this extra infrastructure for handling
# serial numbers. The vote call can just return the tid.
# If there is a conflict error, we can't have a special method
# called just to propagate the error.
self._serials = []
self._seriald = {}
self.__name__ = name or str(addr) # Standard convention for storages
# A ClientStorage only allows one thread to commit at a time.
# Mutual exclusion is achieved using _tpc_cond, which
# protects _transaction. A thread that wants to assign to
# self._transaction must acquire _tpc_cond first. A thread
# that decides it's done with a transaction (whether via success
# or failure) must set _transaction to None and do
# _tpc_cond.notify() before releasing _tpc_cond.
self._tpc_cond = threading.Condition()
self._transaction = None
# Prevent multiple new_oid calls from going out. The _oids
# variable should only be modified while holding the
# _oid_lock.
self._oid_lock = threading.Lock()
self._oids = [] # Object ids retrieved from new_oids()
# load() and tpc_finish() must be serialized to guarantee
# that cache modifications from each occur atomically.
# It also prevents multiple load calls occuring simultaneously,
# which simplifies the cache logic.
self._load_lock = threading.Lock()
# _load_oid and _load_status are protected by _lock
self._load_oid = None
self._load_status = None
# Can't read data in one thread while writing data
# (tpc_finish) in another thread. In general, the lock
# must prevent access to the cache while _update_cache
# is executing.
self._lock = threading.Lock()
# Decide whether to use non-temporary files
if client is not None:
dir = var or os.getcwd()
cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
else:
cache_path = None
self._cache = self.ClientCacheClass(cache_path)
# XXX When should it be opened?
self._cache.open()
self._rpc_mgr = self.ConnectionManagerClass(addr, self,
tmin=min_disconnect_poll,
tmax=max_disconnect_poll)
if wait:
self._wait(wait_timeout)
else:
# attempt_connect() will make an attempt that doesn't block
# "too long," for a very vague notion of too long. If that
# doesn't succeed, call connect() to start a thread.
if not self._rpc_mgr.attempt_connect():
self._rpc_mgr.connect()
def _wait(self, timeout=None):
if timeout is not None:
deadline = time.time() + timeout
log2("Setting deadline to %f" % deadline, level=BLATHER)
else:
deadline = None
# Wait for a connection to be established.
self._rpc_mgr.connect(sync=1)
# When a synchronous connect() call returns, there is
# a valid _connection object but cache validation may
# still be going on. This code must wait until validation
# finishes, but if the connection isn't a zrpc async
# connection it also needs to poll for input.
if self._connection.is_async():
while 1:
self._ready.wait(30)
if self._ready.isSet():
break
if timeout and time.time() > deadline:
log2("Timed out waiting for connection",
level=logging.WARNING)
break
log2("Waiting for cache verification to finish")
else:
self._wait_sync(deadline)
def _wait_sync(self, deadline=None):
# If there is no mainloop running, this code needs
# to call poll() to cause asyncore to handle events.
while 1:
if self._ready.isSet():
break
if deadline and time.time() > deadline:
log2("Timed out waiting for connection", level=logging.WARNING)
break
log2("Waiting for cache verification to finish")
if self._connection is None:
# If the connection was closed while we were
# waiting for it to become ready, start over.
return self._wait(deadline - time.time())
else:
self._connection.pending(30)
def close(self):
"""Storage API: finalize the storage, releasing external resources."""
self._tbuf.close()
if self._cache is not None:
self._cache.close()
self._cache = None
if self._rpc_mgr is not None:
self._rpc_mgr.close()
self._rpc_mgr = None
def registerDB(self, db, limit):
"""Storage API: register a database for invalidation messages.
This is called by ZODB.DB (and by some tests).
The storage isn't really ready to use until after this call.
"""
self._db = db
def is_connected(self):
"""Return whether the storage is currently connected to a server."""
# This function is used by clients, so we only report that a
# connection exists when the connection is ready to use.
return self._ready.isSet()
def sync(self):
"""Handle any pending invalidation messages.
This is called by the sync method in ZODB.Connection.
"""
# If there is no connection, return immediately. Technically,
# there are no pending invalidations so they are all handled.
# There doesn't seem to be much benefit to raising an exception.
cn = self._connection
if cn is not None:
cn.pending()
def doAuth(self, protocol, stub):
if not (self._username and self._password):
raise AuthError, "empty username or password"
module = get_module(protocol)
if not module:
log2("%s: no such an auth protocol: %s" %
(self.__class__.__name__, protocol), level=logging.WARNING)
return
storage_class, client, db_class = module
if not client:
log2("%s: %s isn't a valid protocol, must have a Client class" %
(self.__class__.__name__, protocol), level=logging.WARNING)
raise AuthError, "invalid protocol"
c = client(stub)
# Initiate authentication, returns boolean specifying whether OK
return c.start(self._username, self._realm, self._password)
def testConnection(self, conn):
"""Internal: test the given connection.
This returns: 1 if the connection is an optimal match,
0 if it is a suboptimal but acceptable match.
It can also raise DisconnectedError or ReadOnlyError.
This is called by ZEO.zrpc.ConnectionManager to decide which
connection to use in case there are multiple, and some are
read-only and others are read-write.
This works by calling register() on the server. In read-only
mode, register() is called with the read_only flag set. In
writable mode and in read-only fallback mode, register() is
called with the read_only flag cleared. In read-only fallback
mode only, if the register() call raises ReadOnlyError, it is
retried with the read-only flag set, and if this succeeds,
this is deemed a suboptimal match. In all other cases, a
succeeding register() call is deemed an optimal match, and any
exception raised by register() is passed through.
"""
log2("Testing connection %r" % conn)
# XXX Check the protocol version here?
self._conn_is_read_only = 0
stub = self.StorageServerStubClass(conn)
auth = stub.getAuthProtocol()
log2("Server authentication protocol %r" % auth)
if auth:
skey = self.doAuth(auth, stub)
if skey:
log2("Client authentication successful")
conn.setSessionKey(skey)
else:
log2("Authentication failed")
raise AuthError, "Authentication failed"
try:
stub.register(str(self._storage), self._is_read_only)
return 1
except POSException.ReadOnlyError:
if not self._read_only_fallback:
raise
log2("Got ReadOnlyError; trying again with read_only=1")
stub.register(str(self._storage), read_only=1)
self._conn_is_read_only = 1
return 0
def notifyConnected(self, conn):
"""Internal: start using the given connection.
This is called by ConnectionManager after it has decided which
connection should be used.
"""
if self._cache is None:
# the storage was closed, but the connect thread called
# this method before it was stopped.
return
# XXX would like to report whether we get a read-only connection
if self._connection is not None:
reconnect = 1
else:
reconnect = 0
self.set_server_addr(conn.get_addr())
# If we are upgrading from a read-only fallback connection,
# we must close the old connection to prevent it from being
# used while the cache is verified against the new connection.
if self._connection is not None:
self._connection.close()
self._connection = conn
if reconnect:
log2("Reconnected to storage: %s" % self._server_addr)
else:
log2("Connected to storage: %s" % self._server_addr)
stub = self.StorageServerStubClass(conn)
self._oids = []
self._info.update(stub.get_info())
self.verify_cache(stub)
if not conn.is_async():
log2("Waiting for cache verification to finish")
self._wait_sync()
self._handle_extensions()
def _handle_extensions(self):
for name in self.getExtensionMethods().keys():
if not hasattr(self, name):
setattr(self, name, self._server.extensionMethod(name))
def set_server_addr(self, addr):
# Normalize server address and convert to string
if isinstance(addr, types.StringType):
self._server_addr = addr
else:
assert isinstance(addr, types.TupleType)
# If the server is on a remote host, we need to guarantee
# that all clients used the same name for the server. If
# they don't, the sortKey() may be different for each client.
# The best solution seems to be the official name reported
# by gethostbyaddr().
host = addr[0]
try:
canonical, aliases, addrs = socket.gethostbyaddr(host)
except socket.error, err:
log2("Error resolving host: %s (%s)" % (host, err),
level=BLATHER)
canonical = host
self._server_addr = str((canonical, addr[1]))
def sortKey(self):
# If the client isn't connected to anything, it can't have a
# valid sortKey(). Raise an error to stop the transaction early.
if self._server_addr is None:
raise ClientDisconnected
else:
return '%s:%s' % (self._storage, self._server_addr)
def verify_cache(self, server):
"""Internal routine called to verify the cache.
The return value (indicating which path we took) is used by
the test suite.
"""
# If verify_cache() finishes the cache verification process,
# it should set self._server. If it goes through full cache
# verification, then endVerify() should self._server.
last_inval_tid = self._cache.getLastTid()
if last_inval_tid is not None:
ltid = server.lastTransaction()
if ltid == last_inval_tid:
log2("No verification necessary (last_inval_tid up-to-date)")
self._server = server
self._ready.set()
return "no verification"
# log some hints about last transaction
log2("last inval tid: %r %s\n"
% (last_inval_tid, tid2time(last_inval_tid)))
log2("last transaction: %r %s" %
(ltid, ltid and tid2time(ltid)))
pair = server.getInvalidations(last_inval_tid)
if pair is not None:
log2("Recovering %d invalidations" % len(pair[1]))
self.invalidateTransaction(*pair)
self._server = server
self._ready.set()
return "quick verification"
log2("Verifying cache")
# setup tempfile to hold zeoVerify results
self._tfile = tempfile.TemporaryFile(suffix=".inv")
self._pickler = cPickle.Pickler(self._tfile, 1)
self._pickler.fast = 1 # Don't use the memo
# XXX should batch these operations for efficiency
# XXX need to acquire lock...
for oid, tid, version in self._cache.contents():
server.verify(oid, version, tid)
self._pending_server = server
server.endZeoVerify()
return "full verification"
### Is there a race condition between notifyConnected and
### notifyDisconnected? In Particular, what if we get
### notifyDisconnected in the middle of notifyConnected?
### The danger is that we'll proceed as if we were connected
### without worrying if we were, but this would happen any way if
### notifyDisconnected had to get the instance lock. There's
### nothing to gain by getting the instance lock.
def notifyDisconnected(self):
"""Internal: notify that the server connection was terminated.
This is called by ConnectionManager when the connection is
closed or when certain problems with the connection occur.
"""
log2("Disconnected from storage: %s" % repr(self._server_addr))
self._connection = None
self._ready.clear()
self._server = disconnected_stub
self._midtxn_disconnect = 1
def __len__(self):
"""Return the size of the storage."""
# XXX Where is this used?
return self._info['length']
def getName(self):
"""Storage API: return the storage name as a string.
The return value consists of two parts: the name as determined
by the name and addr argments to the ClientStorage
constructor, and the string 'connected' or 'disconnected' in
parentheses indicating whether the storage is (currently)
connected.
"""
return "%s (%s)" % (
self.__name__,
self.is_connected() and "connected" or "disconnected")
def getSize(self):
"""Storage API: an approximate size of the database, in bytes."""
return self._info['size']
def getExtensionMethods(self):
"""getExtensionMethods
This returns a dictionary whose keys are names of extra methods
provided by this storage. Storage proxies (such as ZEO) should
call this method to determine the extra methods that they need
to proxy in addition to the standard storage methods.
Dictionary values should be None; this will be a handy place
for extra marshalling information, should we need it
"""
return self._info.get('extensionMethods', {})
def supportsUndo(self):
"""Storage API: return whether we support undo."""
return self._info['supportsUndo']
def supportsVersions(self):
"""Storage API: return whether we support versions."""
return self._info['supportsVersions']
def supportsTransactionalUndo(self):
"""Storage API: return whether we support transactional undo."""
return self._info['supportsTransactionalUndo']
def isReadOnly(self):
"""Storage API: return whether we are in read-only mode."""
if self._is_read_only:
return 1
else:
# If the client is configured for a read-write connection
# but has a read-only fallback connection, _conn_is_read_only
# will be True.
return self._conn_is_read_only
def _check_trans(self, trans):
"""Internal helper to check a transaction argument for sanity."""
if self._is_read_only:
raise POSException.ReadOnlyError()
if self._transaction is not trans:
raise POSException.StorageTransactionError(self._transaction,
trans)
def abortVersion(self, version, txn):
"""Storage API: clear any changes made by the given version."""
self._check_trans(txn)
tid, oids = self._server.abortVersion(version, id(txn))
# When a version aborts, invalidate the version and
# non-version data. The non-version data should still be
# valid, but older versions of ZODB will change the
# non-version serialno on an abort version. With those
# versions of ZODB, you'd get a conflict error if you tried to
# commit a transaction with the cached data.
# XXX If we could guarantee that ZODB gave the right answer,
# we could just invalidate the version data.
for oid in oids:
self._tbuf.invalidate(oid, '')
return tid, oids
def commitVersion(self, source, destination, txn):
"""Storage API: commit the source version in the destination."""
self._check_trans(txn)
tid, oids = self._server.commitVersion(source, destination, id(txn))
if destination:
# just invalidate our version data
for oid in oids:
self._tbuf.invalidate(oid, source)
else:
# destination is "", so invalidate version and non-version
for oid in oids:
self._tbuf.invalidate(oid, "")
return tid, oids
def history(self, oid, version, length=1):
"""Storage API: return a sequence of HistoryEntry objects.
This does not support the optional filter argument defined by
the Storage API.
"""
return self._server.history(oid, version, length)
def getSerial(self, oid):
"""Storage API: return current serial number for oid."""
return self._server.getSerial(oid)
def loadSerial(self, oid, serial):
"""Storage API: load a historical revision of an object."""
return self._server.loadSerial(oid, serial)
def load(self, oid, version):
"""Storage API: return the data for a given object.
This returns the pickle data and serial number for the object
specified by the given object id and version, if they exist;
otherwise a KeyError is raised.
"""
return self.loadEx(oid, version)[:2]
def loadEx(self, oid, version):
self._lock.acquire() # for atomic processing of invalidations
try:
t = self._cache.load(oid, version)
if t:
return t
finally:
self._lock.release()
if self._server is None:
raise ClientDisconnected()
self._load_lock.acquire()
try:
self._lock.acquire()
try:
self._load_oid = oid
self._load_status = 1
finally:
self._lock.release()
data, tid, ver = self._server.loadEx(oid, version)
self._lock.acquire() # for atomic processing of invalidations
try:
if self._load_status:
self._cache.store(oid, ver, tid, None, data)
self._load_oid = None
finally:
self._lock.release()
finally:
self._load_lock.release()
return data, tid, ver
def loadBefore(self, oid, tid):
self._lock.acquire()
try:
t = self._cache.loadBefore(oid, tid)
if t is not None:
return t
finally:
self._lock.release()
t = self._server.loadBefore(oid, tid)
if t is None:
return None
data, start, end = t
if end is None:
# This method should not be used to get current data. It
# doesn't use the _load_lock, so it is possble to overlap
# this load with an invalidation for the same object.
# XXX If we call again, we're guaranteed to get the
# post-invalidation data. But if the data is still
# current, we'll still get end == None.
# Maybe the best thing to do is to re-run the test with
# the load lock in the case. That's slow performance, but
# I don't think real application code will ever care about
# it.
return data, start, end
self._lock.acquire()
try:
self._cache.store(oid, "", start, end, data)
finally:
self._lock.release()
return data, start, end
def modifiedInVersion(self, oid):
"""Storage API: return the version, if any, that modfied an object.
If no version modified the object, return an empty string.
"""
self._lock.acquire()
try:
v = self._cache.modifiedInVersion(oid)
if v is not None:
return v
finally:
self._lock.release()
return self._server.modifiedInVersion(oid)
def new_oid(self):
"""Storage API: return a new object identifier."""
if self._is_read_only:
raise POSException.ReadOnlyError()
# avoid multiple oid requests to server at the same time
self._oid_lock.acquire()
try:
if not self._oids:
self._oids = self._server.new_oids()
self._oids.reverse()
return self._oids.pop()
finally:
self._oid_lock.release()
def pack(self, t=None, referencesf=None, wait=1, days=0):
"""Storage API: pack the storage.
Deviations from the Storage API: the referencesf argument is
ignored; two additional optional arguments wait and days are
provided:
wait -- a flag indicating whether to wait for the pack to
complete; defaults to true.
days -- a number of days to subtract from the pack time;
defaults to zero.
"""
# XXX Is it okay that read-only connections allow pack()?
# rf argument ignored; server will provide it's own implementation
if t is None:
t = time.time()
t = t - (days * 86400)
return self._server.pack(t, wait)
def _check_serials(self):
"""Internal helper to move data from _serials to _seriald."""
# XXX serials are always going to be the same, the only
# question is whether an exception has been raised.
if self._serials:
l = len(self._serials)
r = self._serials[:l]
del self._serials[:l]
for oid, s in r:
if isinstance(s, Exception):
raise s
self._seriald[oid] = s
return r
def store(self, oid, serial, data, version, txn):
"""Storage API: store data for an object."""
self._check_trans(txn)
self._server.storea(oid, serial, data, version, id(txn))
self._tbuf.store(oid, version, data)
return self._check_serials()
def tpc_vote(self, txn):
"""Storage API: vote on a transaction."""
if txn is not self._transaction:
return
self._server.vote(id(txn))
return self._check_serials()
def tpc_begin(self, txn, tid=None, status=' '):
"""Storage API: begin a transaction."""
if self._is_read_only:
raise POSException.ReadOnlyError()
self._tpc_cond.acquire()
self._midtxn_disconnect = 0
while self._transaction is not None:
# It is allowable for a client to call two tpc_begins in a
# row with the same transaction, and the second of these
# must be ignored.
if self._transaction == txn:
self._tpc_cond.release()
return
self._tpc_cond.wait(30)
self._transaction = txn
self._tpc_cond.release()
try:
self._server.tpc_begin(id(txn), txn.user, txn.description,
txn._extension, tid, status)
except:
# Client may have disconnected during the tpc_begin().
if self._server is not disconnected_stub:
self.end_transaction()
raise
self._tbuf.clear()
self._seriald.clear()
del self._serials[:]
def end_transaction(self):
"""Internal helper to end a transaction."""
# the right way to set self._transaction to None
# calls notify() on _tpc_cond in case there are waiting threads
self._tpc_cond.acquire()
self._transaction = None
self._tpc_cond.notify()
self._tpc_cond.release()
def lastTransaction(self):
return self._cache.getLastTid()
def tpc_abort(self, txn):
"""Storage API: abort a transaction."""
if txn is not self._transaction:
return
try:
# XXX Are there any transactions that should prevent an
# abort from occurring? It seems wrong to swallow them
# all, yet you want to be sure that other abort logic is
# executed regardless.
try:
self._server.tpc_abort(id(txn))
except ClientDisconnected:
log2("ClientDisconnected in tpc_abort() ignored",
level=BLATHER)
finally:
self._tbuf.clear()
self._seriald.clear()
del self._serials[:]
self.end_transaction()
def tpc_finish(self, txn, f=None):
"""Storage API: finish a transaction."""
if txn is not self._transaction:
return
self._load_lock.acquire()
try:
if self._midtxn_disconnect:
raise ClientDisconnected(
'Calling tpc_finish() on a disconnected transaction')
# The calls to tpc_finish() and _update_cache() should
# never run currently with another thread, because the
# tpc_cond condition variable prevents more than one
# thread from calling tpc_finish() at a time.
tid = self._server.tpc_finish(id(txn))
self._lock.acquire() # for atomic processing of invalidations
try:
self._update_cache(tid)
if f is not None:
f(tid)
finally:
self._lock.release()
r = self._check_serials()
assert r is None or len(r) == 0, "unhandled serialnos: %s" % r
finally:
self._load_lock.release()
self.end_transaction()
def _update_cache(self, tid):
"""Internal helper to handle objects modified by a transaction.
This iterates over the objects in the transaction buffer and
update or invalidate the cache.
"""
# Must be called with _lock already acquired.
# XXX not sure why _update_cache() would be called on
# a closed storage.
if self._cache is None:
return
for oid, version, data in self._tbuf:
self._cache.invalidate(oid, version, tid)
# If data is None, we just invalidate.
if data is not None:
s = self._seriald[oid]
if s != ResolvedSerial:
assert s == tid, (s, tid)
self._cache.store(oid, version, s, None, data)
self._tbuf.clear()
def undo(self, trans_id, txn):
"""Storage API: undo a transaction.
This is executed in a transactional context. It has no effect
until the transaction is committed. It can be undone itself.
Zope uses this to implement undo unless it is not supported by
a storage.
"""
self._check_trans(txn)
tid, oids = self._server.undo(trans_id, id(txn))
for oid in oids:
self._tbuf.invalidate(oid, '')
return tid, oids
def undoInfo(self, first=0, last=-20, specification=None):
"""Storage API: return undo information."""
return self._server.undoInfo(first, last, specification)
def undoLog(self, first=0, last=-20, filter=None):
"""Storage API: return a sequence of TransactionDescription objects.
The filter argument should be None or left unspecified, since
it is impossible to pass the filter function to the server to
be executed there. If filter is not None, an empty sequence
is returned.
"""
if filter is not None:
return []
return self._server.undoLog(first, last)
def versionEmpty(self, version):
"""Storage API: return whether the version has no transactions."""
return self._server.versionEmpty(version)
def versions(self, max=None):
"""Storage API: return a sequence of versions in the storage."""
return self._server.versions(max)
# Below are methods invoked by the StorageServer
def serialnos(self, args):
"""Server callback to pass a list of changed (oid, serial) pairs."""
self._serials.extend(args)
def info(self, dict):
"""Server callback to update the info dictionary."""
self._info.update(dict)
def invalidateVerify(self, args):
"""Server callback to invalidate an (oid, version) pair.
This is called as part of cache validation.
"""
# Invalidation as result of verify_cache().
# Queue an invalidate for the end the verification procedure.
if self._pickler is None:
# XXX This should never happen
return
self._pickler.dump(args)
def _process_invalidations(self, invs):
# Invalidations are sent by the ZEO server as a sequence of
# oid, version pairs. The DB's invalidate() method expects a
# dictionary of oids.
self._lock.acquire()
try:
# versions maps version names to dictionary of invalidations
versions = {}
for oid, version, tid in invs:
if oid == self._load_oid:
self._load_status = 0
self._cache.invalidate(oid, version, tid)
versions.setdefault((version, tid), {})[oid] = tid
if self._db is not None:
for (version, tid), d in versions.items():
self._db.invalidate(tid, d, version=version)
finally:
self._lock.release()
def endVerify(self):
"""Server callback to signal end of cache validation."""
if self._pickler is None:
return
# write end-of-data marker
self._pickler.dump((None, None))
self._pickler = None
self._tfile.seek(0)
f = self._tfile
self._tfile = None
self._process_invalidations(InvalidationLogIterator(f))
f.close()
log2("endVerify finishing")
self._server = self._pending_server
self._ready.set()
self._pending_conn = None
log2("endVerify finished")
def invalidateTransaction(self, tid, args):
"""Invalidate objects modified by tid."""
self._lock.acquire()
try:
self._cache.setLastTid(tid)
finally:
self._lock.release()
if self._pickler is not None:
log2("Transactional invalidation during cache verification",
level=BLATHER)
for t in args:
self._pickler.dump(t)
return
self._process_invalidations([(oid, version, tid)
for oid, version in args])
# The following are for compatibility with protocol version 2.0.0
def invalidateTrans(self, args):
return self.invalidateTransaction(None, args)
invalidate = invalidateVerify
end = endVerify
Invalidate = invalidateTrans
def InvalidationLogIterator(fileobj):
unpickler = cPickle.Unpickler(fileobj)
while 1:
oid, version = unpickler.load()
if oid is None:
break
yield oid, version, None
lib/python/ZEO/ClientStub.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""RPC stubs for interface exported by ClientStorage."""
class ClientStorage:
"""An RPC stub class for the interface exported by ClientStorage.
This is the interface presented by ClientStorage to the
StorageServer; i.e. the StorageServer calls these methods and they
are executed in the ClientStorage.
See the ClientStorage class for documentation on these methods.
It is currently important that all methods here are asynchronous
(meaning they don't have a return value and the caller doesn't
wait for them to complete), *and* that none of them cause any
calls from the client to the storage. This is due to limitations
in the zrpc subpackage.
The on-the-wire names of some of the methods don't match the
Python method names. That's because the on-the-wire protocol was
fixed for ZEO 2 and we don't want to change it. There are some
aliases in ClientStorage.py to make up for this.
"""
def __init__(self, rpc):
"""Constructor.
The argument is a connection: an instance of the
zrpc.connection.Connection class.
"""
self.rpc = rpc
def beginVerify(self):
self.rpc.callAsync('beginVerify')
def invalidateVerify(self, args):
self.rpc.callAsync('invalidateVerify', args)
def endVerify(self):
self.rpc.callAsync('endVerify')
def invalidateTransaction(self, tid, args):
self.rpc.callAsyncNoPoll('invalidateTransaction', tid, args)
def serialnos(self, arg):
self.rpc.callAsync('serialnos', arg)
def info(self, arg):
self.rpc.callAsync('info', arg)
lib/python/ZEO/CommitLog.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Log a transaction's commit info during two-phase commit.
A storage server allows multiple clients to commit transactions, but
must serialize them as the actually execute at the server. The
concurrent commits are achieved by logging actions up until the
tpc_vote(). At that point, the entire transaction is committed on the
real storage.
"""
import cPickle
import tempfile
class CommitLog:
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".log")
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
self.stores = 0
self.read = 0
def size(self):
return self.file.tell()
def store(self, oid, serial, data, version):
self.pickler.dump((oid, serial, data, version))
self.stores += 1
def get_loader(self):
self.read = 1
self.file.seek(0)
return self.stores, cPickle.Unpickler(self.file)
def close(self):
if self.file:
self.file.close()
self.file = None
lib/python/ZEO/DEPENDENCIES.cfg 0 → 100644
View file @ 96172040
BTrees
ThreadedAsync
ZConfig
ZODB
persistent
transaction
zdaemon
lib/python/ZEO/DebugServer.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A debugging version of the server that records network activity."""
import struct
import time
from ZEO.StorageServer import StorageServer, log
from ZEO.zrpc.server import ManagedServerConnection
# a bunch of codes
NEW_CONN = 1
CLOSE_CONN = 2
DATA = 3
ERROR = 4
class DebugManagedServerConnection(ManagedServerConnection):
def __init__(self, sock, addr, obj, mgr):
# mgr is the DebugServer instance
self.mgr = mgr
self.__super_init(sock, addr, obj)
record_id = mgr._record_connection(addr)
self._record = lambda code, data: mgr._record(record_id, code, data)
self.obj.notifyConnected(self)
def close(self):
self._record(CLOSE_CONN, "")
ManagedServerConnection.close(self)
# override the lowest-level of asyncore's connection
def recv(self, buffer_size):
try:
data = self.socket.recv(buffer_size)
if not data:
# a closed connection is indicated by signaling
# a read condition, and having recv() return 0.
self.handle_close()
return ''
else:
self._record(DATA, data)
return data
except socket.error, why:
# winsock sometimes throws ENOTCONN
self._record(ERROR, why)
if why[0] in [ECONNRESET, ENOTCONN, ESHUTDOWN]:
self.handle_close()
return ''
else:
raise socket.error, why
class DebugServer(StorageServer):
ZEOStorageClass = DebugZEOStorage
ManagedServerConnectionClass = DebugManagerConnection
def __init__(self, *args, **kwargs):
StorageServer.__init__(*args, **kwargs)
self._setup_record(kwargs["record"])
self._conn_counter = 1
def _setup_record(self, path):
try:
self._recordfile = open(path, "ab")
except IOError, msg:
self._recordfile = None
log("failed to open recordfile %s: %s" % (path, msg))
def _record_connection(self, addr):
cid = self._conn_counter
self._conn_counter += 1
self._record(cid, NEW_CONN, str(addr))
return cid
def _record(self, conn, code, data):
s = struct.pack(">iii", code, time.time(), len(data)) + data
self._recordfile.write(s)
lib/python/ZEO/Exceptions.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Exceptions for ZEO."""
from ZODB.POSException import StorageError
class ClientStorageError(StorageError):
"""An error occured in the ZEO Client Storage."""
class UnrecognizedResult(ClientStorageError):
"""A server call returned an unrecognized result."""
class ClientDisconnected(ClientStorageError):
"""The database storage is disconnected from the storage."""
class AuthError(StorageError):
"""The client provided invalid authentication credentials."""
lib/python/ZEO/README.txt 0 → 100644
View file @ 96172040
=======
ZEO 2.0
=======
What's ZEO?
-----------
ZEO stands for Zope Enterprise Objects. ZEO is an add-on for Zope
that allows multiple processes to connect to a single ZODB storage.
Those processes can live on different machines, but don't need to.
ZEO 2 has many improvements over ZEO 1, and is incompatible with ZEO 1;
if you upgrade an existing ZEO 1 installation, you must upgrade the
server and all clients simultaneous. If you received ZEO 2 as part of
the ZODB 3 distribution, the ZEO 1 sources are provided in a separate
directory (ZEO1). Some documentation for ZEO is available in the ZODB 3
package in the Doc subdirectory. ZEO depends on the ZODB software; it
can be used with the version of ZODB distributed with Zope 2.5.1 or
later. More information about ZEO can be found in the ZODB Wiki:
http://www.zope.org/Wikis/ZODB
What's here?
------------
This list of filenames is mostly for ZEO developers::
ClientCache.py client-side cache implementation
ClientStorage.py client-side storage implementation
ClientStub.py RPC stubs for callbacks from server to client
CommitLog.py buffer used during two-phase commit on the server
Exceptions.py definitions of exceptions
ICache.py interface definition for the client-side cache
ServerStub.py RPC stubs for the server
StorageServer.py server-side storage implementation
TransactionBuffer.py buffer used for transaction data in the client
__init__.py near-empty file to make this directory a package
simul.py command-line tool to simulate cache behavior
start.py command-line tool to start the storage server
stats.py command-line tool to process client cache traces
tests/ unit tests and other test utilities
util.py utilities used by the server startup tool
version.txt text file indicating the ZEO version
zrpc/ subpackage implementing Remote Procedure Call (RPC)
lib/python/ZEO/ServerStub.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""RPC stubs for interface exported by StorageServer."""
##
# ZEO storage server.
# <p>
# Remote method calls can be synchronous or asynchronous. If the call
# is synchronous, the client thread blocks until the call returns. A
# single client can only have one synchronous request outstanding. If
# several threads share a single client, threads other than the caller
# will block only if the attempt to make another synchronous call.
# An asynchronous call does not cause the client thread to block. An
# exception raised by an asynchronous method is logged on the server,
# but is not returned to the client.
class StorageServer:
"""An RPC stub class for the interface exported by ClientStorage.
This is the interface presented by the StorageServer to the
ClientStorage; i.e. the ClientStorage calls these methods and they
are executed in the StorageServer.
See the StorageServer module for documentation on these methods,
with the exception of _update(), which is documented here.
"""
def __init__(self, rpc):
"""Constructor.
The argument is a connection: an instance of the
zrpc.connection.Connection class.
"""
self.rpc = rpc
# Wait until we know what version the other side is using.
while rpc.peer_protocol_version is None:
rpc.pending()
if rpc.peer_protocol_version == 'Z200':
self.lastTransaction = lambda: None
self.getInvalidations = lambda tid: None
self.getAuthProtocol = lambda: None
def extensionMethod(self, name):
return ExtensionMethodWrapper(self.rpc, name).call
##
# Register current connection with a storage and a mode.
# In effect, it is like an open call.
# @param storage_name a string naming the storage. This argument
# is primarily for backwards compatibility with servers
# that supported multiple storages.
# @param read_only boolean
# @exception ValueError unknown storage_name or already registered
# @exception ReadOnlyError storage is read-only and a read-write
# connectio was requested
def register(self, storage_name, read_only):
self.rpc.call('register', storage_name, read_only)
##
# Return dictionary of meta-data about the storage.
# @defreturn dict
def get_info(self):
return self.rpc.call('get_info')
##
# Check whether the server requires authentication. Returns
# the name of the protocol.
# @defreturn string
def getAuthProtocol(self):
return self.rpc.call('getAuthProtocol')
##
# Return id of the last committed transaction
# @defreturn string
def lastTransaction(self):
# Not in protocol version 2.0.0; see __init__()
return self.rpc.call('lastTransaction')
##
# Return invalidations for all transactions after tid.
# @param tid transaction id
# @defreturn 2-tuple, (tid, list)
# @return tuple containing the last committed transaction
# and a list of oids that were invalidated. Returns
# None and an empty list if the server does not have
# the list of oids available.
def getInvalidations(self, tid):
# Not in protocol version 2.0.0; see __init__()
return self.rpc.call('getInvalidations', tid)
##
# Check whether serial numbers s and sv are current for oid.
# If one or both of the serial numbers are not current, the
# server will make an asynchronous invalidateVerify() call.
# @param oid object id
# @param s serial number on non-version data
# @param sv serial number of version data or None
# @defreturn async
def zeoVerify(self, oid, s, sv):
self.rpc.callAsync('zeoVerify', oid, s, sv)
##
# Check whether current serial number is valid for oid and version.
# If the serial number is not current, the server will make an
# asynchronous invalidateVerify() call.
# @param oid object id
# @param version name of version for oid
# @param serial client's current serial number
# @defreturn async
def verify(self, oid, version, serial):
self.rpc.callAsync('verify', oid, version, serial)
##
# Signal to the server that cache verification is done.
# @defreturn async
def endZeoVerify(self):
self.rpc.callAsync('endZeoVerify')
##
# Generate a new set of oids.
# @param n number of new oids to return
# @defreturn list
# @return list of oids
def new_oids(self, n=None):
if n is None:
return self.rpc.call('new_oids')
else:
return self.rpc.call('new_oids', n)
##
# Pack the storage.
# @param t pack time
# @param wait optional, boolean. If true, the call will not
# return until the pack is complete.
def pack(self, t, wait=None):
if wait is None:
self.rpc.call('pack', t)
else:
self.rpc.call('pack', t, wait)
##
# Return current data for oid. Version data is returned if
# present.
# @param oid object id
# @defreturn 5-tuple
# @return 5-tuple, current non-version data, serial number,
# version name, version data, version data serial number
# @exception KeyError if oid is not found
def zeoLoad(self, oid):
return self.rpc.call('zeoLoad', oid)
##
# Return current data for oid along with tid if transaction that
# wrote the date.
# @param oid object id
# @param version string, name of version
# @defreturn 4-tuple
# @return data, serial number, transaction id, version,
# where version is the name of the version the data came
# from or "" for non-version data
# @exception KeyError if oid is not found
def loadEx(self, oid, version):
return self.rpc.call("loadEx", oid, version)
##
# Return non-current data along with transaction ids that identify
# the lifetime of the specific revision.
# @param oid object id
# @param tid a transaction id that provides an upper bound on
# the lifetime of the revision. That is, loadBefore
# returns the revision that was current before tid committed.
# @defreturn 4-tuple
# @return data, serial numbr, start transaction id, end transaction id
def loadBefore(self, oid, tid):
return self.rpc.call("loadBefore", oid, tid)
##
# Storage new revision of oid.
# @param oid object id
# @param serial serial number that this transaction read
# @param data new data record for oid
# @param version name of version or ""
# @param id id of current transaction
# @defreturn async
def storea(self, oid, serial, data, version, id):
self.rpc.callAsync('storea', oid, serial, data, version, id)
##
# Start two-phase commit for a transaction
# @param id id used by client to identify current transaction. The
# only purpose of this argument is to distinguish among multiple
# threads using a single ClientStorage.
# @param user name of user committing transaction (can be "")
# @param description string containing transaction metadata (can be "")
# @param ext dictionary of extended metadata (?)
# @param tid optional explicit tid to pass to underlying storage
# @param status optional status character, e.g "p" for pack
# @defreturn async
def tpc_begin(self, id, user, descr, ext, tid, status):
return self.rpc.call('tpc_begin', id, user, descr, ext, tid, status)
def vote(self, trans_id):
return self.rpc.call('vote', trans_id)
def tpc_finish(self, id):
return self.rpc.call('tpc_finish', id)
def tpc_abort(self, id):
self.rpc.callAsync('tpc_abort', id)
def abortVersion(self, src, id):
return self.rpc.call('abortVersion', src, id)
def commitVersion(self, src, dest, id):
return self.rpc.call('commitVersion', src, dest, id)
def history(self, oid, version, length=None):
if length is None:
return self.rpc.call('history', oid, version)
else:
return self.rpc.call('history', oid, version, length)
def load(self, oid, version):
return self.rpc.call('load', oid, version)
def getSerial(self, oid):
return self.rpc.call('getSerial', oid)
def loadSerial(self, oid, serial):
return self.rpc.call('loadSerial', oid, serial)
def modifiedInVersion(self, oid):
return self.rpc.call('modifiedInVersion', oid)
def new_oid(self, last=None):
if last is None:
return self.rpc.call('new_oid')
else:
return self.rpc.call('new_oid', last)
def store(self, oid, serial, data, version, trans):
return self.rpc.call('store', oid, serial, data, version, trans)
def undo(self, trans_id, trans):
return self.rpc.call('undo', trans_id, trans)
def undoLog(self, first, last):
return self.rpc.call('undoLog', first, last)
def undoInfo(self, first, last, spec):
return self.rpc.call('undoInfo', first, last, spec)
def versionEmpty(self, vers):
return self.rpc.call('versionEmpty', vers)
def versions(self, max=None):
if max is None:
return self.rpc.call('versions')
else:
return self.rpc.call('versions', max)
class ExtensionMethodWrapper:
def __init__(self, rpc, name):
self.rpc = rpc
self.name = name
def call(self, *a, **kwa):
return self.rpc.call(self.name, *a, **kwa)
lib/python/ZEO/StorageServer.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The StorageServer class and the exception that it may raise.
This server acts as a front-end for one or more real storages, like
file storage or Berkeley storage.
XXX Need some basic access control-- a declaration of the methods
exported for invocation by the server.
"""
import asyncore
import cPickle
import os
import sys
import threading
import time
import logging
import transaction
from ZEO import ClientStub
from ZEO.CommitLog import CommitLog
from ZEO.monitor import StorageStats, StatsServer
from ZEO.zrpc.server import Dispatcher
from ZEO.zrpc.connection import ManagedServerConnection, Delay, MTDelay
from ZEO.zrpc.trigger import trigger
from ZEO.Exceptions import AuthError
from ZODB.ConflictResolution import ResolvedSerial
from ZODB.POSException import StorageError, StorageTransactionError
from ZODB.POSException import TransactionError, ReadOnlyError, ConflictError
from ZODB.serialize import referencesf
from ZODB.utils import u64, oid_repr
from ZODB.loglevels import BLATHER
logger = logging.getLogger('ZEO.StorageServer')
# XXX This used to say "ZSS", which is now implied in the logger name, can this
# be either set to str(os.getpid()) (if that makes sense) or removed?
_label = "" # default label used for logging.
def set_label():
"""Internal helper to reset the logging label (e.g. after fork())."""
global _label
_label = "%s" % os.getpid()
def log(message, level=logging.INFO, label=None, exc_info=False):
"""Internal helper to log a message."""
label = label or _label
if label:
message = "(%s) %s" % (label, message)
logger.log(level, message, exc_info=exc_info)
class StorageServerError(StorageError):
"""Error reported when an unpickleable exception is raised."""
class ZEOStorage:
"""Proxy to underlying storage for a single remote client."""
# Classes we instantiate. A subclass might override.
ClientStorageStubClass = ClientStub.ClientStorage
# A list of extension methods. A subclass with extra methods
# should override.
extensions = []
def __init__(self, server, read_only=0, auth_realm=None):
self.server = server
# timeout and stats will be initialized in register()
self.timeout = None
self.stats = None
self.connection = None
self.client = None
self.storage = None
self.storage_id = "uninitialized"
self.transaction = None
self.read_only = read_only
self.locked = 0
self.verifying = 0
self.store_failed = 0
self.log_label = _label
self.authenticated = 0
self.auth_realm = auth_realm
# The authentication protocol may define extra methods.
self._extensions = {}
for func in self.extensions:
self._extensions[func.func_name] = None
def finish_auth(self, authenticated):
if not self.auth_realm:
return 1
self.authenticated = authenticated
return authenticated
def set_database(self, database):
self.database = database
def notifyConnected(self, conn):
self.connection = conn # For restart_other() below
self.client = self.ClientStorageStubClass(conn)
addr = conn.addr
if isinstance(addr, type("")):
label = addr
else:
host, port = addr
label = str(host) + ":" + str(port)
self.log_label = _label + "/" + label
def notifyDisconnected(self):
# When this storage closes, we must ensure that it aborts
# any pending transaction.
if self.transaction is not None:
self.log("disconnected during transaction %s" % self.transaction)
self._abort()
else:
self.log("disconnected")
if self.stats is not None:
self.stats.clients -= 1
def __repr__(self):
tid = self.transaction and repr(self.transaction.id)
if self.storage:
stid = (self.storage._transaction and
repr(self.storage._transaction.id))
else:
stid = None
name = self.__class__.__name__
return "<%s %X trans=%s s_trans=%s>" % (name, id(self), tid, stid)
def log(self, msg, level=logging.INFO, exc_info=False):
log(msg, level=level, label=self.log_label, exc_info=exc_info)
def setup_delegation(self):
"""Delegate several methods to the storage"""
self.versionEmpty = self.storage.versionEmpty
self.versions = self.storage.versions
self.getSerial = self.storage.getSerial
self.history = self.storage.history
self.load = self.storage.load
self.loadSerial = self.storage.loadSerial
self.modifiedInVersion = self.storage.modifiedInVersion
try:
fn = self.storage.getExtensionMethods
except AttributeError:
# We must be running with a ZODB which
# predates adding getExtensionMethods to
# BaseStorage. Eventually this try/except
# can be removed
pass
else:
d = fn()
self._extensions.update(d)
for name in d.keys():
assert not hasattr(self, name)
setattr(self, name, getattr(self.storage, name))
self.lastTransaction = self.storage.lastTransaction
def _check_tid(self, tid, exc=None):
if self.read_only:
raise ReadOnlyError()
if self.transaction is None:
caller = sys._getframe().f_back.f_code.co_name
self.log("no current transaction: %s()" % caller,
level=logging.WARNING)
if exc is not None:
raise exc(None, tid)
else:
return 0
if self.transaction.id != tid:
caller = sys._getframe().f_back.f_code.co_name
self.log("%s(%s) invalid; current transaction = %s" %
(caller, repr(tid), repr(self.transaction.id)),
logging.WARNING)
if exc is not None:
raise exc(self.transaction.id, tid)
else:
return 0
return 1
def getAuthProtocol(self):
"""Return string specifying name of authentication module to use.
The module name should be auth_%s where %s is auth_protocol."""
protocol = self.server.auth_protocol
if not protocol or protocol == 'none':
return None
return protocol
def register(self, storage_id, read_only):
"""Select the storage that this client will use
This method must be the first one called by the client.
For authenticated storages this method will be called by the client
immediately after authentication is finished.
"""
if self.auth_realm and not self.authenticated:
raise AuthError, "Client was never authenticated with server!"
if self.storage is not None:
self.log("duplicate register() call")
raise ValueError, "duplicate register() call"
storage = self.server.storages.get(storage_id)
if storage is None:
self.log("unknown storage_id: %s" % storage_id)
raise ValueError, "unknown storage: %s" % storage_id
if not read_only and (self.read_only or storage.isReadOnly()):
raise ReadOnlyError()
self.read_only = self.read_only or read_only
self.storage_id = storage_id
self.storage = storage
self.setup_delegation()
self.timeout, self.stats = self.server.register_connection(storage_id,
self)
def get_info(self):
return {'length': len(self.storage),
'size': self.storage.getSize(),
'name': self.storage.getName(),
'supportsUndo': self.storage.supportsUndo(),
'supportsVersions': self.storage.supportsVersions(),
'extensionMethods': self.getExtensionMethods(),
}
def get_size_info(self):
return {'length': len(self.storage),
'size': self.storage.getSize(),
}
def getExtensionMethods(self):
return self._extensions
def loadEx(self, oid, version):
self.stats.loads += 1
return self.storage.loadEx(oid, version)
def loadBefore(self, oid, tid):
self.stats.loads += 1
return self.storage.loadBefore(oid, tid)
def zeoLoad(self, oid):
self.stats.loads += 1
v = self.storage.modifiedInVersion(oid)
if v:
pv, sv = self.storage.load(oid, v)
else:
pv = sv = None
try:
p, s = self.storage.load(oid, '')
except KeyError:
if sv:
# Created in version, no non-version data
p = s = None
else:
raise
return p, s, v, pv, sv
def getInvalidations(self, tid):
invtid, invlist = self.server.get_invalidations(tid)
if invtid is None:
return None
self.log("Return %d invalidations up to tid %s"
% (len(invlist), u64(invtid)))
return invtid, invlist
def verify(self, oid, version, tid):
try:
t = self.storage.getTid(oid)
except KeyError:
self.client.invalidateVerify((oid, ""))
else:
if tid != t:
# This will invalidate non-version data when the
# client only has invalid version data. Since this is
# an uncommon case, we avoid the cost of checking
# whether the serial number matches the current
# non-version data.
self.client.invalidateVerify((oid, version))
def zeoVerify(self, oid, s, sv):
if not self.verifying:
self.verifying = 1
self.stats.verifying_clients += 1
try:
os = self.storage.getTid(oid)
except KeyError:
self.client.invalidateVerify((oid, ''))
# XXX It's not clear what we should do now. The KeyError
# could be caused by an object uncreation, in which case
# invalidation is right. It could be an application bug
# that left a dangling reference, in which case it's bad.
else:
# If the client has version data, the logic is a bit more
# complicated. If the current serial number matches the
# client serial number, then the non-version data must
# also be valid. If the current serialno is for a
# version, then the non-version data can't change.
# If the version serialno isn't valid, then the
# non-version serialno may or may not be valid. Rather
# than trying to figure it whether it is valid, we just
# invalidate it. Sending an invalidation for the
# non-version data implies invalidating the version data
# too, since an update to non-version data can only occur
# after the version is aborted or committed.
if sv:
if sv != os:
self.client.invalidateVerify((oid, ''))
else:
if s != os:
self.client.invalidateVerify((oid, ''))
def endZeoVerify(self):
if self.verifying:
self.stats.verifying_clients -= 1
self.verifying = 0
self.client.endVerify()
def pack(self, time, wait=1):
# Yes, you can pack a read-only server or storage!
if wait:
return run_in_thread(self._pack_impl, time)
else:
# If the client isn't waiting for a reply, start a thread
# and forget about it.
t = threading.Thread(target=self._pack_impl, args=(time,))
t.start()
return None
def _pack_impl(self, time):
self.log("pack(time=%s) started..." % repr(time))
self.storage.pack(time, referencesf)
self.log("pack(time=%s) complete" % repr(time))
# Broadcast new size statistics
self.server.invalidate(0, self.storage_id, None,
(), self.get_size_info())
def new_oids(self, n=100):
"""Return a sequence of n new oids, where n defaults to 100"""
if self.read_only:
raise ReadOnlyError()
if n <= 0:
n = 1
return [self.storage.new_oid() for i in range(n)]
# undoLog and undoInfo are potentially slow methods
def undoInfo(self, first, last, spec):
return run_in_thread(self.storage.undoInfo, first, last, spec)
def undoLog(self, first, last):
return run_in_thread(self.storage.undoLog, first, last)
def tpc_begin(self, id, user, description, ext, tid=None, status=" "):
if self.read_only:
raise ReadOnlyError()
if self.transaction is not None:
if self.transaction.id == id:
self.log("duplicate tpc_begin(%s)" % repr(id))
return
else:
raise StorageTransactionError("Multiple simultaneous tpc_begin"
" requests from one client.")
self.transaction = t = transaction.Transaction()
t.id = id
t.user = user
t.description = description
t._extension = ext
self.serials = []
self.invalidated = []
self.txnlog = CommitLog()
self.tid = tid
self.status = status
self.store_failed = 0
self.stats.active_txns += 1
def tpc_finish(self, id):
if not self._check_tid(id):
return
assert self.locked
self.stats.active_txns -= 1
self.stats.commits += 1
self.storage.tpc_finish(self.transaction)
tid = self.storage.lastTransaction()
if self.invalidated:
self.server.invalidate(self, self.storage_id, tid,
self.invalidated, self.get_size_info())
self._clear_transaction()
# Return the tid, for cache invalidation optimization
return tid
def tpc_abort(self, id):
if not self._check_tid(id):
return
self.stats.active_txns -= 1
self.stats.aborts += 1
if self.locked:
self.storage.tpc_abort(self.transaction)
self._clear_transaction()
def _clear_transaction(self):
# Common code at end of tpc_finish() and tpc_abort()
self.transaction = None
self.txnlog.close()
if self.locked:
self.locked = 0
self.timeout.end(self)
self.stats.lock_time = None
self.log("Transaction released storage lock", BLATHER)
# _handle_waiting() can start another transaction (by
# restarting a waiting one) so must be done last
self._handle_waiting()
def _abort(self):
# called when a connection is closed unexpectedly
if not self.locked:
# Delete (d, zeo_storage) from the _waiting list, if found.
waiting = self.storage._waiting
for i in range(len(waiting)):
d, z = waiting[i]
if z is self:
del waiting[i]
self.log("Closed connection removed from waiting list."
" Clients waiting: %d." % len(waiting))
break
if self.transaction:
self.stats.active_txns -= 1
self.stats.aborts += 1
self.tpc_abort(self.transaction.id)
# The public methods of the ZEO client API do not do the real work.
# They defer work until after the storage lock has been acquired.
# Most of the real implementations are in methods beginning with
# an _.
def storea(self, oid, serial, data, version, id):
self._check_tid(id, exc=StorageTransactionError)
self.stats.stores += 1
self.txnlog.store(oid, serial, data, version)
# The following four methods return values, so they must acquire
# the storage lock and begin the transaction before returning.
def vote(self, id):
self._check_tid(id, exc=StorageTransactionError)
if self.locked:
return self._vote()
else:
return self._wait(lambda: self._vote())
def abortVersion(self, src, id):
self._check_tid(id, exc=StorageTransactionError)
if self.locked:
return self._abortVersion(src)
else:
return self._wait(lambda: self._abortVersion(src))
def commitVersion(self, src, dest, id):
self._check_tid(id, exc=StorageTransactionError)
if self.locked:
return self._commitVersion(src, dest)
else:
return self._wait(lambda: self._commitVersion(src, dest))
def undo(self, trans_id, id):
self._check_tid(id, exc=StorageTransactionError)
if self.locked:
return self._undo(trans_id)
else:
return self._wait(lambda: self._undo(trans_id))
def _tpc_begin(self, txn, tid, status):
self.locked = 1
self.timeout.begin(self)
self.stats.lock_time = time.time()
self.storage.tpc_begin(txn, tid, status)
def _store(self, oid, serial, data, version):
err = None
try:
newserial = self.storage.store(oid, serial, data, version,
self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
self.store_failed = 1
if isinstance(err, ConflictError):
self.stats.conflicts += 1
self.log("conflict error oid=%s msg=%s" %
(oid_repr(oid), str(err)), BLATHER)
if not isinstance(err, TransactionError):
# Unexpected errors are logged and passed to the client
self.log("store error: %s, %s" % sys.exc_info()[:2],
logging.ERROR, exc_info=True)
# Try to pickle the exception. If it can't be pickled,
# the RPC response would fail, so use something else.
pickler = cPickle.Pickler()
pickler.fast = 1
try:
pickler.dump(err, 1)
except:
msg = "Couldn't pickle storage exception: %s" % repr(err)
self.log(msg, logging.ERROR)
err = StorageServerError(msg)
# The exception is reported back as newserial for this oid
newserial = err
else:
if serial != "\0\0\0\0\0\0\0\0":
self.invalidated.append((oid, version))
if newserial == ResolvedSerial:
self.stats.conflicts_resolved += 1
self.log("conflict resolved oid=%s" % oid_repr(oid), BLATHER)
self.serials.append((oid, newserial))
return err is None
def _vote(self):
self.client.serialnos(self.serials)
# If a store call failed, then return to the client immediately.
# The serialnos() call will deliver an exception that will be
# handled by the client in its tpc_vote() method.
if self.store_failed:
return
return self.storage.tpc_vote(self.transaction)
def _abortVersion(self, src):
tid, oids = self.storage.abortVersion(src, self.transaction)
inv = [(oid, src) for oid in oids]
self.invalidated.extend(inv)
return tid, oids
def _commitVersion(self, src, dest):
tid, oids = self.storage.commitVersion(src, dest, self.transaction)
inv = [(oid, dest) for oid in oids]
self.invalidated.extend(inv)
if dest:
inv = [(oid, src) for oid in oids]
self.invalidated.extend(inv)
return tid, oids
def _undo(self, trans_id):
tid, oids = self.storage.undo(trans_id, self.transaction)
inv = [(oid, None) for oid in oids]
self.invalidated.extend(inv)
return tid, oids
# When a delayed transaction is restarted, the dance is
# complicated. The restart occurs when one ZEOStorage instance
# finishes as a transaction and finds another instance is in the
# _waiting list.
# XXX It might be better to have a mechanism to explicitly send
# the finishing transaction's reply before restarting the waiting
# transaction. If the restart takes a long time, the previous
# client will be blocked until it finishes.
def _wait(self, thunk):
# Wait for the storage lock to be acquired.
self._thunk = thunk
if self.storage._transaction:
d = Delay()
self.storage._waiting.append((d, self))
self.log("Transaction blocked waiting for storage. "
"Clients waiting: %d." % len(self.storage._waiting))
return d
else:
self.log("Transaction acquired storage lock.", BLATHER)
return self._restart()
def _restart(self, delay=None):
# Restart when the storage lock is available.
if self.txnlog.stores == 1:
template = "Preparing to commit transaction: %d object, %d bytes"
else:
template = "Preparing to commit transaction: %d objects, %d bytes"
self.log(template % (self.txnlog.stores, self.txnlog.size()),
level=BLATHER)
self._tpc_begin(self.transaction, self.tid, self.status)
loads, loader = self.txnlog.get_loader()
for i in range(loads):
# load oid, serial, data, version
if not self._store(*loader.load()):
break
resp = self._thunk()
if delay is not None:
delay.reply(resp)
else:
return resp
def _handle_waiting(self):
# Restart any client waiting for the storage lock.
while self.storage._waiting:
delay, zeo_storage = self.storage._waiting.pop(0)
if self._restart_other(zeo_storage, delay):
if self.storage._waiting:
n = len(self.storage._waiting)
self.log("Blocked transaction restarted. "
"Clients waiting: %d" % n)
else:
self.log("Blocked transaction restarted.")
return
def _restart_other(self, zeo_storage, delay):
# Return True if the server restarted.
# call the restart() method on the appropriate server.
try:
zeo_storage._restart(delay)
except:
self.log("Unexpected error handling waiting transaction",
level=logging.WARNING, exc_info=True)
zeo_storage.connection.close()
return 0
else:
return 1
class StorageServer:
"""The server side implementation of ZEO.
The StorageServer is the 'manager' for incoming connections. Each
connection is associated with its own ZEOStorage instance (defined
below). The StorageServer may handle multiple storages; each
ZEOStorage instance only handles a single storage.
"""
# Classes we instantiate. A subclass might override.
DispatcherClass = Dispatcher
ZEOStorageClass = ZEOStorage
ManagedServerConnectionClass = ManagedServerConnection
def __init__(self, addr, storages, read_only=0,
invalidation_queue_size=100,
transaction_timeout=None,
monitor_address=None,
auth_protocol=None,
auth_database=None,
auth_realm=None):
"""StorageServer constructor.
This is typically invoked from the start.py script.
Arguments (the first two are required and positional):
addr -- the address at which the server should listen. This
can be a tuple (host, port) to signify a TCP/IP connection
or a pathname string to signify a Unix domain socket
connection. A hostname may be a DNS name or a dotted IP
address.
storages -- a dictionary giving the storage(s) to handle. The
keys are the storage names, the values are the storage
instances, typically FileStorage or Berkeley storage
instances. By convention, storage names are typically
strings representing small integers starting at '1'.
read_only -- an optional flag saying whether the server should
operate in read-only mode. Defaults to false. Note that
even if the server is operating in writable mode,
individual storages may still be read-only. But if the
server is in read-only mode, no write operations are
allowed, even if the storages are writable. Note that
pack() is considered a read-only operation.
invalidation_queue_size -- The storage server keeps a queue
of the objects modified by the last N transactions, where
N == invalidation_queue_size. This queue is used to
speed client cache verification when a client disconnects
for a short period of time.
transaction_timeout -- The maximum amount of time to wait for
a transaction to commit after acquiring the storage lock.
If the transaction takes too long, the client connection
will be closed and the transaction aborted.
monitor_address -- The address at which the monitor server
should listen. If specified, a monitor server is started.
The monitor server provides server statistics in a simple
text format.
auth_protocol -- The name of the authentication protocol to use.
Examples are "digest" and "srp".
auth_database -- The name of the password database filename.
It should be in a format compatible with the authentication
protocol used; for instance, "sha" and "srp" require different
formats.
Note that to implement an authentication protocol, a server
and client authentication mechanism must be implemented in a
auth_* module, which should be stored inside the "auth"
subdirectory. This module may also define a DatabaseClass
variable that should indicate what database should be used
by the authenticator.
"""
self.addr = addr
self.storages = storages
set_label()
msg = ", ".join(
["%s:%s:%s" % (name, storage.isReadOnly() and "RO" or "RW",
storage.getName())
for name, storage in storages.items()])
log("%s created %s with storages: %s" %
(self.__class__.__name__, read_only and "RO" or "RW", msg))
for s in storages.values():
s._waiting = []
self.read_only = read_only
self.auth_protocol = auth_protocol
self.auth_database = auth_database
self.auth_realm = auth_realm
self.database = None
if auth_protocol:
self._setup_auth(auth_protocol)
# A list of at most invalidation_queue_size invalidations.
# The list is kept in sorted order with the most recent
# invalidation at the front. The list never has more than
# self.invq_bound elements.
self.invq = []
self.invq_bound = invalidation_queue_size
self.connections = {}
self.dispatcher = self.DispatcherClass(addr,
factory=self.new_connection)
self.stats = {}
self.timeouts = {}
for name in self.storages.keys():
self.stats[name] = StorageStats()
if transaction_timeout is None:
# An object with no-op methods
timeout = StubTimeoutThread()
else:
timeout = TimeoutThread(transaction_timeout)
timeout.start()
self.timeouts[name] = timeout
if monitor_address:
self.monitor = StatsServer(monitor_address, self.stats)
else:
self.monitor = None
def _setup_auth(self, protocol):
# Can't be done in global scope, because of cyclic references
from ZEO.auth import get_module
name = self.__class__.__name__
module = get_module(protocol)
if not module:
log("%s: no such an auth protocol: %s" % (name, protocol))
return
storage_class, client, db_class = module
if not storage_class or not issubclass(storage_class, ZEOStorage):
log(("%s: %s isn't a valid protocol, must have a StorageClass" %
(name, protocol)))
self.auth_protocol = None
return
self.ZEOStorageClass = storage_class
log("%s: using auth protocol: %s" % (name, protocol))
# We create a Database instance here for use with the authenticator
# modules. Having one instance allows it to be shared between multiple
# storages, avoiding the need to bloat each with a new authenticator
# Database that would contain the same info, and also avoiding any
# possibly synchronization issues between them.
self.database = db_class(self.auth_database)
if self.database.realm != self.auth_realm:
raise ValueError("password database realm %r "
"does not match storage realm %r"
% (self.database.realm, self.auth_realm))
def new_connection(self, sock, addr):
"""Internal: factory to create a new connection.
This is called by the Dispatcher class in ZEO.zrpc.server
whenever accept() returns a socket for a new incoming
connection.
"""
if self.auth_protocol and self.database:
zstorage = self.ZEOStorageClass(self, self.read_only,
auth_realm=self.auth_realm)
zstorage.set_database(self.database)
else:
zstorage = self.ZEOStorageClass(self, self.read_only)
c = self.ManagedServerConnectionClass(sock, addr, zstorage, self)
log("new connection %s: %s" % (addr, repr(c)))
return c
def register_connection(self, storage_id, conn):
"""Internal: register a connection with a particular storage.
This is called by ZEOStorage.register().
The dictionary self.connections maps each storage name to a
list of current connections for that storage; this information
is needed to handle invalidation. This function updates this
dictionary.
Returns the timeout and stats objects for the appropriate storage.
"""
l = self.connections.get(storage_id)
if l is None:
l = self.connections[storage_id] = []
l.append(conn)
stats = self.stats[storage_id]
stats.clients += 1
return self.timeouts[storage_id], stats
def invalidate(self, conn, storage_id, tid, invalidated=(), info=None):
"""Internal: broadcast info and invalidations to clients.
This is called from several ZEOStorage methods.
This can do three different things:
- If the invalidated argument is non-empty, it broadcasts
invalidateTransaction() messages to all clients of the given
storage except the current client (the conn argument).
- If the invalidated argument is empty and the info argument
is a non-empty dictionary, it broadcasts info() messages to
all clients of the given storage, including the current
client.
- If both the invalidated argument and the info argument are
non-empty, it broadcasts invalidateTransaction() messages to all
clients except the current, and sends an info() message to
the current client.
"""
if invalidated:
if len(self.invq) >= self.invq_bound:
self.invq.pop()
self.invq.insert(0, (tid, invalidated))
for p in self.connections.get(storage_id, ()):
if invalidated and p is not conn:
p.client.invalidateTransaction(tid, invalidated)
elif info is not None:
p.client.info(info)
def get_invalidations(self, tid):
"""Return a tid and list of all objects invalidation since tid.
The tid is the most recent transaction id seen by the client.
Returns None if it is unable to provide a complete list
of invalidations for tid. In this case, client should
do full cache verification.
"""
if not self.invq:
log("invq empty")
return None, []
earliest_tid = self.invq[-1][0]
if earliest_tid > tid:
log("tid to old for invq %s < %s" % (u64(tid), u64(earliest_tid)))
return None, []
oids = {}
for _tid, L in self.invq:
if _tid <= tid:
break
for key in L:
oids[key] = 1
latest_tid = self.invq[0][0]
return latest_tid, oids.keys()
def close_server(self):
"""Close the dispatcher so that there are no new connections.
This is only called from the test suite, AFAICT.
"""
self.dispatcher.close()
if self.monitor is not None:
self.monitor.close()
for storage in self.storages.values():
storage.close()
# Force the asyncore mainloop to exit by hackery, i.e. close
# every socket in the map. loop() will return when the map is
# empty.
for s in asyncore.socket_map.values():
try:
s.close()
except:
pass
def close_conn(self, conn):
"""Internal: remove the given connection from self.connections.
This is the inverse of register_connection().
"""
for cl in self.connections.values():
if conn.obj in cl:
cl.remove(conn.obj)
class StubTimeoutThread:
def begin(self, client):
pass
def end(self, client):
pass
class TimeoutThread(threading.Thread):
"""Monitors transaction progress and generates timeouts."""
# There is one TimeoutThread per storage, because there's one
# transaction lock per storage.
def __init__(self, timeout):
threading.Thread.__init__(self)
self.setDaemon(1)
self._timeout = timeout
self._client = None
self._deadline = None
self._cond = threading.Condition() # Protects _client and _deadline
self._trigger = trigger()
def begin(self, client):
# Called from the restart code the "main" thread, whenever the
# storage lock is being acquired. (Serialized by asyncore.)
self._cond.acquire()
try:
assert self._client is None
self._client = client
self._deadline = time.time() + self._timeout
self._cond.notify()
finally:
self._cond.release()
def end(self, client):
# Called from the "main" thread whenever the storage lock is
# being released. (Serialized by asyncore.)
self._cond.acquire()
try:
assert self._client is not None
assert self._client is client
self._client = None
self._deadline = None
finally:
self._cond.release()
def run(self):
# Code running in the thread.
while 1:
self._cond.acquire()
try:
while self._deadline is None:
self._cond.wait()
howlong = self._deadline - time.time()
if howlong <= 0:
# Prevent reporting timeout more than once
self._deadline = None
client = self._client # For the howlong <= 0 branch below
finally:
self._cond.release()
if howlong <= 0:
client.log("Transaction timeout after %s seconds" %
self._timeout)
self._trigger.pull_trigger(lambda: client.connection.close())
else:
time.sleep(howlong)
def run_in_thread(method, *args):
t = SlowMethodThread(method, args)
t.start()
return t.delay
class SlowMethodThread(threading.Thread):
"""Thread to run potentially slow storage methods.
Clients can use the delay attribute to access the MTDelay object
used to send a zrpc response at the right time.
"""
# Some storage methods can take a long time to complete. If we
# run these methods via a standard asyncore read handler, they
# will block all other server activity until they complete. To
# avoid blocking, we spawn a separate thread, return an MTDelay()
# object, and have the thread reply() when it finishes.
def __init__(self, method, args):
threading.Thread.__init__(self)
self._method = method
self._args = args
self.delay = MTDelay()
def run(self):
try:
result = self._method(*self._args)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
self.delay.error(sys.exc_info())
else:
self.delay.reply(result)
lib/python/ZEO/TransactionBuffer.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A TransactionBuffer store transaction updates until commit or abort.
A transaction may generate enough data that it is not practical to
always hold pending updates in memory. Instead, a TransactionBuffer
is used to store the data until a commit or abort.
"""
# A faster implementation might store trans data in memory until it
# reaches a certain size.
import cPickle
import tempfile
from threading import Lock
class TransactionBuffer:
# Valid call sequences:
#
# ((store | invalidate)* begin_iterate next* clear)* close
#
# get_size can be called any time
# The TransactionBuffer is used by client storage to hold update
# data until the tpc_finish(). It is normally used by a single
# thread, because only one thread can be in the two-phase commit
# at one time.
# It is possible, however, for one thread to close the storage
# while another thread is in the two-phase commit. We must use
# a lock to guard against this race, because unpredictable things
# can happen in Python if one thread closes a file that another
# thread is reading. In a debug build, an assert() can fail.
# XXX If an operation is performed on a closed TransactionBuffer,
# it has no effect and does not raise an exception. The only time
# this should occur is when a ClientStorage is closed in one
# thread while another thread is in its tpc_finish(). It's not
# clear what should happen in this case. If the tpc_finish()
# completes without error, the Connection using it could have
# inconsistent data. This should have minimal effect, though,
# because the Connection is connected to a closed storage.
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".tbuf")
self.lock = Lock()
self.closed = 0
self.count = 0
self.size = 0
# It's safe to use a fast pickler because the only objects
# stored are builtin types -- strings or None.
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
def close(self):
self.lock.acquire()
try:
self.closed = 1
try:
self.file.close()
except OSError:
pass
finally:
self.lock.release()
def store(self, oid, version, data):
self.lock.acquire()
try:
self._store(oid, version, data)
finally:
self.lock.release()
def _store(self, oid, version, data):
"""Store oid, version, data for later retrieval"""
if self.closed:
return
self.pickler.dump((oid, version, data))
self.count += 1
# Estimate per-record cache size
self.size = self.size + len(data) + 31
if version:
# Assume version data has same size as non-version data
self.size = self.size + len(version) + len(data) + 12
def invalidate(self, oid, version):
self.lock.acquire()
try:
if self.closed:
return
self.pickler.dump((oid, version, None))
self.count += 1
finally:
self.lock.release()
def clear(self):
"""Mark the buffer as empty"""
self.lock.acquire()
try:
if self.closed:
return
self.file.seek(0)
self.count = 0
self.size = 0
finally:
self.lock.release()
def __iter__(self):
self.lock.acquire()
try:
if self.closed:
return
self.file.flush()
self.file.seek(0)
return TBIterator(self.file, self.count)
finally:
self.lock.release()
class TBIterator(object):
def __init__(self, f, count):
self.file = f
self.count = count
self.unpickler = cPickle.Unpickler(f)
def __iter__(self):
return self
def next(self):
"""Return next tuple of data or None if EOF"""
if self.count == 0:
self.file.seek(0)
self.size = 0
raise StopIteration
oid_ver_data = self.unpickler.load()
self.count -= 1
return oid_ver_data
lib/python/ZEO/__init__.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZEO -- Zope Enterprise Objects.
See the file README.txt in this directory for an overview.
ZEO is now part of ZODB; ZODB's home on the web is
http://www.zope.org/Wikis/ZODB
"""
# The next line must use double quotes, so replace.py recognizes it.
version = "2.3a3"
lib/python/ZEO/auth/__init__.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
_auth_modules = {}
def get_module(name):
if name == 'sha':
from auth_sha import StorageClass, SHAClient, Database
return StorageClass, SHAClient, Database
elif name == 'digest':
from auth_digest import StorageClass, DigestClient, DigestDatabase
return StorageClass, DigestClient, DigestDatabase
else:
return _auth_modules.get(name)
def register_module(name, storage_class, client, db):
if _auth_modules.has_key(name):
raise TypeError, "%s is already registred" % name
_auth_modules[name] = storage_class, client, db
lib/python/ZEO/auth/auth_digest.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Digest authentication for ZEO
This authentication mechanism follows the design of HTTP digest
authentication (RFC 2069). It is a simple challenge-response protocol
that does not send passwords in the clear, but does not offer strong
security. The RFC discusses many of the limitations of this kind of
protocol.
Guard the password database as if it contained plaintext passwords.
It stores the hash of a username and password. This does not expose
the plaintext password, but it is sensitive nonetheless. An attacker
with the hash can impersonate the real user. This is a limitation of
the simple digest scheme.
HTTP is a stateless protocol, and ZEO is a stateful protocol. The
security requirements are quite different as a result. The HTTP
protocol uses a nonce as a challenge. The ZEO protocol requires a
separate session key that is used for message authentication. We
generate a second nonce for this purpose; the hash of nonce and
user/realm/password is used as the session key. XXX I'm not sure if
this is a sound approach; SRP would be preferred.
"""
import os
import random
import sha
import struct
import time
from ZEO.auth.base import Database, Client
from ZEO.StorageServer import ZEOStorage
from ZEO.Exceptions import AuthError
def get_random_bytes(n=8):
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
s = f.read(n)
f.close()
else:
L = [chr(random.randint(0, 255)) for i in range(n)]
s = "".join(L)
return s
def hexdigest(s):
return sha.new(s).hexdigest()
class DigestDatabase(Database):
def __init__(self, filename, realm=None):
Database.__init__(self, filename, realm)
# Initialize a key used to build the nonce for a challenge.
# We need one key for the lifetime of the server, so it
# is convenient to store in on the database.
self.noncekey = get_random_bytes(8)
def _store_password(self, username, password):
dig = hexdigest("%s:%s:%s" % (username, self.realm, password))
self._users[username] = dig
def session_key(h_up, nonce):
# The hash itself is a bit too short to be a session key.
# HMAC wants a 64-byte key. We don't want to use h_up
# directly because it would never change over time. Instead
# use the hash plus part of h_up.
return sha.new("%s:%s" % (h_up, nonce)).digest() + h_up[:44]
class StorageClass(ZEOStorage):
def set_database(self, database):
assert isinstance(database, DigestDatabase)
self.database = database
self.noncekey = database.noncekey
def _get_time(self):
# Return a string representing the current time.
t = int(time.time())
return struct.pack("i", t)
def _get_nonce(self):
# RFC 2069 recommends a nonce of the form
# H(client-IP ":" time-stamp ":" private-key)
dig = sha.sha()
dig.update(str(self.connection.addr))
dig.update(self._get_time())
dig.update(self.noncekey)
return dig.hexdigest()
def auth_get_challenge(self):
"""Return realm, challenge, and nonce."""
self._challenge = self._get_nonce()
self._key_nonce = self._get_nonce()
return self.auth_realm, self._challenge, self._key_nonce
def auth_response(self, resp):
# verify client response
user, challenge, response = resp
# Since zrpc is a stateful protocol, we just store the nonce
# we sent to the client. It will need to generate a new
# nonce for a new connection anyway.
if self._challenge != challenge:
raise ValueError, "invalid challenge"
# lookup user in database
h_up = self.database.get_password(user)
# regeneration resp from user, password, and nonce
check = hexdigest("%s:%s" % (h_up, challenge))
if check == response:
self.connection.setSessionKey(session_key(h_up, self._key_nonce))
return self.finish_auth(check == response)
extensions = [auth_get_challenge, auth_response]
class DigestClient(Client):
extensions = ["auth_get_challenge", "auth_response"]
def start(self, username, realm, password):
_realm, challenge, nonce = self.stub.auth_get_challenge()
if _realm != realm:
raise AuthError("expected realm %r, got realm %r"
% (_realm, realm))
h_up = hexdigest("%s:%s:%s" % (username, realm, password))
resp_dig = hexdigest("%s:%s" % (h_up, challenge))
result = self.stub.auth_response((username, challenge, resp_dig))
if result:
return session_key(h_up, nonce)
else:
return None
lib/python/ZEO/auth/base.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Base classes for defining an authentication protocol.
Database -- abstract base class for password database
Client -- abstract base class for authentication client
"""
import os
import sha
class Client:
# Subclass should override to list the names of methods that
# will be called on the server.
extensions = []
def __init__(self, stub):
self.stub = stub
for m in self.extensions:
setattr(self.stub, m, self.stub.extensionMethod(m))
def sort(L):
"""Sort a list in-place and return it."""
L.sort()
return L
class Database:
"""Abstracts a password database.
This class is used both in the authentication process (via
get_password()) and by client scripts that manage the password
database file.
The password file is a simple, colon-separated text file mapping
usernames to password hashes. The hashes are SHA hex digests
produced from the password string.
"""
realm = None
def __init__(self, filename, realm=None):
"""Creates a new Database
filename: a string containing the full pathname of
the password database file. Must be readable by the user
running ZEO. Must be writeable by any client script that
accesses the database.
realm: the realm name (a string)
"""
self._users = {}
self.filename = filename
self.load()
if realm:
if self.realm and self.realm != realm:
raise ValueError, ("Specified realm %r differs from database "
"realm %r" % (realm or '', self.realm))
else:
self.realm = realm
def save(self, fd=None):
filename = self.filename
if not fd:
fd = open(filename, 'w')
if self.realm:
print >> fd, "realm", self.realm
for username in sort(self._users.keys()):
print >> fd, "%s: %s" % (username, self._users[username])
def load(self):
filename = self.filename
if not filename:
return
if not os.path.exists(filename):
return
fd = open(filename)
L = fd.readlines()
if not L:
return
if L[0].startswith("realm "):
line = L.pop(0).strip()
self.realm = line[len("realm "):]
for line in L:
username, hash = line.strip().split(":", 1)
self._users[username] = hash.strip()
def _store_password(self, username, password):
self._users[username] = self.hash(password)
def get_password(self, username):
"""Returns password hash for specified username.
Callers must check for LookupError, which is raised in
the case of a non-existent user specified."""
if not self._users.has_key(username):
raise LookupError, "No such user: %s" % username
return self._users[username]
def hash(self, s):
return sha.new(s).hexdigest()
def add_user(self, username, password):
if self._users.has_key(username):
raise LookupError, "User %s already exists" % username
self._store_password(username, password)
def del_user(self, username):
if not self._users.has_key(username):
raise LookupError, "No such user: %s" % username
del self._users[username]
def change_password(self, username, password):
if not self._users.has_key(username):
raise LookupError, "No such user: %s" % username
self._store_password(username, password)
lib/python/ZEO/auth/hmac.py 0 → 100644
View file @ 96172040
"""HMAC (Keyed-Hashing for Message Authentication) Python module.
Implements the HMAC algorithm as described by RFC 2104.
"""
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
"""
return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
# The size of the digests returned by HMAC depends on the underlying
# hashing module used.
digest_size = None
class HMAC:
"""RFC2104 HMAC class.
This supports the API for Cryptographic Hash Functions (PEP 247).
"""
def __init__(self, key, msg = None, digestmod = None):
"""Create a new HMAC object.
key: key for the keyed hash object.
msg: Initial input for the hash, if provided.
digestmod: A module supporting PEP 247. Defaults to the md5 module.
"""
if digestmod is None:
import md5
digestmod = md5
self.digestmod = digestmod
self.outer = digestmod.new()
self.inner = digestmod.new()
self.digest_size = digestmod.digest_size
blocksize = 64
ipad = "\x36" * blocksize
opad = "\x5C" * blocksize
if len(key) > blocksize:
key = digestmod.new(key).digest()
key = key + chr(0) * (blocksize - len(key))
self.outer.update(_strxor(key, opad))
self.inner.update(_strxor(key, ipad))
if msg is not None:
self.update(msg)
## def clear(self):
## raise NotImplementedError, "clear() method not available in HMAC."
def update(self, msg):
"""Update this hashing object with the string msg.
"""
self.inner.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
An update to this copy won't affect the original object.
"""
other = HMAC("")
other.digestmod = self.digestmod
other.inner = self.inner.copy()
other.outer = self.outer.copy()
return other
def digest(self):
"""Return the hash value of this hashing object.
This returns a string containing 8-bit data. The object is
not altered in any way by this function; you can continue
updating the object after calling this function.
"""
h = self.outer.copy()
h.update(self.inner.digest())
return h.digest()
def hexdigest(self):
"""Like digest(), but returns a string of hexadecimal digits instead.
"""
return "".join([hex(ord(x))[2:].zfill(2)
for x in tuple(self.digest())])
def new(key, msg = None, digestmod = None):
"""Create a new hashing object and return it.
key: The starting key for the hash.
msg: if available, will immediately be hashed into the object's starting
state.
You can now feed arbitrary strings into the object using its update()
method, and can ask for the hash value at any time by calling its digest()
method.
"""
return HMAC(key, msg, digestmod)
lib/python/ZEO/cache.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Disk-based client cache for ZEO.
ClientCache exposes an API used by the ZEO client storage. FileCache
stores objects one disk using a 2-tuple of oid and tid as key.
The upper cache's API is similar to a storage API with methods like
load(), store(), and invalidate(). It manages in-memory data
structures that allow it to map this richer API onto the simple
key-based API of the lower-level cache.
"""
import bisect
import logging
import os
import struct
import tempfile
import time
from ZODB.utils import z64, u64
##
# A disk-based cache for ZEO clients.
# <p>
# This class provides an interface to a persistent, disk-based cache
# used by ZEO clients to store copies of database records from the
# server.
# <p>
# The details of the constructor as unspecified at this point.
# <p>
# Each entry in the cache is valid for a particular range of transaction
# ids. The lower bound is the transaction that wrote the data. The
# upper bound is the next transaction that wrote a revision of the
# object. If the data is current, the upper bound is stored as None;
# the data is considered current until an invalidate() call is made.
# <p>
# It is an error to call store() twice with the same object without an
# intervening invalidate() to set the upper bound on the first cache
# entry. <em>Perhaps it will be necessary to have a call the removes
# something from the cache outright, without keeping a non-current
# entry.</em>
# <h3>Cache verification</h3>
# <p>
# When the client is connected to the server, it receives
# invalidations every time an object is modified. Whe the client is
# disconnected, it must perform cache verification to make sure its
# cached data is synchronized with the storage's current state.
# <p>
# quick verification
# full verification
# <p>
class ClientCache:
"""A simple in-memory cache."""
##
# Do we put the constructor here?
# @param path path of persistent snapshot of cache state
# @param size maximum size of object data, in bytes
def __init__(self, path=None, size=None, trace=False):
self.path = path
self.size = size
self.log = logging.getLogger("zeo.cache")
if trace and path:
self._setup_trace()
else:
self._trace = self._notrace
# Last transaction seen by the cache, either via setLastTid()
# or by invalidate().
self.tid = None
# The cache stores objects in a dict mapping (oid, tid) pairs
# to Object() records (see below). The tid is the transaction
# id that wrote the object. An object record includes data,
# serialno, and end tid. It has auxillary data structures to
# compute the appropriate tid, given the oid and a transaction id
# representing an arbitrary point in history.
#
# The serialized form of the cache just stores the Object()
# records. The in-memory form can be reconstructed from these
# records.
# Maps oid to current tid. Used to find compute key for objects.
self.current = {}
# Maps oid to list of (start_tid, end_tid) pairs in sorted order.
# Used to find matching key for load of non-current data.
self.noncurrent = {}
# Map oid to version, tid pair. If there is no entry, the object
# is not modified in a version.
self.version = {}
# A double-linked list is used to manage the cache. It makes
# decisions about which objects to keep and which to evict.
self.fc = FileCache(size or 10**6, self.path, self)
def open(self):
self.fc.scan(self.install)
def install(self, f, ent):
# Called by cache storage layer to insert object
o = Object.fromFile(f, ent.key, header_only=True)
if o is None:
return
oid = o.key[0]
if o.version:
self.version[oid] = o.version, o.start_tid
elif o.end_tid is None:
self.current[oid] = o.start_tid
else:
L = self.noncurrent.setdefault(oid, [])
bisect.insort_left(L, (o.start_tid, o.end_tid))
def close(self):
self.fc.close()
##
# Set the last transaction seen by the cache.
# @param tid a transaction id
# @exception ValueError attempt to set a new tid less than the current tid
def setLastTid(self, tid):
self.fc.settid(tid)
##
# Return the last transaction seen by the cache.
# @return a transaction id
# @defreturn string
def getLastTid(self):
if self.fc.tid == z64:
return None
else:
return self.fc.tid
##
# Return the current data record for oid and version.
# @param oid object id
# @param version a version string
# @return data record, serial number, tid or None if the object is not
# in the cache
# @defreturn 3-tuple: (string, string, string)
def load(self, oid, version=""):
tid = None
if version:
p = self.version.get(oid)
if p is None:
return None
elif p[0] == version:
tid = p[1]
# Otherwise, we know the cache has version data but not
# for the requested version. Thus, we know it is safe
# to return the non-version data from the cache.
if tid is None:
tid = self.current.get(oid)
if tid is None:
self._trace(0x20, oid, version)
return None
o = self.fc.access((oid, tid))
if o is None:
return None
self._trace(0x22, oid, version, o.start_tid, o.end_tid, len(o.data))
return o.data, tid, o.version
##
# Return a non-current revision of oid that was current before tid.
# @param oid object id
# @param tid id of transaction that wrote next revision of oid
# @return data record, serial number, start tid, and end tid
# @defreturn 4-tuple: (string, string, string, string)
def loadBefore(self, oid, tid):
L = self.noncurrent.get(oid)
if L is None:
self._trace(0x24, oid, tid)
return None
# A pair with None as the second element will always be less
# than any pair with the same first tid.
i = bisect.bisect_left(L, (tid, None))
# The least element left of tid was written before tid. If
# there is no element, the cache doesn't have old enough data.
if i == 0:
self._trace(0x24, oid, tid)
return
lo, hi = L[i-1]
# XXX lo should always be less than tid
if not lo < tid <= hi:
self._trace(0x24, oid, tid)
return None
o = self.fc.access((oid, lo))
self._trace(0x26, oid, tid)
return o.data, o.start_tid, o.end_tid
##
# Return the version an object is modified in or None for an
# object that is not modified in a version.
# @param oid object id
# @return name of version in which the object is modified
# @defreturn string or None
def modifiedInVersion(self, oid):
p = self.version.get(oid)
if p is None:
return None
version, tid = p
return version
##
# Store a new data record in the cache.
# @param oid object id
# @param version name of version that oid was modified in. The cache
# only stores current version data, so end_tid should
# be None.
# @param start_tid the id of the transaction that wrote this revision
# @param end_tid the id of the transaction that created the next
# revision of oid. If end_tid is None, the data is
# current.
# @param data the actual data
# @exception ValueError tried to store non-current version data
def store(self, oid, version, start_tid, end_tid, data):
# It's hard for the client to avoid storing the same object
# more than once. One case is whether the client requests
# version data that doesn't exist. It checks the cache for
# the requested version, doesn't find it, then asks the server
# for that data. The server returns the non-version data,
# which may already by in the cache.
if (oid, start_tid) in self.fc:
return
o = Object((oid, start_tid), version, data, start_tid, end_tid)
if version:
if end_tid is not None:
raise ValueError("cache only stores current version data")
if oid in self.version:
if self.version[oid] != (version, start_tid):
raise ValueError("data already exists for version %r"
% self.version[oid][0])
self.version[oid] = version, start_tid
self._trace(0x50, oid, version, start_tid, dlen=len(data))
else:
if end_tid is None:
_cur_start = self.current.get(oid)
if _cur_start:
if _cur_start != start_tid:
raise ValueError(
"already have current data for oid")
else:
return
self.current[oid] = start_tid
self._trace(0x52, oid, version, start_tid, dlen=len(data))
else:
L = self.noncurrent.setdefault(oid, [])
p = start_tid, end_tid
if p in L:
return # duplicate store
bisect.insort_left(L, (start_tid, end_tid))
self._trace(0x54, oid, version, start_tid, end_tid,
dlen=len(data))
self.fc.add(o)
##
# Mark the current data for oid as non-current. If there is no
# current data for oid, do nothing.
# @param oid object id
# @param version name of version to invalidate.
# @param tid the id of the transaction that wrote a new revision of oid
def invalidate(self, oid, version, tid):
if tid > self.fc.tid:
self.fc.settid(tid)
if oid in self.version:
self._trace(0x1A, oid, version, tid)
dllversion, dlltid = self.version[oid]
assert not version or version == dllversion, (version, dllversion)
# remove() will call unlink() to delete from self.version
self.fc.remove((oid, dlltid))
# And continue on, we must also remove any non-version data
# from the cache. This is a bit of a failure of the current
# cache consistency approach as the new tid of the version
# data gets confused with the old tid of the non-version data.
# I could sort this out, but it seems simpler to punt and
# have the cache invalidation too much for versions.
if oid not in self.current:
self._trace(0x10, oid, version, tid)
return
cur_tid = self.current.pop(oid)
# XXX Want to fetch object without marking it as accessed
o = self.fc.access((oid, cur_tid))
if o is None:
# XXX is this possible?
return None
o.end_tid = tid
self.fc.update(o)
self._trace(0x1C, oid, version, tid)
L = self.noncurrent.setdefault(oid, [])
bisect.insort_left(L, (cur_tid, tid))
##
# Return the number of object revisions in the cache.
# XXX just return len(self.cache)?
def __len__(self):
n = len(self.current) + len(self.version)
if self.noncurrent:
n += sum(map(len, self.noncurrent))
return n
##
# Generates over, version, serial triples for all objects in the
# cache. This generator is used by cache verification.
def contents(self):
# XXX May need to materialize list instead of iterating,
# depends on whether the caller may change the cache.
for o in self.fc:
oid, tid = o.key
if oid in self.version:
obj = self.fc.access(o.key)
yield oid, tid, obj.version
else:
yield oid, tid, ""
def dump(self):
from ZODB.utils import oid_repr
print "cache size", len(self)
L = list(self.contents())
L.sort()
for oid, tid, version in L:
print oid_repr(oid), oid_repr(tid), repr(version)
print "dll contents"
L = list(self.fc)
L.sort(lambda x,y:cmp(x.key, y.key))
for x in L:
end_tid = x.end_tid or z64
print oid_repr(x.key[0]), oid_repr(x.key[1]), oid_repr(end_tid)
print
def _evicted(self, o):
# Called by Object o to signal its eviction
oid, tid = o.key
if o.end_tid is None:
if o.version:
del self.version[oid]
else:
del self.current[oid]
else:
# XXX Although we use bisect to keep the list sorted,
# we never expect the list to be very long. So the
# brute force approach should normally be fine.
L = self.noncurrent[oid]
L.remove((o.start_tid, o.end_tid))
def _setup_trace(self):
tfn = self.path + ".trace"
self.tracefile = None
try:
self.tracefile = open(tfn, "ab")
self._trace(0x00)
except IOError, msg:
self.tracefile = None
self.log.warning("Could not write to trace file %s: %s",
tfn, msg)
def _notrace(self, *arg, **kwargs):
pass
def _trace(self,
code, oid="", version="", tid="", end_tid=z64, dlen=0,
# The next two are just speed hacks.
time_time=time.time, struct_pack=struct.pack):
# The code argument is two hex digits; bits 0 and 7 must be zero.
# The first hex digit shows the operation, the second the outcome.
# If the second digit is in "02468" then it is a 'miss'.
# If it is in "ACE" then it is a 'hit'.
# This method has been carefully tuned to be as fast as possible.
# Note: when tracing is disabled, this method is hidden by a dummy.
if version:
code |= 0x80
encoded = (dlen + 255) & 0x7fffff00 | code
if tid is None:
tid = z64
if end_tid is None:
end_tid = z64
try:
self.tracefile.write(
struct_pack(">iiH8s8s",
time_time(),
encoded,
len(oid),
tid, end_tid) + oid)
except:
print `tid`, `end_tid`
raise
##
# An Object stores the cached data for a single object.
# <p>
# The cached data includes the actual object data, the key, and three
# data fields that describe the validity period and version of the
# object. The key contains the oid and a redundant start_tid. The
# actual size of an object is variable, depending on the size of the
# data and whether it is in a version.
# <p>
# The serialized format does not include the key, because it is stored
# in the header used by the cache's storage format.
class Object(object):
__slots__ = (# pair, object id, txn id -- something usable as a dict key
# the second part of the part is equal to start_tid below
"key",
"start_tid", # string, id of txn that wrote the data
"end_tid", # string, id of txn that wrote next revision
# or None
"version", # string, name of version
"data", # string, the actual data record for the object
"size", # total size of serialized object
)
def __init__(self, key, version, data, start_tid, end_tid):
self.key = key
self.version = version
self.data = data
self.start_tid = start_tid
self.end_tid = end_tid
# The size of a the serialized object on disk, include the
# 14-byte header, the length of data and version, and a
# copy of the 8-byte oid.
if data is not None:
self.size = 22 + len(data) + len(version)
# The serialization format uses an end tid of "\0" * 8, the least
# 8-byte string, to represent None. It isn't possible for an
# end_tid to be 0, because it must always be strictly greater
# than the start_tid.
fmt = ">8shi"
def serialize(self, f):
# Write standard form of Object to file, f.
self.serialize_header(f)
f.write(self.data)
f.write(self.key[0])
def serialize_header(self, f):
s = struct.pack(self.fmt, self.end_tid or "\0" * 8,
len(self.version), len(self.data))
f.write(s)
f.write(self.version)
def fromFile(cls, f, key, header_only=False):
s = f.read(struct.calcsize(cls.fmt))
if not s:
return None
oid, start_tid = key
end_tid, vlen, dlen = struct.unpack(cls.fmt, s)
if end_tid == z64:
end_tid = None
version = f.read(vlen)
if vlen != len(version):
raise ValueError("corrupted record, version")
if header_only:
data = None
else:
data = f.read(dlen)
if dlen != len(data):
raise ValueError("corrupted record, data")
s = f.read(8)
if s != oid:
raise ValueError("corrupted record, oid")
return cls((oid, start_tid), version, data, start_tid, end_tid)
fromFile = classmethod(fromFile)
def sync(f):
f.flush()
if hasattr(os, 'fsync'):
os.fsync(f.fileno())
class Entry(object):
__slots__ = (# object key -- something usable as a dict key.
'key',
# Offset from start of file to the object's data
# record; this includes all overhead bytes (status
# byte, size bytes, etc). The size of the data
# record is stored in the file near the start of the
# record, but for efficiency we also keep size in a
# dict (filemap; see later).
'offset',
)
def __init__(self, key=None, offset=None):
self.key = key
self.offset = offset
magic = "ZEC3"
OBJECT_HEADER_SIZE = 1 + 4 + 16
##
# FileCache stores a cache in a single on-disk file.
#
# On-disk cache structure
#
# The file begins with a 12-byte header. The first four bytes are the
# file's magic number - ZEC3 - indicating zeo cache version 3. The
# next eight bytes are the last transaction id.
#
# The file is a contiguous sequence of blocks. All blocks begin with
# a one-byte status indicator:
#
# 'a'
# Allocated. The block holds an object; the next 4 bytes are >I
# format total block size.
#
# 'f'
# Free. The block is free; the next 4 bytes are >I format total
# block size.
#
# '1', '2', '3', '4'
# The block is free, and consists of 1, 2, 3 or 4 bytes total.
#
# 'Z'
# File header. The file starts with a magic number, currently
# 'ZEC3' and an 8-byte transaction id.
#
# "Total" includes the status byte, and size bytes. There are no
# empty (size 0) blocks.
# XXX This needs a lot more hair.
# The structure of an allocated block is more complicated:
#
# 1 byte allocation status ('a').
# 4 bytes block size, >I format.
# 16 bytes oid + tid, string.
# size-OBJECT_HEADER_SIZE bytes, the object pickle.
# The cache's currentofs goes around the file, circularly, forever.
# It's always the starting offset of some block.
#
# When a new object is added to the cache, it's stored beginning at
# currentofs, and currentofs moves just beyond it. As many contiguous
# blocks needed to make enough room for the new object are evicted,
# starting at currentofs. Exception: if currentofs is close enough
# to the end of the file that the new object can't fit in one
# contiguous chunk, currentofs is reset to 0 first.
# Do all possible to ensure that the bytes we wrote are really on
# disk.
class FileCache(object):
def __init__(self, maxsize, fpath, parent, reuse=True):
# Maximum total of object sizes we keep in cache.
self.maxsize = maxsize
# Current total of object sizes in cache.
self.currentsize = 0
self.parent = parent
self.tid = None
# Map offset in file to pair (data record size, Entry).
# Entry is None iff the block starting at offset is free.
# filemap always contains a complete account of what's in the
# file -- study method _verify_filemap for executable checking
# of the relevant invariants. An offset is at the start of a
# block iff it's a key in filemap.
self.filemap = {}
# Map key to Entry. There's one entry for each object in the
# cache file. After
# obj = key2entry[key]
# then
# obj.key == key
# is true.
self.key2entry = {}
# Always the offset into the file of the start of a block.
# New and relocated objects are always written starting at
# currentofs.
self.currentofs = 12
self.fpath = fpath
if not reuse or not fpath or not os.path.exists(fpath):
self.new = True
if fpath:
self.f = file(fpath, 'wb+')
else:
self.f = tempfile.TemporaryFile()
# Make sure the OS really saves enough bytes for the file.
self.f.seek(self.maxsize - 1)
self.f.write('x')
self.f.truncate()
# Start with one magic header block
self.f.seek(0)
self.f.write(magic)
self.f.write(z64)
# and one free block.
self.f.write('f' + struct.pack(">I", self.maxsize - 12))
self.sync()
self.filemap[12] = self.maxsize - 12, None
else:
self.new = False
self.f = None
# Statistics: _n_adds, _n_added_bytes,
# _n_evicts, _n_evicted_bytes
self.clearStats()
# Scan the current contents of the cache file, calling install
# for each object found in the cache. This method should only
# be called once to initialize the cache from disk.
def scan(self, install):
if self.new:
return
fsize = os.path.getsize(self.fpath)
self.f = file(self.fpath, 'rb+')
_magic = self.f.read(4)
if _magic != magic:
raise ValueError("unexpected magic number: %r" % _magic)
self.tid = self.f.read(8)
# Remember the largest free block. That seems a
# decent place to start currentofs.
max_free_size = max_free_offset = 0
ofs = 12
while ofs < fsize:
self.f.seek(ofs)
ent = None
status = self.f.read(1)
if status == 'a':
size, rawkey = struct.unpack(">I16s", self.f.read(20))
key = rawkey[:8], rawkey[8:]
assert key not in self.key2entry
self.key2entry[key] = ent = Entry(key, ofs)
install(self.f, ent)
elif status == 'f':
size, = struct.unpack(">I", self.f.read(4))
elif status in '1234':
size = int(status)
else:
assert 0, hex(ord(status))
self.filemap[ofs] = size, ent
if ent is None and size > max_free_size:
max_free_size, max_free_offset = size, ofs
ofs += size
assert ofs == fsize
if __debug__:
self._verify_filemap()
self.currentofs = max_free_offset
def clearStats(self):
self._n_adds = self._n_added_bytes = 0
self._n_evicts = self._n_evicted_bytes = 0
self._n_removes = self._n_removed_bytes = 0
self._n_accesses = 0
def getStats(self):
return (self._n_adds, self._n_added_bytes,
self._n_evicts, self._n_evicted_bytes,
self._n_removes, self._n_removed_bytes,
self._n_accesses
)
def __len__(self):
return len(self.key2entry)
def __iter__(self):
return self.key2entry.itervalues()
def __contains__(self, key):
return key in self.key2entry
def sync(self):
sync(self.f)
def close(self):
if self.f:
self.sync()
self.f.close()
self.f = None
# Evict objects as necessary to free up at least nbytes bytes,
# starting at currentofs. If currentofs is closer than nbytes to
# the end of the file, currentofs is reset to 0. The number of
# bytes actually freed may be (and probably will be) greater than
# nbytes, and is _makeroom's return value. The file is not
# altered by _makeroom. filemap is updated to reflect the
# evictions, and it's the caller's responsibilty both to fiddle
# the file, and to update filemap, to account for all the space
# freed (starting at currentofs when _makeroom returns, and
# spanning the number of bytes retured by _makeroom).
def _makeroom(self, nbytes):
assert 0 < nbytes <= self.maxsize
if self.currentofs + nbytes > self.maxsize:
self.currentofs = 12
ofs = self.currentofs
while nbytes > 0:
size, e = self.filemap.pop(ofs)
if e is not None:
self._evictobj(e, size)
ofs += size
nbytes -= size
return ofs - self.currentofs
# Write Object obj, with data, to file starting at currentofs.
# nfreebytes are already available for overwriting, and it's
# guranteed that's enough. obj.offset is changed to reflect the
# new data record position, and filemap is updated to match.
def _writeobj(self, obj, nfreebytes):
size = OBJECT_HEADER_SIZE + obj.size
assert size <= nfreebytes
excess = nfreebytes - size
# If there's any excess (which is likely), we need to record a
# free block following the end of the data record. That isn't
# expensive -- it's all a contiguous write.
if excess == 0:
extra = ''
elif excess < 5:
extra = "01234"[excess]
else:
extra = 'f' + struct.pack(">I", excess)
self.f.seek(self.currentofs)
self.f.writelines(('a',
struct.pack(">I8s8s", size,
obj.key[0], obj.key[1])))
obj.serialize(self.f)
self.f.write(extra)
e = Entry(obj.key, self.currentofs)
self.key2entry[obj.key] = e
self.filemap[self.currentofs] = size, e
self.currentofs += size
if excess:
# We need to record the free block in filemap, but there's
# no need to advance currentofs beyond it. Instead it
# gives some breathing room for the next object to get
# written.
self.filemap[self.currentofs] = excess, None
def add(self, object):
size = OBJECT_HEADER_SIZE + object.size
if size > self.maxsize:
return
assert size <= self.maxsize
assert object.key not in self.key2entry
assert len(object.key[0]) == 8
assert len(object.key[1]) == 8
self._n_adds += 1
self._n_added_bytes += size
available = self._makeroom(size)
self._writeobj(object, available)
def _verify_filemap(self, display=False):
a = 12
f = self.f
while a < self.maxsize:
f.seek(a)
status = f.read(1)
if status in 'af':
size, = struct.unpack(">I", f.read(4))
else:
size = int(status)
if display:
if a == self.currentofs:
print '*****',
print "%c%d" % (status, size),
size2, obj = self.filemap[a]
assert size == size2
assert (obj is not None) == (status == 'a')
if obj is not None:
assert obj.offset == a
assert self.key2entry[obj.key] is obj
a += size
if display:
print
assert a == self.maxsize
def _evictobj(self, e, size):
self._n_evicts += 1
self._n_evicted_bytes += size
# Load the object header into memory so we know how to
# update the parent's in-memory data structures.
self.f.seek(e.offset + OBJECT_HEADER_SIZE)
o = Object.fromFile(self.f, e.key, header_only=True)
self.parent._evicted(o)
##
# Return object for key or None if not in cache.
def access(self, key):
self._n_accesses += 1
e = self.key2entry.get(key)
if e is None:
return None
offset = e.offset
size, e2 = self.filemap[offset]
assert e is e2
self.f.seek(offset + OBJECT_HEADER_SIZE)
return Object.fromFile(self.f, key)
##
# Remove object for key from cache, if present.
def remove(self, key):
# If an object is being explicitly removed, we need to load
# its header into memory and write a free block marker to the
# disk where the object was stored. We need to load the
# header to update the in-memory data structures held by
# ClientCache.
# XXX Or we could just keep the header in memory at all times.
e = self.key2entry.get(key)
if e is None:
return
offset = e.offset
size, e2 = self.filemap[offset]
self.f.seek(offset + OBJECT_HEADER_SIZE)
o = Object.fromFile(self.f, key, header_only=True)
self.f.seek(offset + OBJECT_HEADER_SIZE)
self.f.write('f')
self.f.flush()
self.parent._evicted(o)
self.filemap[offset] = size, None
##
# Update on-disk representation of obj.
#
# This method should be called when the object header is modified.
def update(self, obj):
e = self.key2entry[obj.key]
self.f.seek(e.offset + OBJECT_HEADER_SIZE)
obj.serialize_header(self.f)
def settid(self, tid):
if self.tid is not None and tid <= self.tid:
raise ValueError("new last tid (%s) must be greater than "
"previous one (%s)" % (u64(tid),
u64(self.tid)))
self.tid = tid
self.f.seek(4)
self.f.write(tid)
self.f.flush()
lib/python/ZEO/component.xml 0 → 100644
View file @ 96172040
<component>
<sectiontype name="zeo">
<description>
The content of a ZEO section describe operational parameters
of a ZEO server except for the storage(s) to be served.
</description>
<key name="address" datatype="socket-address"
required="yes">
<description>
The address at which the server should listen. This can be in
the form 'host:port' to signify a TCP/IP connection or a
pathname string to signify a Unix domain socket connection (at
least one '/' is required). A hostname may be a DNS name or a
dotted IP address. If the hostname is omitted, the platform's
default behavior is used when binding the listening socket (''
is passed to socket.bind() as the hostname portion of the
address).
</description>
</key>
<key name="read-only" datatype="boolean"
required="no"
default="false">
<description>
Flag indicating whether the server should operate in read-only
mode. Defaults to false. Note that even if the server is
operating in writable mode, individual storages may still be
read-only. But if the server is in read-only mode, no write
operations are allowed, even if the storages are writable. Note
that pack() is considered a read-only operation.
</description>
</key>
<key name="invalidation-queue-size" datatype="integer"
required="no"
default="100">
<description>
The storage server keeps a queue of the objects modified by the
last N transactions, where N == invalidation_queue_size. This
queue is used to speed client cache verification when a client
disconnects for a short period of time.
</description>
</key>
<key name="monitor-address" datatype="socket-address"
required="no">
<description>
The address at which the monitor server should listen. If
specified, a monitor server is started. The monitor server
provides server statistics in a simple text format. This can
be in the form 'host:port' to signify a TCP/IP connection or a
pathname string to signify a Unix domain socket connection (at
least one '/' is required). A hostname may be a DNS name or a
dotted IP address. If the hostname is omitted, the platform's
default behavior is used when binding the listening socket (''
is passed to socket.bind() as the hostname portion of the
address).
</description>
</key>
<key name="transaction-timeout" datatype="integer"
required="no">
<description>
The maximum amount of time to wait for a transaction to commit
after acquiring the storage lock, specified in seconds. If the
transaction takes too long, the client connection will be closed
and the transaction aborted.
</description>
</key>
<key name="authentication-protocol" required="no">
<description>
The name of the protocol used for authentication. The
only protocol provided with ZEO is "digest," but extensions
may provide other protocols.
</description>
</key>
<key name="authentication-database" required="no">
<description>
The path of the database containing authentication credentials.
</description>
</key>
<key name="authentication-realm" required="no">
<description>
The authentication realm of the server. Some authentication
schemes use a realm to identify the logical set of usernames
that are accepted by this server.
</description>
</key>
</sectiontype>
</component>
lib/python/ZEO/mkzeoinst.py 0 → 100755
View file @ 96172040
#!python
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""%(program)s -- create a ZEO instance.
Usage: %(program)s home [port]
Given an "instance home directory" <home> and some configuration
options (all of which have default values), create the following:
<home>/etc/zeo.conf -- ZEO config file
<home>/var/ -- Directory for data files: Data.fs etc.
<home>/log/ -- Directory for log files: zeo.log and zeoctl.log
<home>/bin/runzeo -- the zeo server runner
<home>/bin/zeoctl -- start/stop script (a shim for zeoctl.py)
The script will not overwrite existing files; instead, it will issue a
warning if an existing file is found that differs from the file that
would be written if it didn't exist.
"""
# WARNING! Several templates and functions here are reused by ZRS.
# So be careful with changes.
import os
import sys
import stat
import getopt
zeo_conf_template = """\
# ZEO configuration file
%%define INSTANCE %(instance_home)s
<zeo>
address %(port)d
read-only false
invalidation-queue-size 100
# monitor-address PORT
# transaction-timeout SECONDS
</zeo>
<filestorage 1>
path $INSTANCE/var/Data.fs
</filestorage>
<eventlog>
level info
<logfile>
path $INSTANCE/log/zeo.log
</logfile>
</eventlog>
<runner>
program $INSTANCE/bin/runzeo
socket-name $INSTANCE/etc/%(package)s.zdsock
daemon true
forever false
backoff-limit 10
exit-codes 0, 2
directory $INSTANCE
default-to-interactive true
# user zope
python %(python)s
zdrun %(zodb3_home)s/zdaemon/zdrun.py
# This logfile should match the one in the %(package)s.conf file.
# It is used by zdctl's logtail command, zdrun/zdctl doesn't write it.
logfile $INSTANCE/log/%(package)s.log
</runner>
"""
zeoctl_template = """\
#!/bin/sh
# %(PACKAGE)s instance control script
# The following two lines are for chkconfig. On Red Hat Linux (and
# some other systems), you can copy or symlink this script into
# /etc/rc.d/init.d/ and then use chkconfig(8) to automatically start
# %(PACKAGE)s at boot time.
# chkconfig: 345 90 10
# description: start a %(PACKAGE)s server
PYTHON="%(python)s"
ZODB3_HOME="%(zodb3_home)s"
CONFIG_FILE="%(instance_home)s/etc/%(package)s.conf"
PYTHONPATH="$ZODB3_HOME"
export PYTHONPATH
ZEOCTL="$ZODB3_HOME/ZEO/zeoctl.py"
exec "$PYTHON" "$ZEOCTL" -C "$CONFIG_FILE" ${1+"$@"}
"""
runzeo_template = """\
#!/bin/sh
# %(PACKAGE)s instance start script
PYTHON="%(python)s"
ZODB3_HOME="%(zodb3_home)s"
CONFIG_FILE="%(instance_home)s/etc/%(package)s.conf"
PYTHONPATH="$ZODB3_HOME"
export PYTHONPATH
RUNZEO="$ZODB3_HOME/ZEO/runzeo.py"
exec "$PYTHON" "$RUNZEO" -C "$CONFIG_FILE" ${1+"$@"}
"""
def main():
ZEOInstanceBuilder().run()
print "All done."
class ZEOInstanceBuilder:
def run(self):
try:
opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
except getopt.error, msg:
print msg
sys.exit(2)
program = os.path.basename(sys.argv[0])
if opts:
# There's only the help options, so just dump some help:
msg = __doc__ % {"program": program}
print msg
sys.exit()
if len(args) not in [1, 2]:
print "Usage: %s home [port]" % program
sys.exit(2)
instance_home = args[0]
if not os.path.isabs(instance_home):
instance_home = os.path.abspath(instance_home)
for entry in sys.path:
if os.path.exists(os.path.join(entry, 'ZODB')):
zodb3_home = entry
break
else:
print "Can't find the Zope home (not in sys.path)"
sys.exit(2)
if args[1:]:
port = int(args[1])
else:
port = 9999
params = self.get_params(zodb3_home, instance_home, port)
self.create(instance_home, params)
def get_params(self, zodb3_home, instance_home, port):
return {
"package": "zeo",
"PACKAGE": "ZEO",
"zodb3_home": zodb3_home,
"instance_home": instance_home,
"port": port,
"python": sys.executable,
}
def create(self, home, params):
makedir(home)
makedir(home, "etc")
makedir(home, "var")
makedir(home, "log")
makedir(home, "bin")
makefile(zeo_conf_template, home, "etc", "zeo.conf", **params)
makexfile(zeoctl_template, home, "bin", "zeoctl", **params)
makexfile(runzeo_template, home, "bin", "runzeo", **params)
def which(program):
strpath = os.getenv("PATH")
binpath = strpath.split(os.pathsep)
for dir in binpath:
path = os.path.join(dir, program)
if os.path.isfile(path) and os.access(path, os.X_OK):
if not os.path.isabs(path):
path = os.path.abspath(path)
return path
raise IOError, "can't find %r on path %r" % (program, strpath)
def makedir(*args):
path = ""
for arg in args:
path = os.path.join(path, arg)
mkdirs(path)
return path
def mkdirs(path):
if os.path.isdir(path):
return
head, tail = os.path.split(path)
if head and tail and not os.path.isdir(head):
mkdirs(head)
os.mkdir(path)
print "Created directory", path
def makefile(template, *args, **kwds):
path = makedir(*args[:-1])
path = os.path.join(path, args[-1])
data = template % kwds
if os.path.exists(path):
f = open(path)
olddata = f.read().strip()
f.close()
if olddata:
if olddata != data.strip():
print "Warning: not overwriting existing file %r" % path
return path
f = open(path, "w")
f.write(data)
f.close()
print "Wrote file", path
return path
def makexfile(template, *args, **kwds):
path = makefile(template, *args, **kwds)
umask = os.umask(022)
os.umask(umask)
mode = 0777 & ~umask
if stat.S_IMODE(os.stat(path)[stat.ST_MODE]) != mode:
os.chmod(path, mode)
print "Changed mode for %s to %o" % (path, mode)
return path
if __name__ == "__main__":
main()
lib/python/ZEO/monitor.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Monitor behavior of ZEO server and record statistics.
$Id: monitor.py,v 1.6 2004/04/25 11:34:15 gintautasm Exp $
"""
import asyncore
import socket
import time
import types
import logging
import ZEO
class StorageStats:
"""Per-storage usage statistics."""
def __init__(self):
self.loads = 0
self.stores = 0
self.commits = 0
self.aborts = 0
self.active_txns = 0
self.clients = 0
self.verifying_clients = 0
self.lock_time = None
self.conflicts = 0
self.conflicts_resolved = 0
self.start = time.ctime()
def parse(self, s):
# parse the dump format
lines = s.split("\n")
for line in lines:
field, value = line.split(":", 1)
if field == "Server started":
self.start = value
elif field == "Clients":
self.clients = int(value)
elif field == "Clients verifying":
self.verifying_clients = int(value)
elif field == "Active transactions":
self.active_txns = int(value)
elif field == "Commit lock held for":
# This assumes
self.lock_time = time.time() - int(value)
elif field == "Commits":
self.commits = int(value)
elif field == "Aborts":
self.aborts = int(value)
elif field == "Loads":
self.loads = int(value)
elif field == "Stores":
self.stores = int(value)
elif field == "Conflicts":
self.conflicts = int(value)
elif field == "Conflicts resolved":
self.conflicts_resolved = int(value)
def dump(self, f):
print >> f, "Server started:", self.start
print >> f, "Clients:", self.clients
print >> f, "Clients verifying:", self.verifying_clients
print >> f, "Active transactions:", self.active_txns
if self.lock_time:
howlong = time.time() - self.lock_time
print >> f, "Commit lock held for:", int(howlong)
print >> f, "Commits:", self.commits
print >> f, "Aborts:", self.aborts
print >> f, "Loads:", self.loads
print >> f, "Stores:", self.stores
print >> f, "Conflicts:", self.conflicts
print >> f, "Conflicts resolved:", self.conflicts_resolved
class StatsClient(asyncore.dispatcher):
def __init__(self, sock, addr):
asyncore.dispatcher.__init__(self, sock)
self.buf = []
self.closed = 0
def close(self):
self.closed = 1
# The socket is closed after all the data is written.
# See handle_write().
def write(self, s):
self.buf.append(s)
def writable(self):
return len(self.buf)
def readable(self):
# XXX what goes here?
return 0
def handle_write(self):
s = "".join(self.buf)
self.buf = []
n = self.socket.send(s)
if n < len(s):
self.buf.append(s[:n])
if self.closed and not self.buf:
asyncore.dispatcher.close(self)
class StatsServer(asyncore.dispatcher):
StatsConnectionClass = StatsClient
def __init__(self, addr, stats):
asyncore.dispatcher.__init__(self)
self.addr = addr
self.stats = stats
if type(self.addr) == types.TupleType:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
logger = logging.getLogger('ZEO.monitor')
logger.info("listening on %s", repr(self.addr))
self.bind(self.addr)
self.listen(5)
def writable(self):
return 0
def readable(self):
return 1
def handle_accept(self):
try:
sock, addr = self.accept()
except socket.error:
return
f = self.StatsConnectionClass(sock, addr)
self.dump(f)
f.close()
def dump(self, f):
print >> f, "ZEO monitor server version %s" % ZEO.version
print >> f, time.ctime()
print >> f
L = self.stats.keys()
L.sort()
for k in L:
stats = self.stats[k]
print >> f, "Storage:", k
stats.dump(f)
print >> f
lib/python/ZEO/runzeo.py 0 → 100644
View file @ 96172040
#!python
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Start the ZEO storage server.
Usage: %s [-C URL] [-a ADDRESS] [-f FILENAME] [-h]
Options:
-C/--configuration URL -- configuration file or URL
-a/--address ADDRESS -- server address of the form PORT, HOST:PORT, or PATH
(a PATH must contain at least one "/")
-f/--filename FILENAME -- filename for FileStorage
-t/--timeout TIMEOUT -- transaction timeout in seconds (default no timeout)
-h/--help -- print this usage message and exit
-m/--monitor ADDRESS -- address of monitor server ([HOST:]PORT or PATH)
Unless -C is specified, -a and -f are required.
"""
# The code here is designed to be reused by other, similar servers.
# For the forseeable future, it must work under Python 2.1 as well as
# 2.2 and above.
import os
import sys
import signal
import socket
import logging
import ZConfig, ZConfig.datatypes
import ZEO
from zdaemon.zdoptions import ZDOptions
logger = logging.getLogger('ZEO.runzeo')
_pid = str(os.getpid())
def log(msg, level=logging.INFO, exc_info=False):
"""Internal: generic logging function."""
message = "(%s) %s" % (_pid, msg)
logger.log(level, message, exc_info=exc_info)
def parse_address(arg):
# XXX Not part of the official ZConfig API
obj = ZConfig.datatypes.SocketAddress(arg)
return obj.family, obj.address
class ZEOOptionsMixin:
storages = None
def handle_address(self, arg):
self.family, self.address = parse_address(arg)
def handle_monitor_address(self, arg):
self.monitor_family, self.monitor_address = parse_address(arg)
def handle_filename(self, arg):
from ZODB.config import FileStorage # That's a FileStorage *opener*!
class FSConfig:
def __init__(self, name, path):
self._name = name
self.path = path
self.create = 0
self.read_only = 0
self.stop = None
self.quota = None
def getSectionName(self):
return self._name
if not self.storages:
self.storages = []
name = str(1 + len(self.storages))
conf = FileStorage(FSConfig(name, arg))
self.storages.append(conf)
def add_zeo_options(self):
self.add(None, None, "a:", "address=", self.handle_address)
self.add(None, None, "f:", "filename=", self.handle_filename)
self.add("family", "zeo.address.family")
self.add("address", "zeo.address.address",
required="no server address specified; use -a or -C")
self.add("read_only", "zeo.read_only", default=0)
self.add("invalidation_queue_size", "zeo.invalidation_queue_size",
default=100)
self.add("transaction_timeout", "zeo.transaction_timeout",
"t:", "timeout=", float)
self.add("monitor_address", "zeo.monitor_address.address",
"m:", "monitor=", self.handle_monitor_address)
self.add('auth_protocol', 'zeo.authentication_protocol',
None, 'auth-protocol=', default=None)
self.add('auth_database', 'zeo.authentication_database',
None, 'auth-database=')
self.add('auth_realm', 'zeo.authentication_realm',
None, 'auth-realm=')
class ZEOOptions(ZDOptions, ZEOOptionsMixin):
logsectionname = "eventlog"
schemadir = os.path.dirname(ZEO.__file__)
def __init__(self):
ZDOptions.__init__(self)
self.add_zeo_options()
self.add("storages", "storages",
required="no storages specified; use -f or -C")
class ZEOServer:
def __init__(self, options):
self.options = options
def main(self):
self.setup_default_logging()
self.check_socket()
self.clear_socket()
try:
self.open_storages()
self.setup_signals()
self.create_server()
self.loop_forever()
finally:
self.close_storages()
self.clear_socket()
def setup_default_logging(self):
if self.options.config_logger is not None:
return
# No log file is configured; default to stderr.
logger = logging.getLogger()
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
logger.addHandler(handler)
def check_socket(self):
if self.can_connect(self.options.family, self.options.address):
self.options.usage("address %s already in use" %
repr(self.options.address))
def can_connect(self, family, address):
s = socket.socket(family, socket.SOCK_STREAM)
try:
s.connect(address)
except socket.error:
return 0
else:
s.close()
return 1
def clear_socket(self):
if isinstance(self.options.address, type("")):
try:
os.unlink(self.options.address)
except os.error:
pass
def open_storages(self):
self.storages = {}
for opener in self.options.storages:
log("opening storage %r using %s"
% (opener.name, opener.__class__.__name__))
self.storages[opener.name] = opener.open()
def setup_signals(self):
"""Set up signal handlers.
The signal handler for SIGFOO is a method handle_sigfoo().
If no handler method is defined for a signal, the signal
action is not changed from its initial value. The handler
method is called without additional arguments.
"""
if os.name != "posix":
return
if hasattr(signal, 'SIGXFSZ'):
signal.signal(signal.SIGXFSZ, signal.SIG_IGN) # Special case
init_signames()
for sig, name in signames.items():
method = getattr(self, "handle_" + name.lower(), None)
if method is not None:
def wrapper(sig_dummy, frame_dummy, method=method):
method()
signal.signal(sig, wrapper)
def create_server(self):
from ZEO.StorageServer import StorageServer
self.server = StorageServer(
self.options.address,
self.storages,
read_only=self.options.read_only,
invalidation_queue_size=self.options.invalidation_queue_size,
transaction_timeout=self.options.transaction_timeout,
monitor_address=self.options.monitor_address,
auth_protocol=self.options.auth_protocol,
auth_database=self.options.auth_database, # XXX option spelling
auth_realm=self.options.auth_realm)
def loop_forever(self):
import ThreadedAsync.LoopCallback
ThreadedAsync.LoopCallback.loop()
def handle_sigterm(self):
log("terminated by SIGTERM")
sys.exit(0)
def handle_sigint(self):
log("terminated by SIGINT")
sys.exit(0)
def handle_sighup(self):
log("restarted by SIGHUP")
sys.exit(1)
def handle_sigusr2(self):
# XXX this used to reinitialize zLOG. How do I achieve
# the same effect with Python's logging package?
# Should we restart as with SIGHUP?
log("received SIGUSR2, but it was not handled!", level=logging.WARNING)
def close_storages(self):
for name, storage in self.storages.items():
log("closing storage %r" % name)
try:
storage.close()
except: # Keep going
log("failed to close storage %r" % name,
level=logging.EXCEPTION, exc_info=True)
# Signal names
signames = None
def signame(sig):
"""Return a symbolic name for a signal.
Return "signal NNN" if there is no corresponding SIG name in the
signal module.
"""
if signames is None:
init_signames()
return signames.get(sig) or "signal %d" % sig
def init_signames():
global signames
signames = {}
for name, sig in signal.__dict__.items():
k_startswith = getattr(name, "startswith", None)
if k_startswith is None:
continue
if k_startswith("SIG") and not k_startswith("SIG_"):
signames[sig] = name
# Main program
def main(args=None):
options = ZEOOptions()
options.realize(args)
s = ZEOServer(options)
s.main()
if __name__ == "__main__":
main()
lib/python/ZEO/schema.xml 0 → 100644
View file @ 96172040
<schema>
<!-- note that zeoctl.xml is a closely related schema which should
match this schema, but should require the "runner" section -->
<description>
This schema describes the configuration of the ZEO storage server
process.
</description>
<!-- Use the storage types defined by ZODB. -->
<import package="ZODB"/>
<!-- Use the ZEO server information structure. -->
<import package="ZEO"/>
<import package="ZConfig.components.logger"/>
<!-- runner control -->
<import package="zdaemon"/>
<section type="zeo" name="*" required="yes" attribute="zeo" />
<section type="runner" name="*" required="no" attribute="runner" />
<multisection name="+" type="ZODB.storage"
attribute="storages"
required="yes">
<description>
One or more storages that are provided by the ZEO server. The
section names are used as the storage names, and must be unique
within each ZEO storage server. Traditionally, these names
represent small integers starting at '1'.
</description>
</multisection>
<section name="*" type="eventlog" attribute="eventlog" required="no" />
</schema>
lib/python/ZEO/simul.py 0 → 100644
View file @ 96172040
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Cache simulation.
Usage: simul.py [-bflyz] [-X] [-s size] tracefile
Use one of -b, -f, -l, -y or -z select the cache simulator:
-b: buddy system allocator
-f: simple free list allocator
-l: idealized LRU (no allocator)
-y: variation on the existing ZEO cache that copies to current file
-z: existing ZEO cache (default)
Options:
-s size: cache size in MB (default 20 MB)
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
Note: the buddy system allocator rounds the cache size up to a power of 2
"""
import sys
import time
import getopt
import struct
def usage(msg):
print >>sys.stderr, msg
print >>sys.stderr, __doc__
def main():
# Parse options
MB = 1000*1000
cachelimit = 20*MB
simclass = ZEOCacheSimulation
heuristic = 0
try:
opts, args = getopt.getopt(sys.argv[1:], "bflyzs:X")
except getopt.error, msg:
usage(msg)
return 2
for o, a in opts:
if o == '-b':
simclass = BuddyCacheSimulation
if o == '-f':
simclass = SimpleCacheSimulation
if o == '-l':
simclass = LRUCacheSimulation
if o == '-y':
simclass = AltZEOCacheSimulation
if o == '-z':
simclass = ZEOCacheSimulation
if o == '-s':
cachelimit = int(float(a)*MB)
if o == '-X':
heuristic = 1
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file
if filename.endswith(".gz"):
# Open gzipped file
try:
import gzip
except ImportError:
print >>sys.stderr, "can't read gzipped files (no module gzip)"
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >>sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
elif filename == "-":
# Read from stdin
f = sys.stdin
else:
# Open regular file
try:
f = open(filename, "rb")
except IOError, msg:
print >>sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
# Create simulation object
sim = simclass(cachelimit)
# Print output header
sim.printheader()
# Read trace file, simulating cache behavior
offset = 0
records = 0
f_read = f.read
struct_unpack = struct.unpack
while 1:
# Read a record and decode it
r = f_read(10)
if len(r) < 10:
break
offset += 10
ts, code, lenoid = struct_unpack(">iiH", r)
if ts == 0:
# Must be a misaligned record caused by a crash
##print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(8 + lenoid)
if len(r) < 8 + lenoid:
break
offset += 8 + lenoid
records += 1
serial, oid = struct_unpack(">8s%ds" % lenoid, r)
# Decode the code
dlen, version, code, current = (code & 0x7fffff00,
code & 0x80,
code & 0x7e,
code & 0x01)
# And pass it to the simulation
sim.event(ts, dlen, version, code, current, oid, serial)
# Finish simulation
sim.finish()
# Exit code from main()
return 0
class Simulation:
"""Base class for simulations.
The driver program calls: event(), printheader(), finish().
The standard event() method calls these additional methods:
write(), load(), inval(), report(), restart(); the standard
finish() method also calls report().
"""
def __init__(self, cachelimit):
self.cachelimit = cachelimit
# Initialize global statistics
self.epoch = None
self.total_loads = 0
self.total_hits = 0 # Subclass must increment
self.total_invals = 0
self.total_writes = 0
# Reset per-run statistics and set up simulation data
self.restart()
def restart(self):
# Reset per-run statistics
self.loads = 0
self.hits = 0 # Subclass must increment
self.invals = 0
self.writes = 0
self.ts0 = None
def event(self, ts, dlen, _version, code, _current, oid, _serial):
# Record first and last timestamp seen
if self.ts0 is None:
self.ts0 = ts
if self.epoch is None:
self.epoch = ts
self.ts1 = ts
# Simulate cache behavior. Use load hits, updates and stores
# only (each load miss is followed immediately by a store
# unless the object in fact did not exist). Updates always write.
if dlen and code & 0x70 in (0x20, 0x30, 0x50):
if code == 0x3A:
# Update
self.writes += 1
self.total_writes += 1
self.write(oid, dlen)
else:
# Load hit or store -- these are really the load requests
self.loads += 1
self.total_loads += 1
self.load(oid, dlen)
elif code & 0x70 == 0x10:
# Invalidate
self.inval(oid)
elif code == 0x00:
# Restart
self.report()
self.restart()
def write(self, oid, size):
pass
def load(self, oid, size):
pass
def inval(self, oid):
pass
format = "%12s %9s %8s %8s %6s %6s %6s %6s"
# Subclass should override extraname to name known instance variables;
# if extraname is 'foo', both self.foo and self.total_foo must exist:
extraname = "*** please override ***"
def printheader(self):
print "%s, cache size %s bytes" % (self.__class__.__name__,
addcommas(self.cachelimit))
print self.format % (
"START TIME", "DURATION", "LOADS", "HITS",
"INVALS", "WRITES", self.extraname.upper(), "HITRATE")
nreports = 0
def report(self):
if self.loads:
self.nreports += 1
print self.format % (
time.ctime(self.ts0)[4:-8],
duration(self.ts1 - self.ts0),
self.loads, self.hits, self.invals, self.writes,
getattr(self, self.extraname),
hitrate(self.loads, self.hits))
def finish(self):
self.report()
if self.nreports > 1:
print (self.format + " OVERALL") % (
time.ctime(self.epoch)[4:-8],
duration(self.ts1 - self.epoch),
self.total_loads,
self.total_hits,
self.total_invals,
self.total_writes,
getattr(self, "total_" + self.extraname),
hitrate(self.total_loads, self.total_hits))
class ZEOCacheSimulation(Simulation):
"""Simulate the current (ZEO 1.0 and 2.0) ZEO cache behavior.
This assumes the cache is not persistent (we don't know how to
simulate cache validation.)
"""
extraname = "flips"
def __init__(self, cachelimit):
# Initialize base class
Simulation.__init__(self, cachelimit)
# Initialize additional global statistics
self.total_flips = 0
def restart(self):
# Reset base class
Simulation.restart(self)
# Reset additional per-run statistics
self.flips = 0
# Set up simulation
self.filesize = [4, 4] # account for magic number
self.fileoids = [{}, {}]
self.current = 0 # index into filesize, fileoids
def load(self, oid, size):
if (self.fileoids[self.current].get(oid) or
self.fileoids[1 - self.current].get(oid)):
self.hits += 1
self.total_hits += 1
else:
self.write(oid, size)
def write(self, oid, size):
# Fudge because size is rounded up to multiples of 256. (31
# is header overhead per cache record; 127 is to compensate
# for rounding up to multiples of 256.)
size = size + 31 - 127
if self.filesize[self.current] + size > self.cachelimit / 2:
# Cache flip
self.flips += 1
self.total_flips += 1
self.current = 1 - self.current
self.filesize[self.current] = 4
self.fileoids[self.current] = {}
self.filesize[self.current] += size
self.fileoids[self.current][oid] = 1
def inval(self, oid):
if self.fileoids[self.current].get(oid):
self.invals += 1
self.total_invals += 1
del self.fileoids[self.current][oid]
elif self.fileoids[1 - self.current].get(oid):
self.invals += 1
self.total_invals += 1
del self.fileoids[1 - self.current][oid]
class AltZEOCacheSimulation(ZEOCacheSimulation):
"""A variation of the ZEO cache that copies to the current file.
When a hit is found in the non-current cache file, it is copied to
the current cache file. Exception: when the copy would cause a
cache flip, we don't copy (this is part laziness, part concern
over causing extraneous flips).
"""
def load(self, oid, size):
if self.fileoids[self.current].get(oid):
self.hits += 1
self.total_hits += 1
elif self.fileoids[1 - self.current].get(oid):
self.hits += 1
self.total_hits += 1
# Simulate a write, unless it would cause a flip
size = size + 31 - 127
if self.filesize[self.current] + size <= self.cachelimit / 2:
self.filesize[self.current] += size
self.fileoids[self.current][oid] = 1
del self.fileoids[1 - self.current][oid]
else:
self.write(oid, size)
class LRUCacheSimulation(Simulation):
extraname = "evicts"
def __init__(self, cachelimit):
# Initialize base class
Simulation.__init__(self, cachelimit)
# Initialize additional global statistics
self.total_evicts = 0
def restart(self):
# Reset base class
Simulation.restart(self)
# Reset additional per-run statistics
self.evicts = 0
# Set up simulation
self.cache = {}
self.size = 0
self.head = Node(None, None)
self.head.linkbefore(self.head)
def load(self, oid, size):
node = self.cache.get(oid)
if node is not None:
self.hits += 1
self.total_hits += 1
node.linkbefore(self.head)
else:
self.write(oid, size)
def write(self, oid, size):
node = self.cache.get(oid)
if node is not None:
node.unlink()
assert self.head.next is not None
self.size -= node.size
node = Node(oid, size)
self.cache[oid] = node
node.linkbefore(self.head)
self.size += size
# Evict LRU nodes
while self.size > self.cachelimit:
self.evicts += 1
self.total_evicts += 1
node = self.head.next
assert node is not self.head
node.unlink()
assert self.head.next is not None
del self.cache[node.oid]
self.size -= node.size
def inval(self, oid):
node = self.cache.get(oid)
if node is not None:
assert node.oid == oid
self.invals += 1
self.total_invals += 1
node.unlink()
assert self.head.next is not None
del self.cache[oid]
self.size -= node.size
assert self.size >= 0
class Node:
"""Node in a doubly-linked list, storing oid and size as payload.
A node can be linked or unlinked; in the latter case, next and
prev are None. Initially a node is unlinked.
"""
# Make it a new-style class in Python 2.2 and up; no effect in 2.1
__metaclass__ = type
__slots__ = ['prev', 'next', 'oid', 'size']
def __init__(self, oid, size):
self.oid = oid
self.size = size
self.prev = self.next = None
def unlink(self):
prev = self.prev
next = self.next
if prev is not None:
assert next is not None
assert prev.next is self
assert next.prev is self
prev.next = next
next.prev = prev
self.prev = self.next = None
else:
assert next is None
def linkbefore(self, next):
self.unlink()
prev = next.prev
if prev is None:
assert next.next is None
prev = next
self.prev = prev
self.next = next
prev.next = next.prev = self
class BuddyCacheSimulation(LRUCacheSimulation):
def __init__(self, cachelimit):
LRUCacheSimulation.__init__(self, roundup(cachelimit))
def restart(self):
LRUCacheSimulation.restart(self)
self.allocator = self.allocatorFactory(self.cachelimit)
def allocatorFactory(self, size):
return BuddyAllocator(size)
# LRUCacheSimulation.load() is just fine
def write(self, oid, size):
node = self.cache.get(oid)
if node is not None:
node.unlink()
assert self.head.next is not None
self.size -= node.size
self.allocator.free(node)
while 1:
node = self.allocator.alloc(size)
if node is not None:
break
# Failure to allocate. Evict something and try again.
node = self.head.next
assert node is not self.head
self.evicts += 1
self.total_evicts += 1
node.unlink()
assert self.head.next is not None
del self.cache[node.oid]
self.size -= node.size
self.allocator.free(node)
node.oid = oid
self.cache[oid] = node
node.linkbefore(self.head)
self.size += node.size
def inval(self, oid):
node = self.cache.get(oid)
if node is not None:
assert node.oid == oid
self.invals += 1
self.total_invals += 1
node.unlink()
assert self.head.next is not None
del self.cache[oid]
self.size -= node.size
assert self.size >= 0
self.allocator.free(node)
class SimpleCacheSimulation(BuddyCacheSimulation):
def allocatorFactory(self, size):
return SimpleAllocator(size)
def finish(self):
BuddyCacheSimulation.finish(self)
self.allocator.report()
MINSIZE = 256
class BuddyAllocator:
def __init__(self, cachelimit):
cachelimit = roundup(cachelimit)
self.cachelimit = cachelimit
self.avail = {} # Map rounded-up sizes to free list node heads
self.nodes = {} # Map address to node
k = MINSIZE
while k <= cachelimit:
self.avail[k] = n = Node(None, None) # Not BlockNode; has no addr
n.linkbefore(n)
k += k
node = BlockNode(None, cachelimit, 0)
self.nodes[0] = node
node.linkbefore(self.avail[cachelimit])
def alloc(self, size):
size = roundup(size)
k = size
while k <= self.cachelimit:
head = self.avail[k]
node = head.next
if node is not head:
break
k += k
else:
return None # Store is full, or block is too large
node.unlink()
size2 = node.size
while size2 > size:
size2 = size2 / 2
assert size2 >= size
node.size = size2
buddy = BlockNode(None, size2, node.addr + size2)
self.nodes[buddy.addr] = buddy
buddy.linkbefore(self.avail[size2])
node.oid = 1 # Flag as in-use
return node
def free(self, node):
assert node is self.nodes[node.addr]
assert node.prev is node.next is None
node.oid = None # Flag as free
while node.size < self.cachelimit:
buddy_addr = node.addr ^ node.size
buddy = self.nodes[buddy_addr]
assert buddy.addr == buddy_addr
if buddy.oid is not None or buddy.size != node.size:
break
# Merge node with buddy
buddy.unlink()
if buddy.addr < node.addr: # buddy prevails
del self.nodes[node.addr]
node = buddy
else: # node prevails
del self.nodes[buddy.addr]
node.size *= 2
assert node is self.nodes[node.addr]
node.linkbefore(self.avail[node.size])
def dump(self, msg=""):
if msg:
print msg,
size = MINSIZE
blocks = bytes = 0
while size <= self.cachelimit:
head = self.avail[size]
node = head.next
count = 0
while node is not head:
count += 1
node = node.next
if count:
print "%d:%d" % (size, count),
blocks += count
bytes += count*size
size += size
print "-- %d, %d" % (bytes, blocks)
def roundup(size):
k = MINSIZE
while k < size:
k += k
return k
class SimpleAllocator:
def __init__(self, arenasize):
self.arenasize = arenasize
self.avail = BlockNode(None, 0, 0) # Weird: empty block as list head
self.rover = self.avail
node = BlockNode(None, arenasize, 0)
node.linkbefore(self.avail)
self.taglo = {0: node}
self.taghi = {arenasize: node}
# Allocator statistics
self.nallocs = 0
self.nfrees = 0
self.allocloops = 0
self.freebytes = arenasize
self.freeblocks = 1
self.allocbytes = 0
self.allocblocks = 0
def report(self):
print ("NA=%d AL=%d NF=%d ABy=%d ABl=%d FBy=%d FBl=%d" %
(self.nallocs, self.allocloops,
self.nfrees,
self.allocbytes, self.allocblocks,
self.freebytes, self.freeblocks))
def alloc(self, size):
self.nallocs += 1
# First fit algorithm
rover = stop = self.rover
while 1:
self.allocloops += 1
if rover.size >= size:
break
rover = rover.next
if rover is stop:
return None # We went round the list without finding space
if rover.size == size:
self.rover = rover.next
rover.unlink()
del self.taglo[rover.addr]
del self.taghi[rover.addr + size]
self.freeblocks -= 1
self.allocblocks += 1
self.freebytes -= size
self.allocbytes += size
return rover
# Take space from the beginning of the roving pointer
assert rover.size > size
node = BlockNode(None, size, rover.addr)
del self.taglo[rover.addr]
rover.size -= size
rover.addr += size
self.taglo[rover.addr] = rover
#self.freeblocks += 0 # No change here
self.allocblocks += 1
self.freebytes -= size
self.allocbytes += size
return node
def free(self, node):
self.nfrees += 1
self.freeblocks += 1
self.allocblocks -= 1
self.freebytes += node.size
self.allocbytes -= node.size
node.linkbefore(self.avail)
self.taglo[node.addr] = node
self.taghi[node.addr + node.size] = node
x = self.taghi.get(node.addr)
if x is not None:
# Merge x into node
x.unlink()
self.freeblocks -= 1
del self.taglo[x.addr]
del self.taghi[x.addr + x.size]
del self.taglo[node.addr]
node.addr = x.addr
node.size += x.size
self.taglo[node.addr] = node
x = self.taglo.get(node.addr + node.size)
if x is not None:
# Merge x into node
x.unlink()
self.freeblocks -= 1
del self.taglo[x.addr]
del self.taghi[x.addr + x.size]
del self.taghi[node.addr + node.size]
node.size += x.size
self.taghi[node.addr + node.size] = node
# It's possible that either one of the merges above invalidated
# the rover.
# It's simplest to simply reset the rover to the newly freed block.
self.rover = node
def dump(self, msg=""):
if msg:
print msg,
count = 0
bytes = 0
node = self.avail.next
while node is not self.avail:
bytes += node.size
count += 1
node = node.next
print count, "free blocks,", bytes, "free bytes"
self.report()
class BlockNode(Node):
__slots__ = ['addr']
def __init__(self, oid, size, addr):
Node.__init__(self, oid, size)
self.addr = addr
def testallocator(factory=BuddyAllocator):
# Run one of Knuth's experiments as a test
import random
import heapq # This only runs with Python 2.3, folks :-)
reportfreq = 100
cachelimit = 2**17
cache = factory(cachelimit)
queue = []
T = 0
blocks = 0
while T < 5000:
while queue and queue[0][0] <= T:
time, node = heapq.heappop(queue)
assert time == T
##print "free addr=%d, size=%d" % (node.addr, node.size)
cache.free(node)
blocks -= 1
size = random.randint(100, 2000)
lifetime = random.randint(1, 100)
node = cache.alloc(size)
if node is None:
print "out of mem"
cache.dump("T=%4d: %d blocks;" % (T, blocks))
break
else:
##print "alloc addr=%d, size=%d" % (node.addr, node.size)
blocks += 1
heapq.heappush(queue, (T + lifetime, node))
T = T+1
if T % reportfreq == 0:
cache.dump("T=%4d: %d blocks;" % (T, blocks))
def hitrate(loads, hits):
return "%5.1f%%" % (100.0 * hits / max(1, loads))
def duration(secs):
mm, ss = divmod(secs, 60)
hh, mm = divmod(mm, 60)
if hh:
return "%d:%02d:%02d" % (hh, mm, ss)
if mm:
return "%d:%02d" % (mm, ss)
return "%d" % ss
def addcommas(n):
sign, s = '', str(n)
if s[0] == '-':
sign, s = '-', s[1:]
i = len(s) - 3
while i > 0:
s = s[:i] + ',' + s[i:]
i -= 3
return sign + s
if __name__ == "__main__":
sys.exit(main())
lib/python/ZEO/stats.py 0 → 100755
View file @ 96172040
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Trace file statistics analyzer.
Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
"""
"""File format:
Each record is 24 bytes, with the following layout. Numbers are
big-endian integers.
Offset Size Contents
0 4 timestamp (seconds since 1/1/1970)
4 3 data size, in 256-byte increments, rounded up
7 1 code (see below)
8 2 object id length
10 8 serial number
18 variable object id
The code at offset 7 packs three fields:
Mask bits Contents
0x80 1 set if there was a non-empty version string
0x7e 6 function and outcome code
0x01 1 current cache file (0 or 1)
The function and outcome codes are documented in detail at the end of
this file in the 'explain' dictionary. Note that the keys there (and
also the arguments to _trace() in ClientStorage.py) are 'code & 0x7e',
i.e. the low bit is always zero.
"""
import sys
import time
import getopt
import struct
from types import StringType
def usage(msg):
print >>sys.stderr, msg
print >>sys.stderr, __doc__
def main():
# Parse options
verbose = 0
quiet = 0
dostats = 1
print_size_histogram = 0
print_histogram = 0
interval = 900 # Every 15 minutes
heuristic = 0
try:
opts, args = getopt.getopt(sys.argv[1:], "hi:qsSvX")
except getopt.error, msg:
usage(msg)
return 2
for o, a in opts:
if o == '-h':
print_histogram = 1
if o == "-i":
interval = int(60 * float(a))
if interval <= 0:
interval = 60
elif interval > 3600:
interval = 3600
if o == "-q":
quiet = 1
verbose = 0
if o == "-s":
print_size_histogram = 1
if o == "-S":
dostats = 0
if o == "-v":
verbose = 1
if o == '-X':
heuristic = 1
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file
if filename.endswith(".gz"):
# Open gzipped file
try:
import gzip
except ImportError:
print >>sys.stderr, "can't read gzipped files (no module gzip)"
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >>sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
elif filename == '-':
# Read from stdin
f = sys.stdin
else:
# Open regular file
try:
f = open(filename, "rb")
except IOError, msg:
print >>sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
# Read file, gathering statistics, and printing each record if verbose
rt0 = time.time()
# bycode -- map code to count of occurrences
bycode = {}
# records -- number of records
records = 0
# version -- number of records with versions
versions = 0
t0 = te = None
# datarecords -- number of records with dlen set
datarecords = 0
datasize = 0L
# oids -- maps oid to number of times it was loaded
oids = {}
# bysize -- maps data size to number of loads
bysize = {}
# bysize -- maps data size to number of writes
bysizew = {}
total_loads = 0
byinterval = {}
thisinterval = None
h0 = he = None
offset = 0
f_read = f.read
struct_unpack = struct.unpack
try:
while 1:
r = f_read(8)
if len(r) < 8:
break
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
if not quiet:
print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(18)
if len(r) < 10:
break
offset += 10
records += 1
oidlen, start_tid, end_tid = struct_unpack(">H8s8s", r)
oid = f_read(oidlen)
if len(oid) != oidlen:
break
offset += oidlen
if t0 is None:
t0 = ts
thisinterval = t0 / interval
h0 = he = ts
te = ts
if ts / interval != thisinterval:
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts / interval
h0 = ts
he = ts
dlen, code = code & 0x7fffff00, code & 0xff
if dlen:
datarecords += 1
datasize += dlen
version = '-'
if code & 0x80:
version = 'V'
versions += 1
code = code & 0x7e
bycode[code] = bycode.get(code, 0) + 1
byinterval[code] = byinterval.get(code, 0) + 1
if dlen:
if code & 0x70 == 0x20: # All loads
bysize[dlen] = d = bysize.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
elif code & 0x70 == 0x50: # All stores
bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
if verbose:
print "%s %d %02x %s %016x %016x %1s %s" % (
time.ctime(ts)[4:-5],
current,
code,
oid_repr(oid),
U64(start_tid),
U64(end_tid),
version,
dlen and str(dlen) or "")
if code & 0x70 == 0x20:
oids[oid] = oids.get(oid, 0) + 1
total_loads += 1
if code == 0x00:
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts / interval
h0 = he = ts
if not quiet:
print time.ctime(ts)[4:-5],
print '='*20, "Restart", '='*20
except KeyboardInterrupt:
print "\nInterrupted. Stats so far:\n"
f.close()
rte = time.time()
if not quiet:
dumpbyinterval(byinterval, h0, he)
# Error if nothing was read
if not records:
print >>sys.stderr, "No records processed"
return 1
# Print statistics
if dostats:
print
print "Read %s records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(records*24), rte-rt0)
print "Versions: %s records used a version" % addcommas(versions)
print "First time: %s" % time.ctime(t0)
print "Last time: %s" % time.ctime(te)
print "Duration: %s seconds" % addcommas(te-t0)
print "Data recs: %s (%.1f%%), average size %.1f KB" % (
addcommas(datarecords),
100.0 * datarecords / records,
datasize / 1024.0 / datarecords)
print "Hit rate: %.1f%% (load hits / loads)" % hitrate(bycode)
print
codes = bycode.keys()
codes.sort()
print "%13s %4s %s" % ("Count", "Code", "Function (action)")
for code in codes:
print "%13s %02x %s" % (
addcommas(bycode.get(code, 0)),
code,
explain.get(code) or "*** unknown code ***")
# Print histogram
if print_histogram:
print
print "Histogram of object load frequency"
total = len(oids)
print "Unique oids: %s" % addcommas(total)
print "Total loads: %s" % addcommas(total_loads)
s = addcommas(total)
width = max(len(s), len("objects"))
fmt = "%5d %" + str(width) + "s %5.1f%% %5.1f%% %5.1f%%"
hdr = "%5s %" + str(width) + "s %6s %6s %6s"
print hdr % ("loads", "objects", "%obj", "%load", "%cum")
cum = 0.0
for binsize, count in histogram(oids):
obj_percent = 100.0 * count / total
load_percent = 100.0 * count * binsize / total_loads
cum += load_percent
print fmt % (binsize, addcommas(count),
obj_percent, load_percent, cum)
# Print size histogram
if print_size_histogram:
print
print "Histograms of object sizes"
print
dumpbysize(bysizew, "written", "writes")
dumpbysize(bysize, "loaded", "loads")
def dumpbysize(bysize, how, how2):
print
print "Unique sizes %s: %s" % (how, addcommas(len(bysize)))
print "%10s %6s %6s" % ("size", "objs", how2)
sizes = bysize.keys()
sizes.sort()
for size in sizes:
loads = 0
for n in bysize[size].itervalues():
loads += n
print "%10s %6d %6d" % (addcommas(size),
len(bysize.get(size, "")),
loads)
def dumpbyinterval(byinterval, h0, he):
loads = 0
hits = 0
for code in byinterval.keys():
if code & 0x70 == 0x20:
n = byinterval[code]
loads += n
if code in (0x22, 0x26):
hits += n
if not loads:
return
if loads:
hr = 100.0 * hits / loads
else:
hr = 0.0
print "%s-%s %10s loads, %10s hits,%5.1f%% hit rate" % (
time.ctime(h0)[4:-8], time.ctime(he)[14:-8],
addcommas(loads), addcommas(hits), hr)
def hitrate(bycode):
loads = 0
hits = 0
for code in bycode.keys():
if code & 0x70 == 0x20:
n = bycode[code]
loads += n
if code in (0x22, 0x26):
hits += n
if loads:
return 100.0 * hits / loads
else:
return 0.0
def histogram(d):
bins = {}
for v in d.itervalues():
bins[v] = bins.get(v, 0) + 1
L = bins.items()
L.sort()
return L
def U64(s):
h, v = struct.unpack(">II", s)
return (long(h) << 32) + v
def oid_repr(oid):
if isinstance(oid, StringType) and len(oid) == 8:
return '%16x' % U64(oid)
else:
return repr(oid)
def addcommas(n):
sign, s = '', str(n)
if s[0] == '-':
sign, s = '-', s[1:]
i = len(s) - 3
while i > 0:
s = s[:i] + ',' + s[i:]
i -= 3
return sign + s
explain = {
# The first hex digit shows the operation, the second the outcome.
# If the second digit is in "02468" then it is a 'miss'.
# If it is in "ACE" then it is a 'hit'.
0x00: "_setup_trace (initialization)",
0x10: "invalidate (miss)",
0x1A: "invalidate (hit, version)",
0x1C: "invalidate (hit, saving non-current)",
0x20: "load (miss)",
0x22: "load (hit)",
0x24: "load (non-current, miss)",
0x26: "load (non-current, hit)",
0x50: "store (version)",
0x52: "store (current, non-version)",
0x54: "store (non-current)",
}
if __name__ == "__main__":
sys.exit(main())
lib/python/ZEO/tests/Cache.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Tests of the ZEO cache"""
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_unpickle
from transaction import Transaction
class TransUndoStorageWithCache:
def checkUndoInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(23))
revid = self._dostore(oid, revid=revid, data=MinPO(24))
revid = self._dostore(oid, revid=revid, data=MinPO(25))
info = self._storage.undoInfo()
if not info:
# XXX perhaps we have an old storage implementation that
# does do the negative nonsense
info = self._storage.undoInfo(0, 20)
tid = info[0]['id']
# We may need to bail at this point if the storage doesn't
# support transactional undo
if not self._storage.supportsTransactionalUndo():
return
# Now start an undo transaction
t = Transaction()
t.note('undo1')
self._storage.tpc_begin(t)
tid, oids = self._storage.undo(tid, t)
# Make sure this doesn't load invalid data into the cache
self._storage.load(oid, '')
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
assert len(oids) == 1
assert oids[0] == oid
data, revid = self._storage.load(oid, '')
obj = zodb_unpickle(data)
assert obj == MinPO(24)
class StorageWithCache:
def checkAbortVersionInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(1))
revid = self._dostore(oid, revid=revid, data=MinPO(2))
revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
revid = self._dostore(oid, revid=revid, data=MinPO(4), version="foo")
t = Transaction()
self._storage.tpc_begin(t)
self._storage.abortVersion("foo", t)
self._storage.load(oid, "foo")
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
data, revid = self._storage.load(oid, "foo")
obj = zodb_unpickle(data)
assert obj == MinPO(2), obj
def checkCommitEmptyVersionInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(1))
revid = self._dostore(oid, revid=revid, data=MinPO(2))
revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
t = Transaction()
self._storage.tpc_begin(t)
self._storage.commitVersion("foo", "", t)
self._storage.load(oid, "")
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
data, revid = self._storage.load(oid, "")
obj = zodb_unpickle(data)
assert obj == MinPO(3), obj
def checkCommitVersionInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(1))
revid = self._dostore(oid, revid=revid, data=MinPO(2))
revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
t = Transaction()
self._storage.tpc_begin(t)
self._storage.commitVersion("foo", "bar", t)
self._storage.load(oid, "")
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
data, revid = self._storage.load(oid, "bar")
obj = zodb_unpickle(data)
assert obj == MinPO(3), obj
lib/python/ZEO/tests/CommitLockTests.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Tests of the distributed commit lock."""
import threading
import time
from persistent.TimeStamp import TimeStamp
import transaction
from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
import ZEO.ClientStorage
from ZEO.Exceptions import ClientDisconnected
from ZEO.tests.TestThread import TestThread
ZERO = '\0'*8
class DummyDB:
def invalidate(self, *args, **kwargs):
pass
class WorkerThread(TestThread):
# run the entire test in a thread so that the blocking call for
# tpc_vote() doesn't hang the test suite.
def __init__(self, testcase, storage, trans, method="tpc_finish"):
self.storage = storage
self.trans = trans
self.method = method
self.ready = threading.Event()
TestThread.__init__(self, testcase)
def testrun(self):
try:
self.storage.tpc_begin(self.trans)
oid = self.storage.new_oid()
p = zodb_pickle(MinPO("c"))
self.storage.store(oid, ZERO, p, '', self.trans)
oid = self.storage.new_oid()
p = zodb_pickle(MinPO("c"))
self.storage.store(oid, ZERO, p, '', self.trans)
self.myvote()
if self.method == "tpc_finish":
self.storage.tpc_finish(self.trans)
else:
self.storage.tpc_abort(self.trans)
except ClientDisconnected:
pass
def myvote(self):
# The vote() call is synchronous, which makes it difficult to
# coordinate the action of multiple threads that all call
# vote(). This method sends the vote call, then sets the
# event saying vote was called, then waits for the vote
# response. It digs deep into the implementation of the client.
# This method is a replacement for:
# self.ready.set()
# self.storage.tpc_vote(self.trans)
rpc = self.storage._server.rpc
msgid = rpc._deferred_call('vote', id(self.trans))
self.ready.set()
rpc._deferred_wait(msgid)
self.storage._check_serials()
class CommitLockTests:
NUM_CLIENTS = 5
# The commit lock tests verify that the storage successfully
# blocks and restarts transactions when there is contention for a
# single storage. There are a lot of cases to cover.
# The general flow of these tests is to start a transaction by
# getting far enough into 2PC to acquire the commit lock. Then
# begin one or more other connections that also want to commit.
# This causes the commit lock code to be exercised. Once the
# other connections are started, the first transaction completes.
def _cleanup(self):
for store, trans in self._storages:
store.tpc_abort(trans)
store.close()
self._storages = []
def _start_txn(self):
txn = transaction.Transaction()
self._storage.tpc_begin(txn)
oid = self._storage.new_oid()
self._storage.store(oid, ZERO, zodb_pickle(MinPO(1)), '', txn)
return oid, txn
def _begin_threads(self):
# Start a second transaction on a different connection without
# blocking the test thread. Returns only after each thread has
# set it's ready event.
self._storages = []
self._threads = []
for i in range(self.NUM_CLIENTS):
storage = self._duplicate_client()
txn = transaction.Transaction()
tid = self._get_timestamp()
t = WorkerThread(self, storage, txn)
self._threads.append(t)
t.start()
t.ready.wait()
# Close on the connections abnormally to test server response
if i == 0:
storage.close()
else:
self._storages.append((storage, txn))
def _finish_threads(self):
for t in self._threads:
t.cleanup()
def _duplicate_client(self):
"Open another ClientStorage to the same server."
# XXX argh it's hard to find the actual address
# The rpc mgr addr attribute is a list. Each element in the
# list is a socket domain (AF_INET, AF_UNIX, etc.) and an
# address.
addr = self._storage._addr
new = ZEO.ClientStorage.ClientStorage(addr, wait=1)
new.registerDB(DummyDB(), None)
return new
def _get_timestamp(self):
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
return `t`
class CommitLockVoteTests(CommitLockTests):
def checkCommitLockVoteFinish(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.tpc_finish(txn)
self._storage.load(oid, '')
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockVoteAbort(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.tpc_abort(txn)
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockVoteClose(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.close()
self._finish_threads()
self._cleanup()
class CommitLockUndoTests(CommitLockTests):
def _get_trans_id(self):
self._dostore()
L = self._storage.undoInfo()
return L[0]['id']
def _begin_undo(self, trans_id, txn):
rpc = self._storage._server.rpc
return rpc._deferred_call('undo', trans_id, id(txn))
def _finish_undo(self, msgid):
return self._storage._server.rpc._deferred_wait(msgid)
def checkCommitLockUndoFinish(self):
trans_id = self._get_trans_id()
oid, txn = self._start_txn()
msgid = self._begin_undo(trans_id, txn)
self._begin_threads()
self._finish_undo(msgid)
self._storage.tpc_vote(txn)
self._storage.tpc_finish(txn)
self._storage.load(oid, '')
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockUndoAbort(self):
trans_id = self._get_trans_id()
oid, txn = self._start_txn()
msgid = self._begin_undo(trans_id, txn)
self._begin_threads()
self._finish_undo(msgid)
self._storage.tpc_vote(txn)
self._storage.tpc_abort(txn)
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockUndoClose(self):
trans_id = self._get_trans_id()
oid, txn = self._start_txn()
msgid = self._begin_undo(trans_id, txn)
self._begin_threads()
self._finish_undo(msgid)
self._storage.tpc_vote(txn)
self._storage.close()
self._finish_threads()
self._cleanup()
lib/python/ZEO/tests/ConnectionTests.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
import sys
import time
import random
import asyncore
import tempfile
import threading
import logging
import ZEO.ServerStub
from ZEO.ClientStorage import ClientStorage
from ZEO.Exceptions import ClientDisconnected
from ZEO.zrpc.marshal import Marshaller
from ZEO.tests import forker
from ZODB.DB import DB
from ZODB.POSException import ReadOnlyError, ConflictError
from ZODB.tests.StorageTestBase import StorageTestBase
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase \
import zodb_pickle, zodb_unpickle, handle_all_serials, handle_serials
import transaction
from transaction import Transaction
logger = logging.getLogger('ZEO.tests.ConnectionTests')
ZERO = '\0'*8
class TestServerStub(ZEO.ServerStub.StorageServer):
__super_getInvalidations = ZEO.ServerStub.StorageServer.getInvalidations
def getInvalidations(self, tid):
# squirrel the results away for inspection by test case
self._last_invals = self.__super_getInvalidations(tid)
return self._last_invals
class TestClientStorage(ClientStorage):
test_connection = False
StorageServerStubClass = TestServerStub
def verify_cache(self, stub):
self.end_verify = threading.Event()
self.verify_result = ClientStorage.verify_cache(self, stub)
def endVerify(self):
ClientStorage.endVerify(self)
self.end_verify.set()
def testConnection(self, conn):
try:
return ClientStorage.testConnection(self, conn)
finally:
self.test_connection = True
class DummyDB:
def invalidate(self, *args, **kwargs):
pass
class CommonSetupTearDown(StorageTestBase):
"""Common boilerplate"""
__super_setUp = StorageTestBase.setUp
__super_tearDown = StorageTestBase.tearDown
keep = 0
invq = None
timeout = None
monitor = 0
db_class = DummyDB
def setUp(self):
"""Test setup for connection tests.
This starts only one server; a test may start more servers by
calling self._newAddr() and then self.startServer(index=i)
for i in 1, 2, ...
"""
self.__super_setUp()
logging.info("setUp() %s", self.id())
self.file = tempfile.mktemp()
self.addr = []
self._pids = []
self._servers = []
self.conf_paths = []
self.caches = []
self._newAddr()
self.startServer()
def tearDown(self):
"""Try to cause the tests to halt"""
logging.info("tearDown() %s" % self.id())
for p in self.conf_paths:
os.remove(p)
if getattr(self, '_storage', None) is not None:
self._storage.close()
if hasattr(self._storage, 'cleanup'):
logging.debug("cleanup storage %s" %
self._storage.__name__)
self._storage.cleanup()
for adminaddr in self._servers:
if adminaddr is not None:
forker.shutdown_zeo_server(adminaddr)
if hasattr(os, 'waitpid'):
# Not in Windows Python until 2.3
for pid in self._pids:
os.waitpid(pid, 0)
for c in self.caches:
for i in 0, 1:
for ext in "", ".trace":
base = "%s-%s.zec%s" % (c, "1", ext)
path = os.path.join(tempfile.tempdir, base)
# On Windows before 2.3, we don't have a way to wait for
# the spawned server(s) to close, and they inherited
# file descriptors for our open files. So long as those
# processes are alive, we can't delete the files. Try
# a few times then give up.
need_to_delete = False
if os.path.exists(path):
need_to_delete = True
for dummy in range(5):
try:
os.unlink(path)
except:
time.sleep(0.5)
else:
need_to_delete = False
break
if need_to_delete:
os.unlink(path) # sometimes this is just gonna fail
self.__super_tearDown()
def _newAddr(self):
self.addr.append(self._getAddr())
def _getAddr(self):
# port+1 is also used, so only draw even port numbers
return 'localhost', random.randrange(25000, 30000, 2)
def getConfig(self, path, create, read_only):
raise NotImplementedError
cache_id = 1
def openClientStorage(self, cache=None, cache_size=200000, wait=1,
read_only=0, read_only_fallback=0,
username=None, password=None, realm=None):
if cache is None:
cache = str(self.__class__.cache_id)
self.__class__.cache_id += 1
self.caches.append(cache)
storage = TestClientStorage(self.addr,
client=cache,
var=tempfile.tempdir,
cache_size=cache_size,
wait=wait,
min_disconnect_poll=0.1,
read_only=read_only,
read_only_fallback=read_only_fallback,
username=username,
password=password,
realm=realm)
storage.registerDB(DummyDB(), None)
return storage
def getServerConfig(self, addr, ro_svr):
zconf = forker.ZEOConfig(addr)
if ro_svr:
zconf.read_only = 1
if self.monitor:
zconf.monitor_address = ("", 42000)
if self.invq:
zconf.invalidation_queue_size = self.invq
if self.timeout:
zconf.transaction_timeout = self.timeout
return zconf
def startServer(self, create=1, index=0, read_only=0, ro_svr=0, keep=None):
addr = self.addr[index]
logging.info("startServer(create=%d, index=%d, read_only=%d) @ %s" %
(create, index, read_only, addr))
path = "%s.%d" % (self.file, index)
sconf = self.getConfig(path, create, read_only)
zconf = self.getServerConfig(addr, ro_svr)
if keep is None:
keep = self.keep
zeoport, adminaddr, pid, path = forker.start_zeo_server(
sconf, zconf, addr[1], keep)
self.conf_paths.append(path)
self._pids.append(pid)
self._servers.append(adminaddr)
def shutdownServer(self, index=0):
logging.info("shutdownServer(index=%d) @ %s" %
(index, self._servers[index]))
adminaddr = self._servers[index]
if adminaddr is not None:
forker.shutdown_zeo_server(adminaddr)
self._servers[index] = None
def pollUp(self, timeout=30.0, storage=None):
if storage is None:
storage = self._storage
# Poll until we're connected
now = time.time()
giveup = now + timeout
while not storage.is_connected():
asyncore.poll(0.1)
now = time.time()
if now > giveup:
self.fail("timed out waiting for storage to connect")
def pollDown(self, timeout=30.0):
# Poll until we're disconnected
now = time.time()
giveup = now + timeout
while self._storage.is_connected():
asyncore.poll(0.1)
now = time.time()
if now > giveup:
self.fail("timed out waiting for storage to disconnect")
class ConnectionTests(CommonSetupTearDown):
"""Tests that explicitly manage the server process.
To test the cache or re-connection, these test cases explicit
start and stop a ZEO storage server.
"""
def checkMultipleAddresses(self):
for i in range(4):
self._newAddr()
self._storage = self.openClientStorage('test', 100000)
oid = self._storage.new_oid()
obj = MinPO(12)
self._dostore(oid, data=obj)
self._storage.close()
def checkMultipleServers(self):
# XXX crude test at first -- just start two servers and do a
# commit at each one.
self._newAddr()
self._storage = self.openClientStorage('test', 100000)
self._dostore()
self.shutdownServer(index=0)
self.startServer(index=1)
# If we can still store after shutting down one of the
# servers, we must be reconnecting to the other server.
did_a_store = 0
for i in range(10):
try:
self._dostore()
did_a_store = 1
break
except ClientDisconnected:
time.sleep(0.5)
self.assert_(did_a_store)
self._storage.close()
def checkReadOnlyClient(self):
# Open a read-only client to a read-write server; stores fail
# Start a read-only client for a read-write server
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
def checkReadOnlyServer(self):
# Open a read-only client to a read-only *server*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, ro_svr=1)
# Start a read-only client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
# Get rid of the 'test left new threads behind' warning
time.sleep(0.1)
def checkReadOnlyFallbackWritable(self):
# Open a fallback client to a read-write server; stores succeed
# Start a read-only-fallback client for a read-write server
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should succeed here
self._dostore()
self._storage.close()
def checkReadOnlyFallbackReadOnlyServer(self):
# Open a fallback client to a read-only *server*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, ro_svr=1)
# Start a read-only-fallback client
self._storage = self.openClientStorage(read_only_fallback=1)
self.assert_(self._storage.isReadOnly())
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
# XXX Compare checkReconnectXXX() here to checkReconnection()
# further down. Is the code here hopelessly naive, or is
# checkReconnection() overwrought?
def checkReconnectWritable(self):
# A read-write client reconnects to a read-write server
# Start a client
self._storage = self.openClientStorage()
# Stores should succeed here
self._dostore()
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server
self.startServer(create=0)
# Poll until the client connects
self.pollUp()
# Stores should succeed here
self._dostore()
self._storage.close()
def checkDisconnectionError(self):
# Make sure we get a ClientDisconnected when we try to read an
# object when we're not connected to a storage server and the
# object is not in the cache.
self.shutdownServer()
self._storage = self.openClientStorage('test', 1000, wait=0)
self.assertRaises(ClientDisconnected,
self._storage.load, 'fredwash', '')
self._storage.close()
def checkDisconnectedAbort(self):
self._storage = self.openClientStorage()
self._dostore()
oids = [self._storage.new_oid() for i in range(5)]
txn = Transaction()
self._storage.tpc_begin(txn)
for oid in oids:
data = zodb_pickle(MinPO(oid))
self._storage.store(oid, None, data, '', txn)
self.shutdownServer()
self.assertRaises(ClientDisconnected, self._storage.tpc_vote, txn)
self._storage.tpc_abort(txn)
self.startServer(create=0)
self._storage._wait()
self._dostore()
# This test is supposed to cover the following error, although
# I don't have much confidence that it does. The likely
# explanation for the error is that the _tbuf contained
# objects that weren't in the _seriald, because the client was
# interrupted waiting for tpc_vote() to return. When the next
# transaction committed, it tried to do something with the
# bogus _tbuf entries. The exaplanation is wrong/incomplete,
# because tpc_begin() should clear the _tbuf.
# 2003-01-15T15:44:19 ERROR(200) ZODB A storage error occurred
# in the last phase of a two-phase commit. This shouldn't happen.
# Traceback (innermost last):
# Module ZODB.Transaction, line 359, in _finish_one
# Module ZODB.Connection, line 691, in tpc_finish
# Module ZEO.ClientStorage, line 679, in tpc_finish
# Module ZEO.ClientStorage, line 709, in _update_cache
# KeyError: ...
def checkBasicPersistence(self):
# Verify cached data persists across client storage instances.
# To verify that the cache is being used, the test closes the
# server and then starts a new client with the server down.
# When the server is down, a load() gets the data from its cache.
self._storage = self.openClientStorage('test', 100000)
oid = self._storage.new_oid()
obj = MinPO(12)
revid1 = self._dostore(oid, data=obj)
self._storage.close()
self.shutdownServer()
self._storage = self.openClientStorage('test', 100000, wait=0)
data, revid2 = self._storage.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO(12))
self.assertEqual(revid1, revid2)
self._storage.close()
def checkRollover(self):
# Check that the cache works when the files are swapped.
# In this case, only one object fits in a cache file. When the
# cache files swap, the first object is effectively uncached.
self._storage = self.openClientStorage('test', 1000)
oid1 = self._storage.new_oid()
obj1 = MinPO("1" * 500)
self._dostore(oid1, data=obj1)
oid2 = self._storage.new_oid()
obj2 = MinPO("2" * 500)
self._dostore(oid2, data=obj2)
self._storage.close()
self.shutdownServer()
self._storage = self.openClientStorage('test', 1000, wait=0)
self._storage.load(oid1, '')
self._storage.load(oid2, '')
self._storage.close()
def checkReconnection(self):
# Check that the client reconnects when a server restarts.
# XXX Seem to get occasional errors that look like this:
# File ZEO/zrpc2.py, line 217, in handle_request
# File ZEO/StorageServer.py, line 325, in storea
# File ZEO/StorageServer.py, line 209, in _check_tid
# StorageTransactionError: (None, <tid>)
# could system reconnect and continue old transaction?
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
obj = MinPO(12)
self._dostore(oid, data=obj)
logging.info("checkReconnection(): About to shutdown server")
self.shutdownServer()
logging.info("checkReconnection(): About to restart server")
self.startServer(create=0)
oid = self._storage.new_oid()
obj = MinPO(12)
while 1:
try:
self._dostore(oid, data=obj)
break
except ClientDisconnected:
# Maybe the exception mess is better now
logging.info("checkReconnection(): Error after"
" server restart; retrying.", exc_info=True)
transaction.abort()
# Give the other thread a chance to run.
time.sleep(0.1)
logging.info("checkReconnection(): finished")
self._storage.close()
def checkBadMessage1(self):
# not even close to a real message
self._bad_message("salty")
def checkBadMessage2(self):
# just like a real message, but with an unpicklable argument
global Hack
class Hack:
pass
msg = Marshaller().encode(1, 0, "foo", (Hack(),))
self._bad_message(msg)
del Hack
def _bad_message(self, msg):
# Establish a connection, then send the server an ill-formatted
# request. Verify that the connection is closed and that it is
# possible to establish a new connection.
self._storage = self.openClientStorage()
self._dostore()
# break into the internals to send a bogus message
zrpc_conn = self._storage._server.rpc
zrpc_conn.message_output(msg)
try:
self._dostore()
except ClientDisconnected:
pass
else:
self._storage.close()
self.fail("Server did not disconnect after bogus message")
self._storage.close()
self._storage = self.openClientStorage()
self._dostore()
self._storage.close()
# Test case for multiple storages participating in a single
# transaction. This is not really a connection test, but it needs
# about the same infrastructure (several storage servers).
# XXX WARNING: with the current ZEO code, this occasionally fails.
# That's the point of this test. :-)
def NOcheckMultiStorageTransaction(self):
# Configuration parameters (larger values mean more likely deadlocks)
N = 2
# These don't *have* to be all the same, but it's convenient this way
self.nservers = N
self.nthreads = N
self.ntrans = N
self.nobj = N
# Start extra servers
for i in range(1, self.nservers):
self._newAddr()
self.startServer(index=i)
# Spawn threads that each do some transactions on all storages
threads = []
try:
for i in range(self.nthreads):
t = MSTThread(self, "T%d" % i)
threads.append(t)
t.start()
# Wait for all threads to finish
for t in threads:
t.join(60)
self.failIf(t.isAlive(), "%s didn't die" % t.getName())
finally:
for t in threads:
t.closeclients()
def checkCrossDBInvalidations(self):
db1 = DB(self.openClientStorage())
c1 = db1.open()
r1 = c1.root()
r1["a"] = MinPO("a")
transaction.commit()
db2 = DB(self.openClientStorage())
r2 = db2.open().root()
self.assertEqual(r2["a"].value, "a")
r2["b"] = MinPO("b")
transaction.commit()
# make sure the invalidation is received in the other client
for i in range(10):
c1._storage.sync()
if c1._invalidated.has_key(r1._p_oid):
break
time.sleep(0.1)
self.assert_(c1._invalidated.has_key(r1._p_oid))
# force the invalidations to be applied...
c1.sync()
r1.keys() # unghostify
self.assertEqual(r1._p_serial, r2._p_serial)
db2.close()
db1.close()
class InvqTests(CommonSetupTearDown):
invq = 3
def checkQuickVerificationWith2Clients(self):
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
oid2 = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "full verification")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
revid = self._dostore(oid, revid)
# Create a second object and revision to guarantee it doesn't
# show up in the list of invalidations sent when perstore restarts.
revid2 = self._dostore(oid2)
revid2 = self._dostore(oid2, revid2)
# sync() is needed to prevent invalidation for oid from arriving
# in the middle of the load() call.
perstorage.sync()
perstorage.load(oid, '')
perstorage.close()
revid = self._dostore(oid, revid)
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "quick verification")
self.assertEqual(perstorage._server._last_invals,
(revid, [(oid, '')]))
self.assertEqual(perstorage.load(oid, ''),
self._storage.load(oid, ''))
perstorage.close()
def checkVerificationWith2ClientsInvqOverflow(self):
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "full verification")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
revid = self._dostore(oid, revid)
perstorage.load(oid, '')
perstorage.close()
# the test code sets invq bound to 2
for i in range(5):
revid = self._dostore(oid, revid)
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
t = time.time() + 30
while not perstorage.end_verify.isSet():
perstorage.sync()
if time.time() > t:
self.fail("timed out waiting for endVerify")
self.assertEqual(self._storage.load(oid, '')[1], revid)
self.assertEqual(perstorage.load(oid, ''),
self._storage.load(oid, ''))
perstorage.close()
class ReconnectionTests(CommonSetupTearDown):
# The setUp() starts a server automatically. In order for its
# state to persist, we set the class variable keep to 1. In
# order for its state to be cleaned up, the last startServer()
# call in the test must pass keep=0.
keep = 1
invq = 2
def checkReadOnlyStorage(self):
# Open a read-only client to a read-only *storage*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a read-only client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
def checkReadOnlyFallbackReadOnlyStorage(self):
# Open a fallback client to a read-only *storage*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a read-only-fallback client
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
def checkReconnectReadOnly(self):
# A read-only client reconnects from a read-write to a
# read-only server
# Start a client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should still fail
self.assertRaises(ReadOnlyError, self._dostore)
# Restart the server
self.startServer(create=0, read_only=1, keep=0)
# Poll until the client connects
self.pollUp()
# Stores should still fail
self.assertRaises(ReadOnlyError, self._dostore)
def checkReconnectFallback(self):
# A fallback client reconnects from a read-write to a
# read-only server
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should succeed here
self._dostore()
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server
self.startServer(create=0, read_only=1, keep=0)
# Poll until the client connects
self.pollUp()
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
def checkReconnectUpgrade(self):
# A fallback client reconnects from a read-only to a
# read-write server
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, read_only=1)
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server, this time read-write
self.startServer(create=0, keep=0)
# Poll until the client sconnects
self.pollUp()
# Stores should now succeed
self._dostore()
def checkReconnectSwitch(self):
# A fallback client initially connects to a read-only server,
# then discovers a read-write server and switches to that
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Allocate a second address (for the second server)
self._newAddr()
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Start a read-write server
self.startServer(index=1, read_only=0, keep=0)
# After a while, stores should work
for i in range(300): # Try for 30 seconds
try:
self._dostore()
break
except (ClientDisconnected, ReadOnlyError):
# If the client isn't connected at all, sync() returns
# quickly and the test fails because it doesn't wait
# long enough for the client.
time.sleep(0.1)
else:
self.fail("Couldn't store after starting a read-write server")
def checkNoVerificationOnServerRestart(self):
self._storage = self.openClientStorage()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "full verification")
self._dostore()
self.shutdownServer()
self.pollDown()
self._storage.verify_result = None
self.startServer(create=0, keep=0)
self.pollUp()
# There were no transactions committed, so no verification
# should be needed.
self.assertEqual(self._storage.verify_result, "no verification")
def checkNoVerificationOnServerRestartWith2Clients(self):
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "full verification")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
self._dostore(oid, revid)
perstorage.load(oid, '')
self.shutdownServer()
self.pollDown()
self._storage.verify_result = None
perstorage.verify_result = None
logging.info('2ALLBEEF')
self.startServer(create=0, keep=0)
self.pollUp()
self.pollUp(storage=perstorage)
# There were no transactions committed, so no verification
# should be needed.
self.assertEqual(self._storage.verify_result, "no verification")
self.assertEqual(perstorage.verify_result, "no verification")
perstorage.close()
self._storage.close()
class TimeoutTests(CommonSetupTearDown):
timeout = 1
def checkTimeout(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_vote(txn)
time.sleep(2)
self.assertRaises(ClientDisconnected, storage.tpc_finish, txn)
storage.close()
def checkTimeoutOnAbort(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_vote(txn)
storage.tpc_abort(txn)
storage.close()
def checkTimeoutOnAbortNoLock(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_abort(txn)
storage.close()
def checkTimeoutAfterVote(self):
raises = self.assertRaises
unless = self.failUnless
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty
unless(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# Now do a store, sleeping before the finish so as to cause a timeout
t = Transaction()
storage.tpc_begin(t)
revid1 = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
storage.tpc_vote(t)
# Now sleep long enough for the storage to time out
time.sleep(3)
storage.sync()
unless(not storage.is_connected())
storage._wait()
unless(storage.is_connected())
# We expect finish to fail
raises(ClientDisconnected, storage.tpc_finish, t)
# The cache should still be empty
unless(not list(storage._cache.contents()))
# Load should fail since the object should not be in either the cache
# or the server.
raises(KeyError, storage.load, oid, '')
def checkTimeoutProvokingConflicts(self):
eq = self.assertEqual
raises = self.assertRaises
unless = self.failUnless
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty
unless(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# We need to successfully commit an object now so we have something to
# conflict about.
t = Transaction()
storage.tpc_begin(t)
revid1a = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
revid1b = storage.tpc_vote(t)
revid1 = handle_serials(oid, revid1a, revid1b)
storage.tpc_finish(t)
# Now do a store, sleeping before the finish so as to cause a timeout
obj.value = 8
t = Transaction()
storage.tpc_begin(t)
revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
revid2b = storage.tpc_vote(t)
revid2 = handle_serials(oid, revid2a, revid2b)
# Now sleep long enough for the storage to time out
time.sleep(3)
storage.sync()
unless(not storage.is_connected())
storage._wait()
unless(storage.is_connected())
# We expect finish to fail
raises(ClientDisconnected, storage.tpc_finish, t)
# Now we think we've committed the second transaction, but we really
# haven't. A third one should produce a POSKeyError on the server,
# which manifests as a ConflictError on the client.
obj.value = 9
t = Transaction()
storage.tpc_begin(t)
storage.store(oid, revid2, zodb_pickle(obj), '', t)
raises(ConflictError, storage.tpc_vote, t)
# Even aborting won't help
storage.tpc_abort(t)
storage.tpc_finish(t)
# Try again
obj.value = 10
t = Transaction()
storage.tpc_begin(t)
storage.store(oid, revid2, zodb_pickle(obj), '', t)
# Even aborting won't help
raises(ConflictError, storage.tpc_vote, t)
# Abort this one and try a transaction that should succeed
storage.tpc_abort(t)
storage.tpc_finish(t)
# Now do a store, sleeping before the finish so as to cause a timeout
obj.value = 11
t = Transaction()
storage.tpc_begin(t)
revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
revid2b = storage.tpc_vote(t)
revid2 = handle_serials(oid, revid2a, revid2b)
storage.tpc_finish(t)
# Now load the object and verify that it has a value of 11
data, revid = storage.load(oid, '')
eq(zodb_unpickle(data), MinPO(11))
eq(revid, revid2)
class MSTThread(threading.Thread):
__super_init = threading.Thread.__init__
def __init__(self, testcase, name):
self.__super_init(name=name)
self.testcase = testcase
self.clients = []
def run(self):
tname = self.getName()
testcase = self.testcase
# Create client connections to each server
clients = self.clients
for i in range(len(testcase.addr)):
c = testcase.openClientStorage(addr=testcase.addr[i])
c.__name = "C%d" % i
clients.append(c)
for i in range(testcase.ntrans):
# Because we want a transaction spanning all storages,
# we can't use _dostore(). This is several _dostore() calls
# expanded in-line (mostly).
# Create oid->serial mappings
for c in clients:
c.__oids = []
c.__serials = {}
# Begin a transaction
t = Transaction()
for c in clients:
#print "%s.%s.%s begin\n" % (tname, c.__name, i),
c.tpc_begin(t)
for j in range(testcase.nobj):
for c in clients:
# Create and store a new object on each server
oid = c.new_oid()
c.__oids.append(oid)
data = MinPO("%s.%s.t%d.o%d" % (tname, c.__name, i, j))
#print data.value
data = zodb_pickle(data)
s = c.store(oid, ZERO, data, '', t)
c.__serials.update(handle_all_serials(oid, s))
# Vote on all servers and handle serials
for c in clients:
#print "%s.%s.%s vote\n" % (tname, c.__name, i),
s = c.tpc_vote(t)
c.__serials.update(handle_all_serials(None, s))
# Finish on all servers
for c in clients:
#print "%s.%s.%s finish\n" % (tname, c.__name, i),
c.tpc_finish(t)
for c in clients:
# Check that we got serials for all oids
for oid in c.__oids:
testcase.failUnless(c.__serials.has_key(oid))
# Check that we got serials for no other oids
for oid in c.__serials.keys():
testcase.failUnless(oid in c.__oids)
def closeclients(self):
# Close clients opened by run()
for c in self.clients:
try:
c.close()
except:
pass
lib/python/ZEO/tests/InvalidationTests.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import threading
import time
from random import Random
import transaction
from BTrees.check import check, display
from BTrees.OOBTree import OOBTree
from ZEO.tests.TestThread import TestThread
from ZODB.DB import DB
from ZODB.POSException \
import ReadConflictError, ConflictError, VersionLockError
# The tests here let several threads have a go at one or more database
# instances simultaneously. Each thread appends a disjoint (from the
# other threads) sequence of increasing integers to an OOBTree, one at
# at time (per thread). This provokes lots of conflicts, and BTrees
# work hard at conflict resolution too. An OOBTree is used because
# that flavor has the smallest maximum bucket size, and so splits buckets
# more often than other BTree flavors.
#
# When these tests were first written, they provoked an amazing number
# of obscure timing-related bugs in cache consistency logic, revealed
# by failure of the BTree to pass internal consistency checks at the end,
# and/or by failure of the BTree to contain all the keys the threads
# thought they added (i.e., the keys for which transaction.commit()
# did not raise any exception).
class FailableThread(TestThread):
# mixin class
# subclass must provide
# - self.stop attribute (an event)
# - self._testrun() method
def testrun(self):
try:
self._testrun()
except:
# Report the failure here to all the other threads, so
# that they stop quickly.
self.stop.set()
raise
class StressTask:
# Append integers startnum, startnum + step, startnum + 2*step, ...
# to 'tree'. If sleep is given, sleep
# that long after each append. At the end, instance var .added_keys
# is a list of the ints the thread believes it added successfully.
def __init__(self, testcase, db, threadnum, startnum,
step=2, sleep=None):
self.db = db
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.tm = transaction.TransactionManager()
self.cn = self.db.open(txn_mgr=self.tm)
self.cn.sync()
def doStep(self):
tree = self.cn.root()["tree"]
key = self.startnum
tree[key] = self.threadnum
def commit(self):
cn = self.cn
key = self.startnum
self.tm.get().note("add key %s" % key)
try:
self.tm.get().commit()
except ConflictError, msg:
self.tm.get().abort()
cn.sync()
else:
if self.sleep:
time.sleep(self.sleep)
self.added_keys.append(key)
self.startnum += self.step
def cleanup(self):
self.tm.get().abort()
self.cn.close()
def _runTasks(rounds, *tasks):
'''run *task* interleaved for *rounds* rounds.'''
def commit(run, actions):
actions.append(':')
for t in run:
t.commit()
del run[:]
r = Random()
r.seed(1064589285) # make it deterministic
run = []
actions = []
try:
for i in range(rounds):
t = r.choice(tasks)
if t in run:
commit(run, actions)
run.append(t)
t.doStep()
actions.append(`t.startnum`)
commit(run,actions)
# stderr.write(' '.join(actions)+'\n')
finally:
for t in tasks:
t.cleanup()
class StressThread(FailableThread):
# Append integers startnum, startnum + step, startnum + 2*step, ...
# to 'tree' until Event stop is set. If sleep is given, sleep
# that long after each append. At the end, instance var .added_keys
# is a list of the ints the thread believes it added successfully.
def __init__(self, testcase, db, stop, threadnum, commitdict,
startnum, step=2, sleep=None):
TestThread.__init__(self, testcase)
self.db = db
self.stop = stop
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.commitdict = commitdict
def _testrun(self):
cn = self.db.open()
while not self.stop.isSet():
try:
tree = cn.root()["tree"]
break
except (ConflictError, KeyError):
transaction.abort()
cn.sync()
key = self.startnum
while not self.stop.isSet():
try:
tree[key] = self.threadnum
transaction.get().note("add key %s" % key)
transaction.commit()
self.commitdict[self] = 1
if self.sleep:
time.sleep(self.sleep)
except (ReadConflictError, ConflictError), msg:
transaction.abort()
# sync() is necessary here to process invalidations
# if we get a read conflict. In the read conflict case,
# no objects were modified so cn never got registered
# with the transaction.
cn.sync()
else:
self.added_keys.append(key)
key += self.step
cn.close()
class LargeUpdatesThread(FailableThread):
# A thread that performs a lot of updates. It attempts to modify
# more than 25 objects so that it can test code that runs vote
# in a separate thread when it modifies more than 25 objects.
def __init__(self, testcase, db, stop, threadnum, commitdict, startnum,
step=2, sleep=None):
TestThread.__init__(self, testcase)
self.db = db
self.stop = stop
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.commitdict = commitdict
def testrun(self):
try:
self._testrun()
except:
# Report the failure here to all the other threads, so
# that they stop quickly.
self.stop.set()
raise
def _testrun(self):
cn = self.db.open()
while not self.stop.isSet():
try:
tree = cn.root()["tree"]
break
except (ConflictError, KeyError):
# print "%d getting tree abort" % self.threadnum
transaction.abort()
cn.sync()
keys_added = {} # set of keys we commit
tkeys = []
while not self.stop.isSet():
# The test picks 50 keys spread across many buckets.
# self.startnum and self.step ensure that all threads use
# disjoint key sets, to minimize conflict errors.
nkeys = len(tkeys)
if nkeys < 50:
tkeys = range(self.startnum, 3000, self.step)
nkeys = len(tkeys)
step = max(int(nkeys / 50), 1)
keys = [tkeys[i] for i in range(0, nkeys, step)]
for key in keys:
try:
tree[key] = self.threadnum
except (ReadConflictError, ConflictError), msg:
# print "%d setting key %s" % (self.threadnum, msg)
transaction.abort()
cn.sync()
break
else:
# print "%d set #%d" % (self.threadnum, len(keys))
transaction.get().note("keys %s" % ", ".join(map(str, keys)))
try:
transaction.commit()
self.commitdict[self] = 1
if self.sleep:
time.sleep(self.sleep)
except ConflictError, msg:
# print "%d commit %s" % (self.threadnum, msg)
transaction.abort()
cn.sync()
continue
for k in keys:
tkeys.remove(k)
keys_added[k] = 1
# sync() is necessary here to process invalidations
# if we get a read conflict. In the read conflict case,
# no objects were modified so cn never got registered
# with the transaction.
cn.sync()
self.added_keys = keys_added.keys()
cn.close()
class VersionStressThread(FailableThread):
def __init__(self, testcase, db, stop, threadnum, commitdict, startnum,
step=2, sleep=None):
TestThread.__init__(self, testcase)
self.db = db
self.stop = stop
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.commitdict = commitdict
def testrun(self):
try:
self._testrun()
except:
# Report the failure here to all the other threads, so
# that they stop quickly.
self.stop.set()
raise
def _testrun(self):
commit = 0
key = self.startnum
while not self.stop.isSet():
version = "%s:%s" % (self.threadnum, key)
commit = not commit
if self.oneupdate(version, key, commit):
self.added_keys.append(key)
self.commitdict[self] = 1
key += self.step
def oneupdate(self, version, key, commit=1):
# The mess of sleeps below were added to reduce the number
# of VersionLockErrors, based on empirical observation.
# It looks like the threads don't switch enough without
# the sleeps.
cn = self.db.open(version)
while not self.stop.isSet():
try:
tree = cn.root()["tree"]
break
except (ConflictError, KeyError):
transaction.abort()
cn.sync()
while not self.stop.isSet():
try:
tree[key] = self.threadnum
transaction.commit()
if self.sleep:
time.sleep(self.sleep)
break
except (VersionLockError, ReadConflictError, ConflictError), msg:
transaction.abort()
# sync() is necessary here to process invalidations
# if we get a read conflict. In the read conflict case,
# no objects were modified so cn never got registered
# with the transaction.
cn.sync()
if self.sleep:
time.sleep(self.sleep)
try:
while not self.stop.isSet():
try:
if commit:
self.db.commitVersion(version)
transaction.get().note("commit version %s" % version)
else:
self.db.abortVersion(version)
transaction.get().note("abort version %s" % version)
transaction.commit()
if self.sleep:
time.sleep(self.sleep)
return commit
except ConflictError, msg:
transaction.abort()
cn.sync()
finally:
cn.close()
return 0
class InvalidationTests:
level = 2
# Minimum # of seconds the main thread lets the workers run. The
# test stops as soon as this much time has elapsed, and all threads
# have managed to commit a change.
MINTIME = 10
# Maximum # of seconds the main thread lets the workers run. We
# stop after this long has elapsed regardless of whether all threads
# have managed to commit a change.
MAXTIME = 300
StressThread = StressThread
def _check_tree(self, cn, tree):
# Make sure the BTree is sane at the C level.
retries = 3
while retries:
retries -= 1
try:
check(tree)
tree._check()
except ReadConflictError:
if retries:
transaction.abort()
cn.sync()
else:
raise
except:
display(tree)
raise
def _check_threads(self, tree, *threads):
# Make sure the thread's view of the world is consistent with
# the actual database state.
expected_keys = []
errormsgs = []
err = errormsgs.append
for t in threads:
if not t.added_keys:
err("thread %d didn't add any keys" % t.threadnum)
expected_keys.extend(t.added_keys)
expected_keys.sort()
actual_keys = list(tree.keys())
if expected_keys != actual_keys:
err("expected keys != actual keys")
for k in expected_keys:
if k not in actual_keys:
err("key %s expected but not in tree" % k)
for k in actual_keys:
if k not in expected_keys:
err("key %s in tree but not expected" % k)
if errormsgs:
display(tree)
self.fail('\n'.join(errormsgs))
def go(self, stop, commitdict, *threads):
# Run the threads
for t in threads:
t.start()
delay = self.MINTIME
start = time.time()
while time.time() - start <= self.MAXTIME:
stop.wait(delay)
if stop.isSet():
# Some thread failed. Stop right now.
break
delay = 2.0
if len(commitdict) >= len(threads):
break
# Some thread still hasn't managed to commit anything.
stop.set()
for t in threads:
t.cleanup()
def checkConcurrentUpdates2Storages_emulated(self):
self._storage = storage1 = self.openClientStorage()
storage2 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(storage2)
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
# DM: allow time for invalidations to come in and process them
time.sleep(0.1)
# Run two threads that update the BTree
t1 = StressTask(self, db1, 1, 1,)
t2 = StressTask(self, db2, 2, 2,)
_runTasks(100, t1, t2)
cn.sync()
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def checkConcurrentUpdates2Storages(self):
self._storage = storage1 = self.openClientStorage()
storage2 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(storage2)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1)
t2 = self.StressThread(self, db2, stop, 2, cd, 2)
self.go(stop, cd, t1, t2)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def checkConcurrentUpdates1Storage(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1, sleep=0.01)
t2 = self.StressThread(self, db1, stop, 2, cd, 2, sleep=0.01)
self.go(stop, cd, t1, t2)
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
def checkConcurrentUpdates2StoragesMT(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1, 3)
t2 = self.StressThread(self, db2, stop, 2, cd, 2, 3, 0.01)
t3 = self.StressThread(self, db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
def checkConcurrentUpdatesInVersions(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = VersionStressThread(self, db1, stop, 1, cd, 1, 3)
t2 = VersionStressThread(self, db2, stop, 2, cd, 2, 3, 0.01)
t3 = VersionStressThread(self, db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
def checkConcurrentLargeUpdates(self):
# Use 3 threads like the 2StorageMT test above.
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
for i in range(0, 3000, 2):
tree[i] = 0
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = LargeUpdatesThread(self, db1, stop, 1, cd, 1, 3, 0.02)
t2 = LargeUpdatesThread(self, db2, stop, 2, cd, 2, 3, 0.01)
t3 = LargeUpdatesThread(self, db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
# Purge the tree of the dummy entries mapping to 0.
losers = [k for k, v in tree.items() if v == 0]
for k in losers:
del tree[k]
transaction.commit()
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
lib/python/ZEO/tests/TestThread.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A Thread base class for use with unittest."""
from cStringIO import StringIO
import threading
import traceback
class TestThread(threading.Thread):
__super_init = threading.Thread.__init__
__super_run = threading.Thread.run
def __init__(self, testcase, group=None, target=None, name=None,
args=(), kwargs={}, verbose=None):
self.__super_init(group, target, name, args, kwargs, verbose)
self.setDaemon(1)
self._testcase = testcase
def run(self):
try:
self.testrun()
except Exception:
s = StringIO()
traceback.print_exc(file=s)
self._testcase.fail("Exception in thread %s:\n%s\n" %
(self, s.getvalue()))
def cleanup(self, timeout=15):
self.join(timeout)
if self.isAlive():
self._testcase.fail("Thread did not finish: %s" % self)
lib/python/ZEO/tests/ThreadTests.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Compromising positions involving threads."""
import threading
import transaction
from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
import ZEO.ClientStorage
ZERO = '\0'*8
class BasicThread(threading.Thread):
def __init__(self, storage, doNextEvent, threadStartedEvent):
self.storage = storage
self.trans = transaction.Transaction()
self.doNextEvent = doNextEvent
self.threadStartedEvent = threadStartedEvent
self.gotValueError = 0
self.gotDisconnected = 0
threading.Thread.__init__(self)
self.setDaemon(1)
def join(self):
threading.Thread.join(self, 10)
assert not self.isAlive()
class GetsThroughVoteThread(BasicThread):
# This thread gets partially through a transaction before it turns
# execution over to another thread. We're trying to establish that a
# tpc_finish() after a storage has been closed by another thread will get
# a ClientStorageError error.
#
# This class gets does a tpc_begin(), store(), tpc_vote() and is waiting
# to do the tpc_finish() when the other thread closes the storage.
def run(self):
self.storage.tpc_begin(self.trans)
oid = self.storage.new_oid()
self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans)
self.storage.tpc_vote(self.trans)
self.threadStartedEvent.set()
self.doNextEvent.wait(10)
try:
self.storage.tpc_finish(self.trans)
except ZEO.ClientStorage.ClientStorageError:
self.gotValueError = 1
self.storage.tpc_abort(self.trans)
class GetsThroughBeginThread(BasicThread):
# This class is like the above except that it is intended to be run when
# another thread is already in a tpc_begin(). Thus, this thread will
# block in the tpc_begin until another thread closes the storage. When
# that happens, this one will get disconnected too.
def run(self):
try:
self.storage.tpc_begin(self.trans)
except ZEO.ClientStorage.ClientStorageError:
self.gotValueError = 1
class ThreadTests:
# Thread 1 should start a transaction, but not get all the way through it.
# Main thread should close the connection. Thread 1 should then get
# disconnected.
def checkDisconnectedOnThread2Close(self):
doNextEvent = threading.Event()
threadStartedEvent = threading.Event()
thread1 = GetsThroughVoteThread(self._storage,
doNextEvent, threadStartedEvent)
thread1.start()
threadStartedEvent.wait(10)
self._storage.close()
doNextEvent.set()
thread1.join()
self.assertEqual(thread1.gotValueError, 1)
# Thread 1 should start a transaction, but not get all the way through
# it. While thread 1 is in the middle of the transaction, a second thread
# should start a transaction, and it will block in the tcp_begin() --
# because thread 1 has acquired the lock in its tpc_begin(). Now the main
# thread closes the storage and both sub-threads should get disconnected.
def checkSecondBeginFails(self):
doNextEvent = threading.Event()
threadStartedEvent = threading.Event()
thread1 = GetsThroughVoteThread(self._storage,
doNextEvent, threadStartedEvent)
thread2 = GetsThroughBeginThread(self._storage,
doNextEvent, threadStartedEvent)
thread1.start()
threadStartedEvent.wait(1)
thread2.start()
self._storage.close()
doNextEvent.set()
thread1.join()
thread2.join()
self.assertEqual(thread1.gotValueError, 1)
self.assertEqual(thread2.gotValueError, 1)
# Run a bunch of threads doing small and large stores in parallel
def checkMTStores(self):
threads = []
for i in range(5):
t = threading.Thread(target=self.mtstorehelper)
threads.append(t)
t.start()
for t in threads:
t.join(30)
for i in threads:
self.failUnless(not t.isAlive())
# Helper for checkMTStores
def mtstorehelper(self):
name = threading.currentThread().getName()
objs = []
for i in range(10):
objs.append(MinPO("X" * 200000))
objs.append(MinPO("X"))
for obj in objs:
self._dostore(data=obj)
lib/python/ZEO/tests/__init__.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
lib/python/ZEO/tests/auth_plaintext.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Implements plaintext password authentication. The password is stored in
an SHA hash in the Database. The client sends over the plaintext
password, and the SHA hashing is done on the server side.
This mechanism offers *no network security at all*; the only security
is provided by not storing plaintext passwords on disk.
"""
import sha
from ZEO.StorageServer import ZEOStorage
from ZEO.auth import register_module
from ZEO.auth.base import Client, Database
def session_key(username, realm, password):
return sha.new("%s:%s:%s" % (username, realm, password)).hexdigest()
class StorageClass(ZEOStorage):
def auth(self, username, password):
try:
dbpw = self.database.get_password(username)
except LookupError:
return 0
password_dig = sha.new(password).hexdigest()
if dbpw == password_dig:
self.connection.setSessionKey(session_key(username,
self.database.realm,
password))
return self.finish_auth(dbpw == password_dig)
class PlaintextClient(Client):
extensions = ["auth"]
def start(self, username, realm, password):
if self.stub.auth(username, password):
return session_key(username, realm, password)
else:
return None
register_module("plaintext", StorageClass, PlaintextClient, Database)
lib/python/ZEO/tests/deadlock.py 0 → 100644
View file @ 96172040
import ZODB
from ZODB.POSException import ConflictError
from ZEO.ClientStorage import ClientStorage, ClientDisconnected
from ZEO.zrpc.error import DisconnectedError
import os
import random
import time
L = range(1, 100)
def main():
z1 = ClientStorage(('localhost', 2001), wait=1)
z2 = ClientStorage(('localhost', 2002), wait=2)
db1 = ZODB.DB(z1)
db2 = ZODB.DB(z2)
c1 = db1.open()
c2 = db2.open()
r1 = c1.root()
r2 = c2.root()
while 1:
try:
try:
update(r1, r2)
except ConflictError, msg:
print msg
transaction.abort()
c1.sync()
c2.sync()
except (ClientDisconnected, DisconnectedError), err:
print "disconnected", err
time.sleep(2)
def update(r1, r2):
k1 = random.choice(L)
k2 = random.choice(L)
updates = [(k1, r1),
(k2, r2)]
random.shuffle(updates)
for key, root in updates:
root[key] = time.time()
transaction.commit()
print os.getpid(), k1, k2
if __name__ == "__main__":
main()
lib/python/ZEO/tests/forker.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Library for forking storage server and connecting client storage"""
import os
import sys
import time
import errno
import socket
import logging
import StringIO
import tempfile
import logging
logger = logging.getLogger('ZEO.tests.forker')
class ZEOConfig:
"""Class to generate ZEO configuration file. """
def __init__(self, addr):
self.address = addr
self.read_only = None
self.invalidation_queue_size = None
self.monitor_address = None
self.transaction_timeout = None
self.authentication_protocol = None
self.authentication_database = None
self.authentication_realm = None
def dump(self, f):
print >> f, "<zeo>"
print >> f, "address %s:%s" % self.address
if self.read_only is not None:
print >> f, "read-only", self.read_only and "true" or "false"
if self.invalidation_queue_size is not None:
print >> f, "invalidation-queue-size", self.invalidation_queue_size
if self.monitor_address is not None:
print >> f, "monitor-address %s:%s" % self.monitor_address
if self.transaction_timeout is not None:
print >> f, "transaction-timeout", self.transaction_timeout
if self.authentication_protocol is not None:
print >> f, "authentication-protocol", self.authentication_protocol
if self.authentication_database is not None:
print >> f, "authentication-database", self.authentication_database
if self.authentication_realm is not None:
print >> f, "authentication-realm", self.authentication_realm
print >> f, "</zeo>"
logger = logging.getLogger()
print >> f
print >> f, "<eventlog>"
print >> f, "level", logger.level
for handler in logger.handlers:
if isinstance(handler, logging.FileHandler):
path = handler.baseFilename
elif isinstance(handler, logging.StreamHandler):
stream = handler.stream
if stream.name == "<stdout>":
path = "STDOUT"
elif stream.name == "<stderr>":
path = "STDERR"
else:
# just drop it on the floor; unlikely an issue when testing
continue
else:
# just drop it on the floor; unlikely an issue when testing
continue
# This doesn't convert the level values to names, so the
# generated configuration isn't as nice as it could be,
# but it doesn't really need to be.
print >> f, "<logfile>"
print >> f, "level", handler.level
print >> f, "path ", path
if handler.formatter:
formatter = handler.formatter
if formatter._fmt:
print >> f, "format", encode_format(formatter._fmt)
if formatter.datefmt:
print >> f, "dateformat", encode_format(formatter.datefmt)
print >> f, "</logfile>"
print >> f, "</eventlog>"
def __str__(self):
f = StringIO.StringIO()
self.dump(f)
return f.getvalue()
def encode_format(fmt):
# The list of replacements mirrors
# ZConfig.components.logger.handlers._control_char_rewrites
for xform in (("\n", r"\n"), ("\t", r"\t"), ("\b", r"\b"),
("\f", r"\f"), ("\r", r"\r")):
fmt = fmt.replace(*xform)
return fmt
def start_zeo_server(storage_conf, zeo_conf, port, keep=0):
"""Start a ZEO server in a separate process.
Takes two positional arguments a string containing the storage conf
and a ZEOConfig object.
Returns the ZEO port, the test server port, the pid, and the path
to the config file.
"""
# Store the config info in a temp file.
tmpfile = tempfile.mktemp(".conf")
fp = open(tmpfile, 'w')
zeo_conf.dump(fp)
fp.write(storage_conf)
fp.close()
# Find the zeoserver script
import ZEO.tests.zeoserver
script = ZEO.tests.zeoserver.__file__
if script.endswith('.pyc'):
script = script[:-1]
# Create a list of arguments, which we'll tuplify below
qa = _quote_arg
args = [qa(sys.executable), qa(script), '-C', qa(tmpfile)]
if keep:
args.append("-k")
d = os.environ.copy()
d['PYTHONPATH'] = os.pathsep.join(sys.path)
pid = os.spawnve(os.P_NOWAIT, sys.executable, tuple(args), d)
adminaddr = ('localhost', port + 1)
# We need to wait until the server starts, but not forever.
# 30 seconds is a somewhat arbitrary upper bound. A BDBStorage
# takes a long time to open -- more than 10 seconds on occasion.
for i in range(120):
time.sleep(0.25)
try:
logger.debug('connect %s', i)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(adminaddr)
ack = s.recv(1024)
s.close()
logging.debug('acked: %s' % ack)
break
except socket.error, e:
if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
raise
s.close()
else:
logging.debug('boo hoo')
raise
return ('localhost', port), adminaddr, pid, tmpfile
if sys.platform[:3].lower() == "win":
def _quote_arg(s):
return '"%s"' % s
else:
def _quote_arg(s):
return s
def shutdown_zeo_server(adminaddr):
# Do this in a loop to guard against the possibility that the
# client failed to connect to the adminaddr earlier. That really
# only requires two iterations, but do a third for pure
# superstition.
for i in range(3):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(adminaddr)
except socket.error, e:
if e[0] == errno.ECONNREFUSED and i > 0:
break
raise
try:
ack = s.recv(1024)
except socket.error, e:
if e[0] == errno.ECONNRESET:
raise
ack = 'no ack received'
logger.debug('shutdown_zeo_server(): acked: %s' % ack)
s.close()
lib/python/ZEO/tests/multi.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A multi-client test of the ZEO storage server"""
# XXX This code is currently broken.
import ZODB, ZODB.DB, ZODB.FileStorage, ZODB.POSException
import persistent
import persistent.mapping
import transaction
from ZEO.tests import forker
import os
import tempfile
import time
import types
VERBOSE = 1
CLIENTS = 4
RECORDS_PER_CLIENT = 100
CONFLICT_DELAY = 0.1
CONNECT_DELAY = 0.1
CLIENT_CACHE = '' # use temporary cache
class Record(persistent.Persistent):
def __init__(self, client=None, value=None):
self.client = client
self.value = None
self.next = None
def set_next(self, next):
self.next = next
class Stats(persistent.Persistent):
def __init__(self):
self.begin = time.time()
self.end = None
def done(self):
self.end = time.time()
def init_storage():
path = tempfile.mktemp()
if VERBOSE:
print "FileStorage path:", path
fs = ZODB.FileStorage.FileStorage(path)
db = ZODB.DB(fs)
root = db.open().root()
root["multi"] = persistent.mapping.PersistentMapping()
transaction.commit()
return fs
def start_server(addr):
storage = init_storage()
pid, exit = forker.start_zeo_server(storage, addr)
return pid, exit
def start_client(addr, client_func=None):
pid = os.fork()
if pid == 0:
try:
import ZEO.ClientStorage
if VERBOSE:
print "Client process started:", os.getpid()
cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE)
if client_func is None:
run(cli)
else:
client_func(cli)
cli.close()
finally:
os._exit(0)
else:
return pid
def run(storage):
if hasattr(storage, 'is_connected'):
while not storage.is_connected():
time.sleep(CONNECT_DELAY)
pid = os.getpid()
print "Client process connected:", pid, storage
db = ZODB.DB(storage)
root = db.open().root()
while 1:
try:
s = root[pid] = Stats()
transaction.commit()
except ZODB.POSException.ConflictError:
transaction.abort()
time.sleep(CONFLICT_DELAY)
else:
break
dict = root["multi"]
prev = None
i = 0
while i < RECORDS_PER_CLIENT:
try:
size = len(dict)
r = dict[size] = Record(pid, size)
if prev:
prev.set_next(r)
transaction.commit()
except ZODB.POSException.ConflictError, err:
transaction.abort()
time.sleep(CONFLICT_DELAY)
else:
i = i + 1
if VERBOSE and (i < 5 or i % 10 == 0):
print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT)
s.done()
transaction.commit()
print "Client completed:", pid
def main(client_func=None):
if VERBOSE:
print "Main process:", os.getpid()
addr = tempfile.mktemp()
t0 = time.time()
server_pid, server = start_server(addr)
t1 = time.time()
pids = []
for i in range(CLIENTS):
pids.append(start_client(addr, client_func))
for pid in pids:
assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \
(repr(pid), type(pid))
try:
if VERBOSE:
print "waitpid(%s)" % repr(pid)
os.waitpid(pid, 0)
except os.error, err:
print "waitpid(%s) failed: %s" % (repr(pid), err)
t2 = time.time()
server.close()
os.waitpid(server_pid, 0)
# XXX Should check that the results are consistent!
print "Total time:", t2 - t0
print "Server start time", t1 - t0
print "Client time:", t2 - t1
if __name__ == "__main__":
main()
lib/python/ZEO/tests/speed.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
usage="""Test speed of a ZODB storage
Options:
-d file The data file to use as input.
The default is this script.
-n n The number of repititions
-s module A module that defines a 'Storage'
attribute, which is an open storage.
If not specified, a FileStorage will ne
used.
-z Test compressing data
-D Run in debug mode
-L Test loads as well as stores by minimizing
the cache after eachrun
-M Output means only
-C Run with a persistent client cache
-U Run ZEO using a Unix domain socket
-t n Number of concurrent threads to run.
"""
import asyncore
import sys, os, getopt, time
##sys.path.insert(0, os.getcwd())
import persistent
import transaction
import ZODB
from ZODB.POSException import ConflictError
from ZEO.tests import forker
class P(persistent.Persistent):
pass
fs_name = "zeo-speed.fs"
class ZEOExit(asyncore.file_dispatcher):
"""Used to exit ZEO.StorageServer when run is done"""
def writable(self):
return 0
def readable(self):
return 1
def handle_read(self):
buf = self.recv(4)
assert buf == "done"
self.delete_fs()
os._exit(0)
def handle_close(self):
print "Parent process exited unexpectedly"
self.delete_fs()
os._exit(0)
def delete_fs(self):
os.unlink(fs_name)
os.unlink(fs_name + ".lock")
os.unlink(fs_name + ".tmp")
def work(db, results, nrep, compress, data, detailed, minimize, threadno=None):
for j in range(nrep):
for r in 1, 10, 100, 1000:
t = time.time()
conflicts = 0
jar = db.open()
while 1:
try:
transaction.begin()
rt = jar.root()
key = 's%s' % r
if rt.has_key(key):
p = rt[key]
else:
rt[key] = p =P()
for i in range(r):
v = getattr(p, str(i), P())
if compress is not None:
v.d = compress(data)
else:
v.d = data
setattr(p, str(i), v)
transaction.commit()
except ConflictError:
conflicts = conflicts + 1
else:
break
jar.close()
t = time.time() - t
if detailed:
if threadno is None:
print "%s\t%s\t%.4f\t%d" % (j, r, t, conflicts)
else:
print "%s\t%s\t%.4f\t%d\t%d" % (j, r, t, conflicts,
threadno)
results[r].append((t, conflicts))
rt=d=p=v=None # release all references
if minimize:
time.sleep(3)
jar.cacheMinimize()
def main(args):
opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U')
s = None
compress = None
data=sys.argv[0]
nrep=5
minimize=0
detailed=1
cache = None
domain = 'AF_INET'
threads = 1
for o, v in opts:
if o=='-n': nrep = int(v)
elif o=='-d': data = v
elif o=='-s': s = v
elif o=='-z':
import zlib
compress = zlib.compress
elif o=='-L':
minimize=1
elif o=='-M':
detailed=0
elif o=='-D':
global debug
os.environ['STUPID_LOG_FILE']=''
os.environ['STUPID_LOG_SEVERITY']='-999'
debug = 1
elif o == '-C':
cache = 'speed'
elif o == '-U':
domain = 'AF_UNIX'
elif o == '-t':
threads = int(v)
zeo_pipe = None
if s:
s = __import__(s, globals(), globals(), ('__doc__',))
s = s.Storage
server = None
else:
s, server, pid = forker.start_zeo("FileStorage",
(fs_name, 1), domain=domain)
data=open(data).read()
db=ZODB.DB(s,
# disable cache deactivation
cache_size=4000,
cache_deactivate_after=6000,)
print "Beginning work..."
results={1:[], 10:[], 100:[], 1000:[]}
if threads > 1:
import threading
l = []
for i in range(threads):
t = threading.Thread(target=work,
args=(db, results, nrep, compress, data,
detailed, minimize, i))
l.append(t)
for t in l:
t.start()
for t in l:
t.join()
else:
work(db, results, nrep, compress, data, detailed, minimize)
if server is not None:
server.close()
os.waitpid(pid, 0)
if detailed:
print '-'*24
print "num\tmean\tmin\tmax"
for r in 1, 10, 100, 1000:
times = []
for time, conf in results[r]:
times.append(time)
t = mean(times)
print "%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times))
def mean(l):
tot = 0
for v in l:
tot = tot + v
return tot / len(l)
##def compress(s):
## c = zlib.compressobj()
## o = c.compress(s)
## return o + c.flush()
if __name__=='__main__':
main(sys.argv[1:])
lib/python/ZEO/tests/stress.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A ZEO client-server stress test to look for leaks.
The stress test should run in an infinite loop and should involve
multiple connections.
"""
# XXX This code is currently broken.
import transaction
import ZODB
from ZODB.MappingStorage import MappingStorage
from ZODB.tests import MinPO
from ZEO.ClientStorage import ClientStorage
from ZEO.tests import forker
import os
import random
import types
NUM_TRANSACTIONS_PER_CONN = 10
NUM_CONNECTIONS = 10
NUM_ROOTS = 20
MAX_DEPTH = 20
MIN_OBJSIZE = 128
MAX_OBJSIZE = 2048
def an_object():
"""Return an object suitable for a PersistentMapping key"""
size = random.randrange(MIN_OBJSIZE, MAX_OBJSIZE)
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
buf = f.read(size)
f.close()
return buf
else:
f = open(MinPO.__file__)
l = list(f.read(size))
f.close()
random.shuffle(l)
return "".join(l)
def setup(cn):
"""Initialize the database with some objects"""
root = cn.root()
for i in range(NUM_ROOTS):
prev = an_object()
for j in range(random.randrange(1, MAX_DEPTH)):
o = MinPO.MinPO(prev)
prev = o
root[an_object()] = o
transaction.commit()
cn.close()
def work(cn):
"""Do some work with a transaction"""
cn.sync()
root = cn.root()
obj = random.choice(root.values())
# walk down to the bottom
while not isinstance(obj.value, types.StringType):
obj = obj.value
obj.value = an_object()
transaction.commit()
def main():
# Yuck! Need to cleanup forker so that the API is consistent
# across Unix and Windows, at least if that's possible.
if os.name == "nt":
zaddr, tport, pid = forker.start_zeo_server('MappingStorage', ())
def exitserver():
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(tport)
s.close()
else:
zaddr = '', random.randrange(20000, 30000)
pid, exitobj = forker.start_zeo_server(MappingStorage(), zaddr)
def exitserver():
exitobj.close()
while 1:
pid = start_child(zaddr)
print "started", pid
os.waitpid(pid, 0)
exitserver()
def start_child(zaddr):
pid = os.fork()
if pid != 0:
return pid
try:
_start_child(zaddr)
finally:
os._exit(0)
def _start_child(zaddr):
storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1)
db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS)
setup(db.open())
conns = []
conn_count = 0
for i in range(NUM_CONNECTIONS):
c = db.open()
c.__count = 0
conns.append(c)
conn_count += 1
while conn_count < 25:
c = random.choice(conns)
if c.__count > NUM_TRANSACTIONS_PER_CONN:
conns.remove(c)
c.close()
conn_count += 1
c = db.open()
c.__count = 0
conns.append(c)
else:
c.__count += 1
work(c)
if __name__ == "__main__":
main()
lib/python/ZEO/tests/testAuth.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for AuthZEO."""
import os
import tempfile
import time
import unittest
from ZEO import zeopasswd
from ZEO.Exceptions import ClientDisconnected
from ZEO.tests.ConnectionTests import CommonSetupTearDown
class AuthTest(CommonSetupTearDown):
__super_getServerConfig = CommonSetupTearDown.getServerConfig
__super_setUp = CommonSetupTearDown.setUp
__super_tearDown = CommonSetupTearDown.tearDown
realm = None
def setUp(self):
self.pwfile = tempfile.mktemp()
if self.realm:
self.pwdb = self.dbclass(self.pwfile, self.realm)
else:
self.pwdb = self.dbclass(self.pwfile)
self.pwdb.add_user("foo", "bar")
self.pwdb.save()
self._checkZEOpasswd()
self.__super_setUp()
def _checkZEOpasswd(self):
args = ["-f", self.pwfile, "-p", self.protocol]
if self.protocol == "plaintext":
from ZEO.auth.base import Database
zeopasswd.main(args + ["-d", "foo"], Database)
zeopasswd.main(args + ["foo", "bar"], Database)
else:
zeopasswd.main(args + ["-d", "foo"])
zeopasswd.main(args + ["foo", "bar"])
def tearDown(self):
self.__super_tearDown()
os.remove(self.pwfile)
def getConfig(self, path, create, read_only):
return "<mappingstorage 1/>"
def getServerConfig(self, addr, ro_svr):
zconf = self.__super_getServerConfig(addr, ro_svr)
zconf.authentication_protocol = self.protocol
zconf.authentication_database = self.pwfile
zconf.authentication_realm = self.realm
return zconf
def wait(self):
for i in range(25):
time.sleep(0.1)
if self._storage.test_connection:
return
self.fail("Timed out waiting for client to authenticate")
def testOK(self):
# Sleep for 0.2 seconds to give the server some time to start up
# seems to be needed before and after creating the storage
self._storage = self.openClientStorage(wait=0, username="foo",
password="bar", realm=self.realm)
self.wait()
self.assert_(self._storage._connection)
self._storage._connection.poll()
self.assert_(self._storage.is_connected())
# Make a call to make sure the mechanism is working
self._storage.versions()
def testNOK(self):
self._storage = self.openClientStorage(wait=0, username="foo",
password="noogie",
realm=self.realm)
self.wait()
# If the test established a connection, then it failed.
self.failIf(self._storage._connection)
def testUnauthenticatedMessage(self):
# Test that an unauthenticated message is rejected by the server
# if it was sent after the connection was authenticated.
# Sleep for 0.2 seconds to give the server some time to start up
# seems to be needed before and after creating the storage
self._storage = self.openClientStorage(wait=0, username="foo",
password="bar", realm=self.realm)
self.wait()
self._storage.versions()
# Manually clear the state of the hmac connection
self._storage._connection._SizedMessageAsyncConnection__hmac_send = None
# Once the client stops using the hmac, it should be disconnected.
self.assertRaises(ClientDisconnected, self._storage.versions)
class PlainTextAuth(AuthTest):
import ZEO.tests.auth_plaintext
protocol = "plaintext"
database = "authdb.sha"
dbclass = ZEO.tests.auth_plaintext.Database
realm = "Plaintext Realm"
class DigestAuth(AuthTest):
import ZEO.auth.auth_digest
protocol = "digest"
database = "authdb.digest"
dbclass = ZEO.auth.auth_digest.DigestDatabase
realm = "Digest Realm"
test_classes = [PlainTextAuth, DigestAuth]
def test_suite():
suite = unittest.TestSuite()
for klass in test_classes:
sub = unittest.makeSuite(klass)
suite.addTest(sub)
return suite
if __name__ == "__main__":
unittest.main(defaultTest='test_suite')
lib/python/ZEO/tests/testConnection.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test setup for ZEO connection logic.
The actual tests are in ConnectionTests.py; this file provides the
platform-dependent scaffolding.
"""
# System imports
import unittest
# Import the actual test class
from ZEO.tests import ConnectionTests, InvalidationTests
class FileStorageConfig:
def getConfig(self, path, create, read_only):
return """\
<filestorage 1>
path %s
create %s
read-only %s
</filestorage>""" % (path,
create and 'yes' or 'no',
read_only and 'yes' or 'no')
class BerkeleyStorageConfig:
def getConfig(self, path, create, read_only):
return """\
<fullstorage 1>
envdir %s
read-only %s
</fullstorage>""" % (path, read_only and "yes" or "no")
class MappingStorageConfig:
def getConfig(self, path, create, read_only):
return """<mappingstorage 1/>"""
class FileStorageConnectionTests(
FileStorageConfig,
ConnectionTests.ConnectionTests,
InvalidationTests.InvalidationTests
):
"""FileStorage-specific connection tests."""
level = 2
class FileStorageReconnectionTests(
FileStorageConfig,
ConnectionTests.ReconnectionTests,
):
"""FileStorage-specific re-connection tests."""
# Run this at level 1 because MappingStorage can't do reconnection tests
level = 1
class FileStorageInvqTests(
FileStorageConfig,
ConnectionTests.InvqTests
):
"""FileStorage-specific invalidation queue tests."""
level = 1
class FileStorageTimeoutTests(
FileStorageConfig,
ConnectionTests.TimeoutTests
):
level = 2
class BDBConnectionTests(
BerkeleyStorageConfig,
ConnectionTests.ConnectionTests,
InvalidationTests.InvalidationTests
):
"""Berkeley storage connection tests."""
level = 2
class BDBReconnectionTests(
BerkeleyStorageConfig,
ConnectionTests.ReconnectionTests
):
"""Berkeley storage re-connection tests."""
level = 2
class BDBInvqTests(
BerkeleyStorageConfig,
ConnectionTests.InvqTests
):
"""Berkeley storage invalidation queue tests."""
level = 2
class BDBTimeoutTests(
BerkeleyStorageConfig,
ConnectionTests.TimeoutTests
):
level = 2
class MappingStorageConnectionTests(
MappingStorageConfig,
ConnectionTests.ConnectionTests
):
"""Mapping storage connection tests."""
level = 1
# The ReconnectionTests can't work with MappingStorage because it's only an
# in-memory storage and has no persistent state.
class MappingStorageTimeoutTests(
MappingStorageConfig,
ConnectionTests.TimeoutTests
):
level = 1
test_classes = [FileStorageConnectionTests,
FileStorageReconnectionTests,
FileStorageInvqTests,
FileStorageTimeoutTests,
MappingStorageConnectionTests,
MappingStorageTimeoutTests]
def test_suite():
suite = unittest.TestSuite()
for klass in test_classes:
sub = unittest.makeSuite(klass, 'check')
suite.addTest(sub)
return suite
if __name__ == "__main__":
unittest.main(defaultTest='test_suite')
lib/python/ZEO/tests/testMonitor.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test that the monitor produce sensible results.
$Id: testMonitor.py,v 1.9 2004/02/27 00:31:52 faassen Exp $
"""
import socket
import unittest
from ZEO.tests.ConnectionTests import CommonSetupTearDown
from ZEO.monitor import StorageStats
class MonitorTests(CommonSetupTearDown):
monitor = 1
def get_monitor_output(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 42000))
L = []
while 1:
buf = s.recv(8192)
if buf:
L.append(buf)
else:
break
s.close()
return "".join(L)
def parse(self, s):
# Return a list of StorageStats, one for each storage.
lines = s.split("\n")
self.assert_(lines[0].startswith("ZEO monitor server"))
# lines[1] is a date
# Break up rest of lines into sections starting with Storage:
# and ending with a blank line.
sections = []
cur = None
for line in lines[2:]:
if line.startswith("Storage:"):
cur = [line]
elif line:
cur.append(line)
else:
if cur is not None:
sections.append(cur)
cur = None
assert cur is None # bug in the test code if this fails
d = {}
for sect in sections:
hdr = sect[0]
key, value = hdr.split(":")
storage = int(value)
s = d[storage] = StorageStats()
s.parse("\n".join(sect[1:]))
return d
def getConfig(self, path, create, read_only):
return """<mappingstorage 1/>"""
def testMonitor(self):
# just open a client to know that the server is up and running
# XXX should put this in setUp
self.storage = self.openClientStorage()
s = self.get_monitor_output()
self.storage.close()
self.assert_(s.find("monitor") != -1)
d = self.parse(s)
stats = d[1]
self.assertEqual(stats.clients, 1)
self.assertEqual(stats.commits, 0)
def test_suite():
return unittest.makeSuite(MonitorTests)
lib/python/ZEO/tests/testTransactionBuffer.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import random
import unittest
from ZEO.TransactionBuffer import TransactionBuffer
def random_string(size):
"""Return a random string of size size."""
l = [chr(random.randrange(256)) for i in range(size)]
return "".join(l)
def new_store_data():
"""Return arbitrary data to use as argument to store() method."""
return random_string(8), '', random_string(random.randrange(1000))
def new_invalidate_data():
"""Return arbitrary data to use as argument to invalidate() method."""
return random_string(8), ''
class TransBufTests(unittest.TestCase):
def checkTypicalUsage(self):
tbuf = TransactionBuffer()
tbuf.store(*new_store_data())
tbuf.invalidate(*new_invalidate_data())
for o in tbuf:
pass
def doUpdates(self, tbuf):
data = []
for i in range(10):
d = new_store_data()
tbuf.store(*d)
data.append(d)
d = new_invalidate_data()
tbuf.invalidate(*d)
data.append(d)
for i, x in enumerate(tbuf):
if x[2] is None:
# the tbuf add a dummy None to invalidates
x = x[:2]
self.assertEqual(x, data[i])
def checkOrderPreserved(self):
tbuf = TransactionBuffer()
self.doUpdates(tbuf)
def checkReusable(self):
tbuf = TransactionBuffer()
self.doUpdates(tbuf)
tbuf.clear()
self.doUpdates(tbuf)
tbuf.clear()
self.doUpdates(tbuf)
def test_suite():
return unittest.makeSuite(TransBufTests, 'check')
lib/python/ZEO/tests/testZEO.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for ZEO based on ZODB.tests."""
# System imports
import os
import random
import socket
import asyncore
import tempfile
import unittest
import logging
# ZODB test support
import ZODB
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_unpickle
# ZODB test mixin classes
from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \
TransactionalUndoStorage, TransactionalUndoVersionStorage, \
PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \
MTStorage, ReadOnlyStorage
from ZEO.ClientStorage import ClientStorage
from ZEO.tests import forker, Cache, CommitLockTests, ThreadTests
logger = logging.getLogger('ZEO.tests.testZEO')
class DummyDB:
def invalidate(self, *args):
pass
class MiscZEOTests:
"""ZEO tests that don't fit in elsewhere."""
def checkLargeUpdate(self):
obj = MinPO("X" * (10 * 128 * 1024))
self._dostore(data=obj)
def checkZEOInvalidation(self):
addr = self._storage._addr
storage2 = ClientStorage(addr, wait=1, min_disconnect_poll=0.1)
try:
oid = self._storage.new_oid()
ob = MinPO('first')
revid1 = self._dostore(oid, data=ob)
data, serial = storage2.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO('first'))
self.assertEqual(serial, revid1)
revid2 = self._dostore(oid, data=MinPO('second'), revid=revid1)
for n in range(3):
# Let the server and client talk for a moment.
# Is there a better way to do this?
asyncore.poll(0.1)
data, serial = storage2.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO('second'),
'Invalidation message was not sent!')
self.assertEqual(serial, revid2)
finally:
storage2.close()
def get_port():
"""Return a port that is not in use.
Checks if a port is in use by trying to connect to it. Assumes it
is not in use if connect raises an exception.
Raises RuntimeError after 10 tries.
"""
for i in range(10):
port = random.randrange(20000, 30000)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
try:
s.connect(('localhost', port))
except socket.error:
# XXX check value of error?
return port
finally:
s.close()
raise RuntimeError, "Can't find port"
class GenericTests(
# Base class for all ZODB tests
StorageTestBase.StorageTestBase,
# ZODB test mixin classes (in the same order as imported)
BasicStorage.BasicStorage,
PackableStorage.PackableStorage,
Synchronization.SynchronizedStorage,
MTStorage.MTStorage,
ReadOnlyStorage.ReadOnlyStorage,
# ZEO test mixin classes (in the same order as imported)
CommitLockTests.CommitLockVoteTests,
ThreadTests.ThreadTests,
# Locally defined (see above)
MiscZEOTests
):
"""Combine tests from various origins in one class."""
def setUp(self):
logger.info("setUp() %s", self.id()) # XXX is this really needed?
port = get_port()
zconf = forker.ZEOConfig(('', port))
zport, adminaddr, pid, path = forker.start_zeo_server(self.getConfig(),
zconf, port)
self._pids = [pid]
self._servers = [adminaddr]
self._conf_path = path
self._storage = ClientStorage(zport, '1', cache_size=20000000,
min_disconnect_poll=0.5, wait=1,
wait_timeout=60)
self._storage.registerDB(DummyDB(), None)
def tearDown(self):
self._storage.close()
os.remove(self._conf_path)
for server in self._servers:
forker.shutdown_zeo_server(server)
if hasattr(os, 'waitpid'):
# Not in Windows Python until 2.3
for pid in self._pids:
os.waitpid(pid, 0)
def open(self, read_only=0):
# XXX Needed to support ReadOnlyStorage tests. Ought to be a
# cleaner way.
addr = self._storage._addr
self._storage.close()
self._storage = ClientStorage(addr, read_only=read_only, wait=1)
def checkWriteMethods(self):
# ReadOnlyStorage defines checkWriteMethods. The decision
# about where to raise the read-only error was changed after
# Zope 2.5 was released. So this test needs to detect Zope
# of the 2.5 vintage and skip the test.
# The __version__ attribute was not present in Zope 2.5.
if hasattr(ZODB, "__version__"):
ReadOnlyStorage.ReadOnlyStorage.checkWriteMethods(self)
def checkSortKey(self):
key = '%s:%s' % (self._storage._storage, self._storage._server_addr)
self.assertEqual(self._storage.sortKey(), key)
class FullGenericTests(
GenericTests,
Cache.StorageWithCache,
Cache.TransUndoStorageWithCache,
CommitLockTests.CommitLockUndoTests,
ConflictResolution.ConflictResolvingStorage,
ConflictResolution.ConflictResolvingTransUndoStorage,
PackableStorage.PackableUndoStorage,
RevisionStorage.RevisionStorage,
TransactionalUndoStorage.TransactionalUndoStorage,
TransactionalUndoVersionStorage.TransactionalUndoVersionStorage,
VersionStorage.VersionStorage,
):
"""Extend GenericTests with tests that MappingStorage can't pass."""
class FileStorageTests(FullGenericTests):
"""Test ZEO backed by a FileStorage."""
level = 2
def getConfig(self):
filename = self.__fs_base = tempfile.mktemp()
return """\
<filestorage 1>
path %s
</filestorage>
""" % filename
class BDBTests(FullGenericTests):
"""ZEO backed by a Berkeley full storage."""
level = 2
def getConfig(self):
self._envdir = tempfile.mktemp()
return """\
<fullstorage 1>
envdir %s
</fullstorage>
""" % self._envdir
class MappingStorageTests(GenericTests):
"""ZEO backed by a Mapping storage."""
def getConfig(self):
return """<mappingstorage 1/>"""
# XXX There are still a bunch of tests that fail. Are there
# still test classes in GenericTests that shouldn't be there?
# XXX Is the above comment still relevant?
test_classes = [FileStorageTests, MappingStorageTests]
def test_suite():
suite = unittest.TestSuite()
for klass in test_classes:
sub = unittest.makeSuite(klass, "check")
suite.addTest(sub)
return suite
if __name__ == "__main__":
unittest.main(defaultTest="test_suite")
lib/python/ZEO/tests/testZEOOptions.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for ZEO.runzeo.ZEOOptions."""
import os
import sys
import tempfile
import unittest
import ZODB.config
from ZEO.runzeo import ZEOOptions
from zdaemon.tests.testzdoptions import TestZDOptions
# When a hostname isn't specified in an address, ZConfig supplies a
# platform-dependent default value.
DEFAULT_HOSTNAME = ''
if sys.platform in ['win32',]:
DEFAULT_HOSTNAME = 'localhost'
class TestZEOOptions(TestZDOptions):
OptionsClass = ZEOOptions
input_args = ["-f", "Data.fs", "-a", "5555"]
output_opts = [("-f", "Data.fs"), ("-a", "5555")]
output_args = []
configdata = """
<zeo>
address 5555
</zeo>
<filestorage fs>
path Data.fs
</filestorage>
"""
def setUp(self):
self.tempfilename = tempfile.mktemp()
f = open(self.tempfilename, "w")
f.write(self.configdata)
f.close()
def tearDown(self):
try:
os.remove(self.tempfilename)
except os.error:
pass
def test_configure(self):
# Hide the base class test_configure
pass
def test_defaults_with_schema(self):
options = self.OptionsClass()
options.realize(["-C", self.tempfilename])
self.assertEqual(options.address, (DEFAULT_HOSTNAME, 5555))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.name, "fs")
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_defaults_without_schema(self):
options = self.OptionsClass()
options.realize(["-a", "5555", "-f", "Data.fs"])
self.assertEqual(options.address, (DEFAULT_HOSTNAME, 5555))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.name, "1")
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(opener.config.path, "Data.fs")
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_commandline_overrides(self):
options = self.OptionsClass()
options.realize(["-C", self.tempfilename,
"-a", "6666", "-f", "Wisdom.fs"])
self.assertEqual(options.address, (DEFAULT_HOSTNAME, 6666))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(opener.config.path, "Wisdom.fs")
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_suite():
suite = unittest.TestSuite()
for cls in [TestZEOOptions]:
suite.addTest(unittest.makeSuite(cls))
return suite
if __name__ == "__main__":
unittest.main(defaultTest='test_suite')
lib/python/ZEO/tests/test_cache.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Basic unit tests for a multi-version client cache."""
import os
import tempfile
import unittest
import ZEO.cache
from ZODB.utils import p64
n1 = p64(1)
n2 = p64(2)
n3 = p64(3)
n4 = p64(4)
n5 = p64(5)
class CacheTests(unittest.TestCase):
def setUp(self):
self.cache = ZEO.cache.ClientCache()
self.cache.open()
def tearDown(self):
if self.cache.path:
os.remove(self.cache.path)
def testLastTid(self):
self.assertEqual(self.cache.getLastTid(), None)
self.cache.setLastTid(n2)
self.assertEqual(self.cache.getLastTid(), n2)
self.cache.invalidate(None, "", n1)
self.assertEqual(self.cache.getLastTid(), n2)
self.cache.invalidate(None, "", n3)
self.assertEqual(self.cache.getLastTid(), n3)
self.assertRaises(ValueError, self.cache.setLastTid, n2)
def testLoad(self):
data1 = "data for n1"
self.assertEqual(self.cache.load(n1, ""), None)
self.assertEqual(self.cache.load(n1, "version"), None)
self.cache.store(n1, "", n3, None, data1)
self.assertEqual(self.cache.load(n1, ""), (data1, n3, ""))
# The cache doesn't know whether version exists, because it
# only has non-version data.
self.assertEqual(self.cache.load(n1, "version"), None)
self.assertEqual(self.cache.modifiedInVersion(n1), None)
def testInvalidate(self):
data1 = "data for n1"
self.cache.store(n1, "", n3, None, data1)
self.cache.invalidate(n1, "", n4)
self.cache.invalidate(n2, "", n2)
self.assertEqual(self.cache.load(n1, ""), None)
self.assertEqual(self.cache.loadBefore(n1, n4),
(data1, n3, n4))
def testVersion(self):
data1 = "data for n1"
data1v = "data for n1 in version"
self.cache.store(n1, "version", n3, None, data1v)
self.assertEqual(self.cache.load(n1, ""), None)
self.assertEqual(self.cache.load(n1, "version"),
(data1v, n3, "version"))
self.assertEqual(self.cache.load(n1, "random"), None)
self.assertEqual(self.cache.modifiedInVersion(n1), "version")
self.cache.invalidate(n1, "version", n4)
self.assertEqual(self.cache.load(n1, "version"), None)
def testNonCurrent(self):
data1 = "data for n1"
data2 = "data for n2"
self.cache.store(n1, "", n4, None, data1)
self.cache.store(n1, "", n2, n3, data2)
# can't say anything about state before n2
self.assertEqual(self.cache.loadBefore(n1, n2), None)
# n3 is the upper bound of non-current record n2
self.assertEqual(self.cache.loadBefore(n1, n3), (data2, n2, n3))
# no data for between n2 and n3
self.assertEqual(self.cache.loadBefore(n1, n4), None)
self.cache.invalidate(n1, "", n5)
self.assertEqual(self.cache.loadBefore(n1, n5), (data1, n4, n5))
self.assertEqual(self.cache.loadBefore(n2, n4), None)
def testException(self):
self.assertRaises(ValueError,
self.cache.store,
n1, "version", n2, n3, "data")
self.cache.store(n1, "", n2, None, "data")
self.assertRaises(ValueError,
self.cache.store,
n1, "", n3, None, "data")
def testEviction(self):
# Manually override the current maxsize
maxsize = self.cache.size = self.cache.fc.maxsize = 3395 # 1245
self.cache.fc = ZEO.cache.FileCache(3395, None, self.cache)
# Trivial test of eviction code. Doesn't test non-current
# eviction.
data = ["z" * i for i in range(100)]
for i in range(50):
n = p64(i)
self.cache.store(n, "", n, None, data[i])
self.assertEquals(len(self.cache), i + 1)
self.assert_(self.cache.fc.currentsize < maxsize)
# The cache now uses 1225 bytes. The next insert
# should delete some objects.
n = p64(50)
self.cache.store(n, "", n, None, data[51])
self.assert_(len(self.cache) < 51)
self.assert_(self.cache.fc.currentsize <= maxsize)
# XXX Need to make sure eviction of non-current data
# and of version data are handled correctly.
def testSerialization(self):
self.cache.store(n1, "", n2, None, "data for n1")
self.cache.store(n2, "version", n2, None, "version data for n2")
self.cache.store(n3, "", n3, n4, "non-current data for n3")
self.cache.store(n3, "", n4, n5, "more non-current data for n3")
path = tempfile.mktemp()
# Copy data from self.cache into path, reaching into the cache
# guts to make the copy.
dst = open(path, "wb+")
src = self.cache.fc.f
src.seek(0)
dst.write(src.read(self.cache.fc.maxsize))
dst.close()
copy = ZEO.cache.ClientCache(path)
copy.open()
# Verify that internals of both objects are the same.
# Could also test that external API produces the same results.
eq = self.assertEqual
eq(copy.tid, self.cache.tid)
eq(len(copy), len(self.cache))
eq(copy.version, self.cache.version)
eq(copy.current, self.cache.current)
eq(copy.noncurrent, self.cache.noncurrent)
def test_suite():
return unittest.makeSuite(CacheTests)
lib/python/ZEO/tests/zeoserver.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Helper file used to launch a ZEO server cross platform"""
import os
import sys
import time
import errno
import getopt
import socket
import signal
import asyncore
import threading
import logging
import ThreadedAsync.LoopCallback
from ZEO.StorageServer import StorageServer
from ZEO.runzeo import ZEOOptions
def cleanup(storage):
# FileStorage and the Berkeley storages have this method, which deletes
# all files and directories used by the storage. This prevents @-files
# from clogging up /tmp
try:
storage.cleanup()
except AttributeError:
pass
logger = logging.getLogger('ZEO.tests.zeoserver')
def log(label, msg, *args):
message = "(%s) %s" % (label, msg)
logger.debug(message, args)
class ZEOTestServer(asyncore.dispatcher):
"""A server for killing the whole process at the end of a test.
The first time we connect to this server, we write an ack character down
the socket. The other end should block on a recv() of the socket so it
can guarantee the server has started up before continuing on.
The second connect to the port immediately exits the process, via
os._exit(), without writing data on the socket. It does close and clean
up the storage first. The other end will get the empty string from its
recv() which will be enough to tell it that the server has exited.
I think this should prevent us from ever getting a legitimate addr-in-use
error.
"""
__super_init = asyncore.dispatcher.__init__
def __init__(self, addr, server, keep):
self.__super_init()
self._server = server
self._sockets = [self]
self._keep = keep
# Count down to zero, the number of connects
self._count = 1
self._label ='%d @ %s' % (os.getpid(), addr)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
# Some ZEO tests attempt a quick start of the server using the same
# port so we have to set the reuse flag.
self.set_reuse_addr()
try:
self.bind(addr)
except:
# We really want to see these exceptions
import traceback
traceback.print_exc()
raise
self.listen(5)
self.log('bound and listening')
def log(self, msg, *args):
log(self._label, msg, *args)
def handle_accept(self):
sock, addr = self.accept()
self.log('in handle_accept()')
# When we're done with everything, close the storage. Do not write
# the ack character until the storage is finished closing.
if self._count <= 0:
self.log('closing the storage')
self._server.close_server()
if not self._keep:
for storage in self._server.storages.values():
cleanup(storage)
self.log('exiting')
# Close all the other sockets so that we don't have to wait
# for os._exit() to get to it before starting the next
# server process.
for s in self._sockets:
s.close()
# Now explicitly close the socket returned from accept(),
# since it didn't go through the wrapper.
sock.close()
os._exit(0)
self.log('continuing')
sock.send('X')
self._count -= 1
def register_socket(self, sock):
# Register a socket to be closed when server shutsdown.
self._sockets.append(sock)
class Suicide(threading.Thread):
def __init__(self, addr):
threading.Thread.__init__(self)
self._adminaddr = addr
def run(self):
# If this process doesn't exit in 330 seconds, commit suicide.
# The client threads in the ConcurrentUpdate tests will run for
# as long as 300 seconds. Set this timeout to 330 to minimize
# chance that the server gives up before the clients.
time.sleep(330)
log(str(os.getpid()), "suicide thread invoking shutdown")
# If the server hasn't shut down yet, the client may not be
# able to connect to it. If so, try to kill the process to
# force it to shutdown.
if hasattr(os, "kill"):
os.kill(pid, signal.SIGTERM)
time.sleep(5)
os.kill(pid, signal.SIGKILL)
else:
from ZEO.tests.forker import shutdown_zeo_server
# XXX If the -k option was given to zeoserver, then the
# process will go away but the temp files won't get
# cleaned up.
shutdown_zeo_server(self._adminaddr)
def main():
global pid
pid = os.getpid()
label = str(pid)
log(label, "starting")
# We don't do much sanity checking of the arguments, since if we get it
# wrong, it's a bug in the test suite.
keep = 0
configfile = None
# Parse the arguments and let getopt.error percolate
opts, args = getopt.getopt(sys.argv[1:], 'kC:')
for opt, arg in opts:
if opt == '-k':
keep = 1
elif opt == '-C':
configfile = arg
zo = ZEOOptions()
zo.realize(["-C", configfile])
zeo_port = int(zo.address[1])
# XXX a hack
if zo.auth_protocol == "plaintext":
import ZEO.tests.auth_plaintext
# Open the config file and let ZConfig parse the data there. Then remove
# the config file, otherwise we'll leave turds.
# The rest of the args are hostname, portnum
test_port = zeo_port + 1
test_addr = ('localhost', test_port)
addr = ('localhost', zeo_port)
log(label, 'creating the storage server')
storage = zo.storages[0].open()
mon_addr = None
if zo.monitor_address:
mon_addr = zo.monitor_address
server = StorageServer(
zo.address,
{"1": storage},
read_only=zo.read_only,
invalidation_queue_size=zo.invalidation_queue_size,
transaction_timeout=zo.transaction_timeout,
monitor_address=mon_addr,
auth_protocol=zo.auth_protocol,
auth_database=zo.auth_database,
auth_realm=zo.auth_realm)
try:
log(label, 'creating the test server, keep: %s', keep)
t = ZEOTestServer(test_addr, server, keep)
except socket.error, e:
if e[0] <> errno.EADDRINUSE: raise
log(label, 'addr in use, closing and exiting')
storage.close()
cleanup(storage)
sys.exit(2)
t.register_socket(server.dispatcher)
# Create daemon suicide thread
d = Suicide(test_addr)
d.setDaemon(1)
d.start()
# Loop for socket events
log(label, 'entering ThreadedAsync loop')
ThreadedAsync.LoopCallback.loop()
if __name__ == '__main__':
main()
lib/python/ZEO/util.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Utilities for setting up the server environment."""
import os
def parentdir(p, n=1):
"""Return the ancestor of p from n levels up."""
d = p
while n:
d = os.path.dirname(d)
if not d or d == '.':
d = os.getcwd()
n -= 1
return d
class Environment:
"""Determine location of the Data.fs & ZEO_SERVER.pid files.
Pass the argv[0] used to start ZEO to the constructor.
Use the zeo_pid and fs attributes to get the filenames.
"""
def __init__(self, argv0):
v = os.environ.get("INSTANCE_HOME")
if v is None:
# looking for a Zope/var directory assuming that this code
# is installed in Zope/lib/python/ZEO
p = parentdir(argv0, 4)
if os.path.isdir(os.path.join(p, "var")):
v = p
else:
v = os.getcwd()
self.home = v
self.var = os.path.join(v, "var")
if not os.path.isdir(self.var):
self.var = self.home
pid = os.environ.get("ZEO_SERVER_PID")
if pid is None:
pid = os.path.join(self.var, "ZEO_SERVER.pid")
self.zeo_pid = pid
self.fs = os.path.join(self.var, "Data.fs")
lib/python/ZEO/zeoctl.py 0 → 100644
View file @ 96172040
"""Wrapper script for zdctl.py that causes it to use the ZEO schema."""
import os
import ZEO
import zdaemon.zdctl
# Main program
def main(args=None):
options = zdaemon.zdctl.ZDCtlOptions()
options.schemadir = os.path.dirname(ZEO.__file__)
options.schemafile = "zeoctl.xml"
zdaemon.zdctl.main(args, options)
if __name__ == "__main__":
main()
lib/python/ZEO/zeoctl.xml 0 → 100644
View file @ 96172040
<schema>
<description>
This schema describes the configuration of the ZEO storage server
controller. It differs from the schema for the storage server
only in that the "runner" section is required.
</description>
<!-- Use the storage types defined by ZODB. -->
<import package="ZODB"/>
<!-- Use the ZEO server information structure. -->
<import package="ZEO"/>
<import package="ZConfig.components.logger"/>
<!-- runner control -->
<import package="zdaemon"/>
<section type="zeo" name="*" required="yes" attribute="zeo" />
<section type="runner" name="*" required="yes" attribute="runner" />
<multisection name="+" type="ZODB.storage"
attribute="storages"
required="yes" />
<section name="*" type="eventlog" attribute="eventlog" required="no" />
</schema>
lib/python/ZEO/zeopasswd.py 0 → 100644
View file @ 96172040
#!python
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Update a user's authentication tokens for a ZEO server.
usage: python zeopasswd.py [options] username [password]
Specify either a configuration file:
-C/--configuration -- ZConfig configuration file
or the individual options:
-f/--filename -- authentication database filename
-p/--protocol -- authentication protocol name
-r/--realm -- authentication database realm
Additional options:
-d/--delete -- delete user instead of updating password
"""
import getopt
import getpass
import sys
import os
import ZConfig
import ZEO
def usage(msg):
print __doc__
print msg
sys.exit(2)
def options(args):
"""Password-specific options loaded from regular ZEO config file."""
try:
opts, args = getopt.getopt(args, "dr:p:f:C:", ["configure=",
"protocol=",
"filename=",
"realm"])
except getopt.error, msg:
usage(msg)
config = None
delete = 0
auth_protocol = None
auth_db = ""
auth_realm = None
for k, v in opts:
if k == '-C' or k == '--configure':
schemafile = os.path.join(os.path.dirname(ZEO.__file__),
"schema.xml")
schema = ZConfig.loadSchema(schemafile)
config, nil = ZConfig.loadConfig(schema, v)
if k == '-d' or k == '--delete':
delete = 1
if k == '-p' or k == '--protocol':
auth_protocol = v
if k == '-f' or k == '--filename':
auth_db = v
if k == '-r' or k == '--realm':
auth_realm = v
if config is not None:
if auth_protocol or auth_db:
usage("Error: Conflicting options; use either -C *or* -p and -f")
auth_protocol = config.zeo.authentication_protocol
auth_db = config.zeo.authentication_database
auth_realm = config.zeo.authentication_realm
elif not (auth_protocol and auth_db):
usage("Error: Must specifiy configuration file or protocol and database")
password = None
if delete:
if not args:
usage("Error: Must specify a username to delete")
elif len(args) > 1:
usage("Error: Too many arguments")
username = args[0]
else:
if not args:
usage("Error: Must specify a username")
elif len(args) > 2:
usage("Error: Too many arguments")
elif len(args) == 1:
username = args[0]
else:
username, password = args
return auth_protocol, auth_db, auth_realm, delete, username, password
def main(args=None, dbclass=None):
p, auth_db, auth_realm, delete, username, password = options(args)
if p is None:
usage("Error: configuration does not specify auth protocol")
if p == "digest":
from ZEO.auth.auth_digest import DigestDatabase as Database
elif p == "srp":
from ZEO.auth.auth_srp import SRPDatabase as Database
elif dbclass:
# dbclass is used for testing tests.auth_plaintext, see testAuth.py
Database = dbclass
else:
raise ValueError, "Unknown database type %r" % p
if auth_db is None:
usage("Error: configuration does not specify auth database")
db = Database(auth_db, auth_realm)
if delete:
db.del_user(username)
else:
if password is None:
password = getpass.getpass("Enter password: ")
db.add_user(username, password)
db.save()
if __name__ == "__main__":
main(sys.argv[1:])
lib/python/ZEO/zrpc/__init__.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# zrpc is a package with the following modules
# client -- manages connection creation to remote server
# connection -- object dispatcher
# log -- logging helper
# error -- exceptions raised by zrpc
# marshal -- internal, handles basic protocol issues
# server -- manages incoming connections from remote clients
# smac -- sized message async connections
# trigger -- medusa's trigger
# zrpc is not an advertised subpackage of ZEO; its interfaces are internal
lib/python/ZEO/zrpc/_hmac.py 0 → 100644
View file @ 96172040
# This file is a slightly modified copy of Python 2.3's Lib/hmac.py.
# This file is under the Python Software Foundation (PSF) license.
"""HMAC (Keyed-Hashing for Message Authentication) Python module.
Implements the HMAC algorithm as described by RFC 2104.
"""
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
"""
return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
# The size of the digests returned by HMAC depends on the underlying
# hashing module used.
digest_size = None
class HMAC:
"""RFC2104 HMAC class.
This supports the API for Cryptographic Hash Functions (PEP 247).
"""
def __init__(self, key, msg = None, digestmod = None):
"""Create a new HMAC object.
key: key for the keyed hash object.
msg: Initial input for the hash, if provided.
digestmod: A module supporting PEP 247. Defaults to the md5 module.
"""
if digestmod is None:
import md5
digestmod = md5
self.digestmod = digestmod
self.outer = digestmod.new()
self.inner = digestmod.new()
# Python 2.1 and 2.2 differ about the correct spelling
try:
self.digest_size = digestmod.digestsize
except AttributeError:
self.digest_size = digestmod.digest_size
blocksize = 64
ipad = "\x36" * blocksize
opad = "\x5C" * blocksize
if len(key) > blocksize:
key = digestmod.new(key).digest()
key = key + chr(0) * (blocksize - len(key))
self.outer.update(_strxor(key, opad))
self.inner.update(_strxor(key, ipad))
if msg is not None:
self.update(msg)
## def clear(self):
## raise NotImplementedError, "clear() method not available in HMAC."
def update(self, msg):
"""Update this hashing object with the string msg.
"""
self.inner.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
An update to this copy won't affect the original object.
"""
other = HMAC("")
other.digestmod = self.digestmod
other.inner = self.inner.copy()
other.outer = self.outer.copy()
return other
def digest(self):
"""Return the hash value of this hashing object.
This returns a string containing 8-bit data. The object is
not altered in any way by this function; you can continue
updating the object after calling this function.
"""
h = self.outer.copy()
h.update(self.inner.digest())
return h.digest()
def hexdigest(self):
"""Like digest(), but returns a string of hexadecimal digits instead.
"""
return "".join([hex(ord(x))[2:].zfill(2)
for x in tuple(self.digest())])
def new(key, msg = None, digestmod = None):
"""Create a new hashing object and return it.
key: The starting key for the hash.
msg: if available, will immediately be hashed into the object's starting
state.
You can now feed arbitrary strings into the object using its update()
method, and can ask for the hash value at any time by calling its digest()
method.
"""
return HMAC(key, msg, digestmod)
lib/python/ZEO/zrpc/client.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import errno
import select
import socket
import sys
import threading
import time
import types
import logging
import ThreadedAsync
from ZODB.POSException import ReadOnlyError
from ZODB.loglevels import BLATHER
from ZEO.zrpc.log import log
from ZEO.zrpc.trigger import trigger
from ZEO.zrpc.connection import ManagedConnection
class ConnectionManager(object):
"""Keeps a connection up over time"""
def __init__(self, addrs, client, tmin=1, tmax=180):
self.addrlist = self._parse_addrs(addrs)
self.client = client
self.tmin = tmin
self.tmax = tmax
self.cond = threading.Condition(threading.Lock())
self.connection = None # Protected by self.cond
self.closed = 0
# If thread is not None, then there is a helper thread
# attempting to connect.
self.thread = None # Protected by self.cond
self.trigger = None
self.thr_async = 0
ThreadedAsync.register_loop_callback(self.set_async)
def __repr__(self):
return "<%s for %s>" % (self.__class__.__name__, self.addrlist)
def _parse_addrs(self, addrs):
# Return a list of (addr_type, addr) pairs.
# For backwards compatibility (and simplicity?) the
# constructor accepts a single address in the addrs argument --
# a string for a Unix domain socket or a 2-tuple with a
# hostname and port. It can also accept a list of such addresses.
addr_type = self._guess_type(addrs)
if addr_type is not None:
return [(addr_type, addrs)]
else:
addrlist = []
for addr in addrs:
addr_type = self._guess_type(addr)
if addr_type is None:
raise ValueError, (
"unknown address in list: %s" % repr(addr))
addrlist.append((addr_type, addr))
return addrlist
def _guess_type(self, addr):
if isinstance(addr, types.StringType):
return socket.AF_UNIX
if (len(addr) == 2
and isinstance(addr[0], types.StringType)
and isinstance(addr[1], types.IntType)):
return socket.AF_INET
# not anything I know about
return None
def close(self):
"""Prevent ConnectionManager from opening new connections"""
self.closed = 1
ThreadedAsync.remove_loop_callback(self.set_async)
self.cond.acquire()
try:
t = self.thread
self.thread = None
conn = self.connection
finally:
self.cond.release()
if t is not None:
log("CM.close(): stopping and joining thread")
t.stop()
t.join(30)
if t.isAlive():
log("CM.close(): self.thread.join() timed out",
level=logging.WARNING)
if conn is not None:
# This will call close_conn() below which clears self.connection
conn.close()
if self.trigger is not None:
self.trigger.close()
self.trigger = None
ThreadedAsync.remove_loop_callback(self.set_async)
def set_async(self, map):
# This is the callback registered with ThreadedAsync. The
# callback might be called multiple times, so it shouldn't
# create a trigger every time and should never do anything
# after it's closed.
# It may be that the only case where it is called multiple
# times is in the test suite, where ThreadedAsync's loop can
# be started in a child process after a fork. Regardless,
# it's good to be defensive.
# XXX need each connection started with async==0 to have a
# callback
log("CM.set_async(%s)" % repr(map), level=logging.DEBUG)
if not self.closed and self.trigger is None:
log("CM.set_async(): first call")
self.trigger = trigger()
self.thr_async = 1 # XXX needs to be set on the Connection
def attempt_connect(self):
"""Attempt a connection to the server without blocking too long.
There isn't a crisp definition for too long. When a
ClientStorage is created, it attempts to connect to the
server. If the server isn't immediately available, it can
operate from the cache. This method will start the background
connection thread and wait a little while to see if it
finishes quickly.
"""
# XXX Will a single attempt take too long?
# XXX Answer: it depends -- normally, you'll connect or get a
# connection refused error very quickly. Packet-eating
# firewalls and other mishaps may cause the connect to take a
# long time to time out though. It's also possible that you
# connect quickly to a slow server, and the attempt includes
# at least one roundtrip to the server (the register() call).
# But that's as fast as you can expect it to be.
self.connect()
self.cond.acquire()
try:
t = self.thread
conn = self.connection
finally:
self.cond.release()
if t is not None and conn is None:
event = t.one_attempt
event.wait()
self.cond.acquire()
try:
conn = self.connection
finally:
self.cond.release()
return conn is not None
def connect(self, sync=0):
self.cond.acquire()
try:
if self.connection is not None:
return
t = self.thread
if t is None:
log("CM.connect(): starting ConnectThread")
self.thread = t = ConnectThread(self, self.client,
self.addrlist,
self.tmin, self.tmax)
t.setDaemon(1)
t.start()
if sync:
while self.connection is None:
self.cond.wait(30)
if self.connection is None:
log("CM.connect(sync=1): still waiting...")
finally:
self.cond.release()
if sync:
assert self.connection is not None
def connect_done(self, conn, preferred):
# Called by ConnectWrapper.notify_client() after notifying the client
log("CM.connect_done(preferred=%s)" % preferred)
self.cond.acquire()
try:
self.connection = conn
if preferred:
self.thread = None
self.cond.notifyAll() # Wake up connect(sync=1)
finally:
self.cond.release()
def close_conn(self, conn):
# Called by the connection when it is closed
self.cond.acquire()
try:
if conn is not self.connection:
# Closing a non-current connection
log("CM.close_conn() non-current", level=BLATHER)
return
log("CM.close_conn()")
self.connection = None
finally:
self.cond.release()
self.client.notifyDisconnected()
if not self.closed:
self.connect()
def is_connected(self):
self.cond.acquire()
try:
return self.connection is not None
finally:
self.cond.release()
# When trying to do a connect on a non-blocking socket, some outcomes
# are expected. Set _CONNECT_IN_PROGRESS to the errno value(s) expected
# when an initial connect can't complete immediately. Set _CONNECT_OK
# to the errno value(s) expected if the connect succeeds *or* if it's
# already connected (our code can attempt redundant connects).
if hasattr(errno, "WSAEWOULDBLOCK"): # Windows
# XXX The official Winsock docs claim that WSAEALREADY should be
# treated as yet another "in progress" indicator, but we've never
# seen this.
_CONNECT_IN_PROGRESS = (errno.WSAEWOULDBLOCK,)
# Win98: WSAEISCONN; Win2K: WSAEINVAL
_CONNECT_OK = (0, errno.WSAEISCONN, errno.WSAEINVAL)
else: # Unix
_CONNECT_IN_PROGRESS = (errno.EINPROGRESS,)
_CONNECT_OK = (0, errno.EISCONN)
class ConnectThread(threading.Thread):
"""Thread that tries to connect to server given one or more addresses.
The thread is passed a ConnectionManager and the manager's client
as arguments. It calls testConnection() on the client when a
socket connects; that should return 1 or 0 indicating whether this
is a preferred or a fallback connection. It may also raise an
exception, in which case the connection is abandoned.
The thread will continue to run, attempting connections, until a
preferred connection is seen and successfully handed over to the
manager and client.
As soon as testConnection() finds a preferred connection, or after
all sockets have been tried and at least one fallback connection
has been seen, notifyConnected(connection) is called on the client
and connect_done() on the manager. If this was a preferred
connection, the thread then exits; otherwise, it keeps trying
until it gets a preferred connection, and then reconnects the
client using that connection.
"""
__super_init = threading.Thread.__init__
# We don't expect clients to call any methods of this Thread other
# than close() and those defined by the Thread API.
def __init__(self, mgr, client, addrlist, tmin, tmax):
self.__super_init(name="Connect(%s)" % addrlist)
self.mgr = mgr
self.client = client
self.addrlist = addrlist
self.tmin = tmin
self.tmax = tmax
self.stopped = 0
self.one_attempt = threading.Event()
# A ConnectThread keeps track of whether it has finished a
# call to try_connecting(). This allows the ConnectionManager
# to make an attempt to connect right away, but not block for
# too long if the server isn't immediately available.
def stop(self):
self.stopped = 1
def run(self):
delay = self.tmin
success = 0
# Don't wait too long the first time.
# XXX make timeout configurable?
attempt_timeout = 5
while not self.stopped:
success = self.try_connecting(attempt_timeout)
if not self.one_attempt.isSet():
self.one_attempt.set()
attempt_timeout = 75
if success > 0:
break
time.sleep(delay)
if self.mgr.is_connected():
log("CT: still trying to replace fallback connection",
level=logging.INFO)
delay = min(delay*2, self.tmax)
log("CT: exiting thread: %s" % self.getName())
def try_connecting(self, timeout):
"""Try connecting to all self.addrlist addresses.
Return 1 if a preferred connection was found; 0 if no
connection was found; and -1 if a fallback connection was
found.
If no connection is found within timeout seconds, return 0.
"""
log("CT: attempting to connect on %d sockets" % len(self.addrlist))
deadline = time.time() + timeout
wrappers = self._create_wrappers()
for wrap in wrappers.keys():
if wrap.state == "notified":
return 1
try:
if time.time() > deadline:
return 0
r = self._connect_wrappers(wrappers, deadline)
if r is not None:
return r
if time.time() > deadline:
return 0
r = self._fallback_wrappers(wrappers, deadline)
if r is not None:
return r
# Alas, no luck.
assert not wrappers
finally:
for wrap in wrappers.keys():
wrap.close()
del wrappers
return 0
def _create_wrappers(self):
# Create socket wrappers
wrappers = {} # keys are active wrappers
for domain, addr in self.addrlist:
wrap = ConnectWrapper(domain, addr, self.mgr, self.client)
wrap.connect_procedure()
if wrap.state == "notified":
for w in wrappers.keys():
w.close()
return {wrap: wrap}
if wrap.state != "closed":
wrappers[wrap] = wrap
return wrappers
def _connect_wrappers(self, wrappers, deadline):
# Next wait until they all actually connect (or fail)
# The deadline is necessary, because we'd wait forever if a
# sockets never connects or fails.
while wrappers:
if self.stopped:
for wrap in wrappers.keys():
wrap.close()
return 0
# Select connecting wrappers
connecting = [wrap
for wrap in wrappers.keys()
if wrap.state == "connecting"]
if not connecting:
break
if time.time() > deadline:
break
try:
r, w, x = select.select([], connecting, connecting, 1.0)
log("CT: select() %d, %d, %d" % tuple(map(len, (r,w,x))))
except select.error, msg:
log("CT: select failed; msg=%s" % str(msg),
level=logging.WARNING) # XXX Is this the right level?
continue
# Exceptable wrappers are in trouble; close these suckers
for wrap in x:
log("CT: closing troubled socket %s" % str(wrap.addr))
del wrappers[wrap]
wrap.close()
# Writable sockets are connected
for wrap in w:
wrap.connect_procedure()
if wrap.state == "notified":
del wrappers[wrap] # Don't close this one
for wrap in wrappers.keys():
wrap.close()
return 1
if wrap.state == "closed":
del wrappers[wrap]
def _fallback_wrappers(self, wrappers, deadline):
# If we've got wrappers left at this point, they're fallback
# connections. Try notifying them until one succeeds.
for wrap in wrappers.keys():
assert wrap.state == "tested" and wrap.preferred == 0
if self.mgr.is_connected():
wrap.close()
else:
wrap.notify_client()
if wrap.state == "notified":
del wrappers[wrap] # Don't close this one
for wrap in wrappers.keys():
wrap.close()
return -1
assert wrap.state == "closed"
del wrappers[wrap]
# XXX should check deadline
class ConnectWrapper:
"""An object that handles the connection procedure for one socket.
This is a little state machine with states:
closed
opened
connecting
connected
tested
notified
"""
def __init__(self, domain, addr, mgr, client):
"""Store arguments and create non-blocking socket."""
self.domain = domain
self.addr = addr
self.mgr = mgr
self.client = client
# These attributes are part of the interface
self.state = "closed"
self.sock = None
self.conn = None
self.preferred = 0
log("CW: attempt to connect to %s" % repr(addr))
try:
self.sock = socket.socket(domain, socket.SOCK_STREAM)
except socket.error, err:
log("CW: can't create socket, domain=%s: %s" % (domain, err),
level=logging.ERROR)
self.close()
return
self.sock.setblocking(0)
self.state = "opened"
def connect_procedure(self):
"""Call sock.connect_ex(addr) and interpret result."""
if self.state in ("opened", "connecting"):
try:
err = self.sock.connect_ex(self.addr)
except socket.error, msg:
log("CW: connect_ex(%r) failed: %s" % (self.addr, msg),
level=logging.ERROR)
self.close()
return
log("CW: connect_ex(%s) returned %s" %
(self.addr, errno.errorcode.get(err) or str(err)))
if err in _CONNECT_IN_PROGRESS:
self.state = "connecting"
return
if err not in _CONNECT_OK:
log("CW: error connecting to %s: %s" %
(self.addr, errno.errorcode.get(err) or str(err)),
level=logging.WARNING)
self.close()
return
self.state = "connected"
if self.state == "connected":
self.test_connection()
def test_connection(self):
"""Establish and test a connection at the zrpc level.
Call the client's testConnection(), giving the client a chance
to do app-level check of the connection.
"""
self.conn = ManagedConnection(self.sock, self.addr,
self.client, self.mgr)
self.sock = None # The socket is now owned by the connection
try:
self.preferred = self.client.testConnection(self.conn)
self.state = "tested"
except ReadOnlyError:
log("CW: ReadOnlyError in testConnection (%s)" % repr(self.addr))
self.close()
return
except:
log("CW: error in testConnection (%s)" % repr(self.addr),
level=logging.ERROR, exc_info=True)
self.close()
return
if self.preferred:
self.notify_client()
def notify_client(self):
"""Call the client's notifyConnected().
If this succeeds, call the manager's connect_done().
If the client is already connected, we assume it's a fallback
connection, and the new connection must be a preferred
connection. The client will close the old connection.
"""
try:
self.client.notifyConnected(self.conn)
except:
log("CW: error in notifyConnected (%s)" % repr(self.addr),
level=logging.ERROR, exc_info=True)
self.close()
return
self.state = "notified"
self.mgr.connect_done(self.conn, self.preferred)
def close(self):
"""Close the socket and reset everything."""
self.state = "closed"
self.mgr = self.client = None
self.preferred = 0
if self.conn is not None:
# Closing the ZRPC connection will eventually close the
# socket, somewhere in asyncore.
# XXX Why do we care? --Guido
self.conn.close()
self.conn = None
if self.sock is not None:
self.sock.close()
self.sock = None
def fileno(self):
return self.sock.fileno()
lib/python/ZEO/zrpc/connection.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import errno
import select
import sys
import threading
import types
import logging
import ThreadedAsync
from ZEO.zrpc import smac
from ZEO.zrpc.error import ZRPCError, DisconnectedError
from ZEO.zrpc.marshal import Marshaller
from ZEO.zrpc.trigger import trigger
from ZEO.zrpc.log import short_repr, log
from ZODB.loglevels import BLATHER, TRACE
REPLY = ".reply" # message name used for replies
ASYNC = 1
class Delay:
"""Used to delay response to client for synchronous calls.
When a synchronous call is made and the original handler returns
without handling the call, it returns a Delay object that prevents
the mainloop from sending a response.
"""
def set_sender(self, msgid, send_reply, return_error):
self.msgid = msgid
self.send_reply = send_reply
self.return_error = return_error
def reply(self, obj):
self.send_reply(self.msgid, obj)
def error(self, exc_info):
log("Error raised in delayed method", logging.ERROR, exc_info=True)
self.return_error(self.msgid, 0, *exc_info[:2])
class MTDelay(Delay):
def __init__(self):
self.ready = threading.Event()
def set_sender(self, msgid, send_reply, return_error):
Delay.set_sender(self, msgid, send_reply, return_error)
self.ready.set()
def reply(self, obj):
self.ready.wait()
Delay.reply(self, obj)
def error(self, exc_info):
self.ready.wait()
Delay.error(self, exc_info)
class Connection(smac.SizedMessageAsyncConnection, object):
"""Dispatcher for RPC on object on both sides of socket.
The connection supports synchronous calls, which expect a return,
and asynchronous calls, which do not.
It uses the Marshaller class to handle encoding and decoding of
method calls and arguments. Marshaller uses pickle to encode
arbitrary Python objects. The code here doesn't ever see the wire
format.
A Connection is designed for use in a multithreaded application,
where a synchronous call must block until a response is ready.
A socket connection between a client and a server allows either
side to invoke methods on the other side. The processes on each
end of the socket use a Connection object to manage communication.
The Connection deals with decoded RPC messages. They are
represented as four-tuples containing: msgid, flags, method name,
and a tuple of method arguments.
The msgid starts at zero and is incremented by one each time a
method call message is sent. Each side of the connection has a
separate msgid state.
When one side of the connection (the client) calls a method, it
sends a message with a new msgid. The other side (the server),
replies with a message that has the same msgid, the string
".reply" (the global variable REPLY) as the method name, and the
actual return value in the args position. Note that each side of
the Connection can initiate a call, in which case it will be the
client for that particular call.
The protocol also supports asynchronous calls. The client does
not wait for a return value for an asynchronous call. The only
defined flag is ASYNC. If a method call message has the ASYNC
flag set, the server will raise an exception.
If a method call raises an Exception, the exception is propagated
back to the client via the REPLY message. The client side will
raise any exception it receives instead of returning the value to
the caller.
"""
__super_init = smac.SizedMessageAsyncConnection.__init__
__super_close = smac.SizedMessageAsyncConnection.close
__super_setSessionKey = smac.SizedMessageAsyncConnection.setSessionKey
# Protocol variables:
#
# oldest_protocol_version -- the oldest protocol version we support
# protocol_version -- the newest protocol version we support; preferred
oldest_protocol_version = "Z200"
protocol_version = "Z201"
# Protocol history:
#
# Z200 -- Original ZEO 2.0 protocol
#
# Z201 -- Added invalidateTransaction() to client.
# Renamed several client methods.
# Added several sever methods:
# lastTransaction()
# getAuthProtocol() and scheme-specific authentication methods
# getExtensionMethods().
# getInvalidations().
def __init__(self, sock, addr, obj=None):
self.obj = None
self.marshal = Marshaller()
self.closed = False
self.msgid = 0
self.peer_protocol_version = None # Set in recv_handshake()
self.logger = logging.getLogger('ZEO.zrpc.Connection')
if isinstance(addr, types.TupleType):
self.log_label = "(%s:%d) " % addr
else:
self.log_label = "(%s) " % addr
self.__super_init(sock, addr)
# A Connection either uses asyncore directly or relies on an
# asyncore mainloop running in a separate thread. If
# thr_async is true, then the mainloop is running in a
# separate thread. If thr_async is true, then the asyncore
# trigger (self.trigger) is used to notify that thread of
# activity on the current thread.
self.thr_async = False
self.trigger = None
self._prepare_async()
# The singleton dict is used in synchronous mode when a method
# needs to call into asyncore to try to force some I/O to occur.
# The singleton dict is a socket map containing only this object.
self._singleton = {self._fileno: self}
# msgid_lock guards access to msgid
self.msgid_lock = threading.Lock()
# replies_cond is used to block when a synchronous call is
# waiting for a response
self.replies_cond = threading.Condition()
self.replies = {}
# waiting_for_reply is used internally to indicate whether
# a call is in progress. setting a session key is deferred
# until after the call returns.
self.waiting_for_reply = False
self.delay_sesskey = None
self.register_object(obj)
self.handshake()
def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.addr)
__str__ = __repr__ # Defeat asyncore's dreaded __getattr__
def log(self, message, level=BLATHER, exc_info=False):
self.logger.log(level, self.log_label + message, exc_info=exc_info)
def close(self):
if self.closed:
return
self._singleton.clear()
self.closed = True
self.close_trigger()
self.__super_close()
def close_trigger(self):
# Overridden by ManagedConnection
if self.trigger is not None:
self.trigger.close()
def register_object(self, obj):
"""Register obj as the true object to invoke methods on."""
self.obj = obj
def handshake(self, proto=None):
# Overridden by ManagedConnection
# When a connection is created the first message sent is a
# 4-byte protocol version. This mechanism should allow the
# protocol to evolve over time, and let servers handle clients
# using multiple versions of the protocol.
# The mechanism replaces the message_input() method for the
# first message received.
# The client sends the protocol version it is using.
self.message_input = self.recv_handshake
self.message_output(proto or self.protocol_version)
def recv_handshake(self, proto):
# Extended by ManagedConnection
del self.message_input
self.peer_protocol_version = proto
if self.oldest_protocol_version <= proto <= self.protocol_version:
self.log("received handshake %r" % proto, level=logging.INFO)
else:
self.log("bad handshake %s" % short_repr(proto),
level=logging.ERROR)
raise ZRPCError("bad handshake %r" % proto)
def message_input(self, message):
"""Decoding an incoming message and dispatch it"""
# If something goes wrong during decoding, the marshaller
# will raise an exception. The exception will ultimately
# result in asycnore calling handle_error(), which will
# close the connection.
msgid, flags, name, args = self.marshal.decode(message)
if __debug__:
self.log("recv msg: %s, %s, %s, %s" % (msgid, flags, name,
short_repr(args)),
level=TRACE)
if name == REPLY:
self.handle_reply(msgid, flags, args)
else:
self.handle_request(msgid, flags, name, args)
def handle_reply(self, msgid, flags, args):
if __debug__:
self.log("recv reply: %s, %s, %s"
% (msgid, flags, short_repr(args)), level=TRACE)
self.replies_cond.acquire()
try:
self.replies[msgid] = flags, args
self.replies_cond.notifyAll()
finally:
self.replies_cond.release()
def handle_request(self, msgid, flags, name, args):
if not self.check_method(name):
msg = "Invalid method name: %s on %s" % (name, repr(self.obj))
raise ZRPCError(msg)
if __debug__:
self.log("calling %s%s" % (name, short_repr(args)),
level=logging.DEBUG)
meth = getattr(self.obj, name)
try:
self.waiting_for_reply = True
try:
ret = meth(*args)
finally:
self.waiting_for_reply = False
except (SystemExit, KeyboardInterrupt):
raise
except Exception, msg:
self.log("%s() raised exception: %s" % (name, msg), logging.INFO,
exc_info=True)
error = sys.exc_info()[:2]
return self.return_error(msgid, flags, *error)
if flags & ASYNC:
if ret is not None:
raise ZRPCError("async method %s returned value %s" %
(name, short_repr(ret)))
else:
if __debug__:
self.log("%s returns %s" % (name, short_repr(ret)),
logging.DEBUG)
if isinstance(ret, Delay):
ret.set_sender(msgid, self.send_reply, self.return_error)
else:
self.send_reply(msgid, ret)
if self.delay_sesskey:
self.__super_setSessionKey(self.delay_sesskey)
self.delay_sesskey = None
def handle_error(self):
if sys.exc_info()[0] == SystemExit:
raise sys.exc_info()
self.log("Error caught in asyncore",
level=logging.ERROR, exc_info=True)
self.close()
def check_method(self, name):
# XXX Is this sufficient "security" for now?
if name.startswith('_'):
return None
return hasattr(self.obj, name)
def send_reply(self, msgid, ret):
try:
msg = self.marshal.encode(msgid, 0, REPLY, ret)
except self.marshal.errors:
try:
r = short_repr(ret)
except:
r = "<unreprable>"
err = ZRPCError("Couldn't pickle return %.100s" % r)
msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err))
self.message_output(msg)
self.poll()
def return_error(self, msgid, flags, err_type, err_value):
if flags & ASYNC:
self.log("Asynchronous call raised exception: %s" % self,
level=logging.ERROR, exc_info=True)
return
if type(err_value) is not types.InstanceType:
err_value = err_type, err_value
try:
msg = self.marshal.encode(msgid, 0, REPLY, (err_type, err_value))
except self.marshal.errors:
try:
r = short_repr(err_value)
except:
r = "<unreprable>"
err = ZRPCError("Couldn't pickle error %.100s" % r)
msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err))
self.message_output(msg)
self.poll()
def setSessionKey(self, key):
if self.waiting_for_reply:
self.delay_sesskey = key
else:
self.__super_setSessionKey(key)
# The next two public methods (call and callAsync) are used by
# clients to invoke methods on remote objects
def send_call(self, method, args, flags):
# send a message and return its msgid
self.msgid_lock.acquire()
try:
msgid = self.msgid
self.msgid = self.msgid + 1
finally:
self.msgid_lock.release()
if __debug__:
self.log("send msg: %d, %d, %s, ..." % (msgid, flags, method),
level=TRACE)
buf = self.marshal.encode(msgid, flags, method, args)
self.message_output(buf)
return msgid
def call(self, method, *args):
if self.closed:
raise DisconnectedError()
msgid = self.send_call(method, args, 0)
r_flags, r_args = self.wait(msgid)
if (isinstance(r_args, types.TupleType) and len(r_args) > 1
and type(r_args[0]) == types.ClassType
and issubclass(r_args[0], Exception)):
inst = r_args[1]
raise inst # error raised by server
else:
return r_args
# For testing purposes, it is useful to begin a synchronous call
# but not block waiting for its response. Since these methods are
# used for testing they can assume they are not in async mode and
# call asyncore.poll() directly to get the message out without
# also waiting for the reply.
def _deferred_call(self, method, *args):
if self.closed:
raise DisconnectedError()
msgid = self.send_call(method, args, 0)
asyncore.poll(0.01, self._singleton)
return msgid
def _deferred_wait(self, msgid):
r_flags, r_args = self.wait(msgid)
if (isinstance(r_args, types.TupleType)
and type(r_args[0]) == types.ClassType
and issubclass(r_args[0], Exception)):
inst = r_args[1]
raise inst # error raised by server
else:
return r_args
def callAsync(self, method, *args):
if self.closed:
raise DisconnectedError()
self.send_call(method, args, ASYNC)
self.poll()
def callAsyncNoPoll(self, method, *args):
# Like CallAsync but doesn't poll. This exists so that we can
# send invalidations atomically to all clients without
# allowing any client to sneak in a load request.
if self.closed:
raise DisconnectedError()
self.send_call(method, args, ASYNC)
# handle IO, possibly in async mode
def _prepare_async(self):
self.thr_async = False
ThreadedAsync.register_loop_callback(self.set_async)
# XXX If we are not in async mode, this will cause dead
# Connections to be leaked.
def set_async(self, map):
self.trigger = trigger()
self.thr_async = True
def is_async(self):
# Overridden by ManagedConnection
if self.thr_async:
return 1
else:
return 0
def _pull_trigger(self, tryagain=10):
try:
self.trigger.pull_trigger()
except OSError:
self.trigger.close()
self.trigger = trigger()
if tryagain > 0:
self._pull_trigger(tryagain=tryagain-1)
def wait(self, msgid):
"""Invoke asyncore mainloop and wait for reply."""
if __debug__:
self.log("wait(%d), async=%d" % (msgid, self.is_async()),
level=TRACE)
if self.is_async():
self._pull_trigger()
# Delay used when we call asyncore.poll() directly.
# Start with a 1 msec delay, double until 1 sec.
delay = 0.001
self.replies_cond.acquire()
try:
while 1:
if self.closed:
raise DisconnectedError()
reply = self.replies.get(msgid)
if reply is not None:
del self.replies[msgid]
if __debug__:
self.log("wait(%d): reply=%s" %
(msgid, short_repr(reply)), level=TRACE)
return reply
if self.is_async():
self.replies_cond.wait(10.0)
else:
self.replies_cond.release()
try:
try:
if __debug__:
self.log("wait(%d): asyncore.poll(%s)" %
(msgid, delay), level=TRACE)
asyncore.poll(delay, self._singleton)
if delay < 1.0:
delay += delay
except select.error, err:
self.log("Closing. asyncore.poll() raised %s."
% err, level=BLATHER)
self.close()
finally:
self.replies_cond.acquire()
finally:
self.replies_cond.release()
def flush(self):
"""Invoke poll() until the output buffer is empty."""
if __debug__:
self.log("flush")
while self.writable():
self.poll()
def poll(self):
"""Invoke asyncore mainloop to get pending message out."""
if __debug__:
self.log("poll(), async=%d" % self.is_async(), level=TRACE)
if self.is_async():
self._pull_trigger()
else:
asyncore.poll(0.0, self._singleton)
def pending(self, timeout=0):
"""Invoke mainloop until any pending messages are handled."""
if __debug__:
self.log("pending(), async=%d" % self.is_async(), level=TRACE)
if self.is_async():
return
# Inline the asyncore poll() function to know whether any input
# was actually read. Repeat until no input is ready.
# Pending does reads and writes. In the case of server
# startup, we may need to write out zeoVerify() messages.
# Always check for read status, but don't check for write status
# only there is output to do. Only continue in this loop as
# long as there is data to read.
r = r_in = [self._fileno]
x_in = []
while r and not self.closed:
if self.writable():
w_in = [self._fileno]
else:
w_in = []
try:
r, w, x = select.select(r_in, w_in, x_in, timeout)
except select.error, err:
if err[0] == errno.EINTR:
timeout = 0
continue
else:
raise
else:
# Make sure any subsequent select does not block. The
# loop is only intended to make sure all incoming data is
# returned.
# XXX What if the server sends a lot of invalidations,
# such that pending never finishes? Seems unlikely, but
# not impossible.
timeout = 0
if r:
try:
self.handle_read_event()
except asyncore.ExitNow:
raise
except:
self.handle_error()
if w:
try:
self.handle_write_event()
except asyncore.ExitNow:
raise
except:
self.handle_error()
class ManagedServerConnection(Connection):
"""Server-side Connection subclass."""
__super_init = Connection.__init__
__super_close = Connection.close
def __init__(self, sock, addr, obj, mgr):
self.mgr = mgr
self.__super_init(sock, addr, obj)
self.obj.notifyConnected(self)
def close(self):
self.obj.notifyDisconnected()
self.mgr.close_conn(self)
self.__super_close()
class ManagedConnection(Connection):
"""Client-side Connection subclass."""
__super_init = Connection.__init__
__super_close = Connection.close
def __init__(self, sock, addr, obj, mgr):
self.mgr = mgr
self.__super_init(sock, addr, obj)
self.check_mgr_async()
# PROTOCOL NEGOTIATION:
#
# The code implementing protocol version 2.0.0 (which is deployed
# in the field and cannot be changed) *only* talks to peers that
# send a handshake indicating protocol version 2.0.0. In that
# version, both the client and the server immediately send out
# their protocol handshake when a connection is established,
# without waiting for their peer, and disconnect when a different
# handshake is receive.
#
# The new protocol uses this to enable new clients to talk to
# 2.0.0 servers: in the new protocol, the client waits until it
# receives the server's protocol handshake before sending its own
# handshake. The client sends the lower of its own protocol
# version and the server protocol version, allowing it to talk to
# servers using later protocol versions (2.0.2 and higher) as
# well: the effective protocol used will be the lower of the
# client and server protocol.
#
# The ZEO modules ClientStorage and ServerStub have backwards
# compatibility code for dealing with the previous version of the
# protocol. The client accept the old version of some messages,
# and will not send new messages when talking to an old server.
#
# As long as the client hasn't sent its handshake, it can't send
# anything else; output messages are queued during this time.
# (Output can happen because the connection testing machinery can
# start sending requests before the handshake is received.)
#
# UPGRADING FROM ZEO 2.0.0 TO NEWER VERSIONS:
#
# Because a new client can talk to an old server, but not vice
# versa, all clients should be upgraded before upgrading any
# servers. Protocol upgrades beyond 2.0.1 will not have this
# restriction, because clients using protocol 2.0.1 or later can
# talk to both older and newer servers.
#
# No compatibility with protocol version 1 is provided.
def handshake(self):
self.message_input = self.recv_handshake
self.message_output = self.queue_output
self.output_queue = []
# The handshake is sent by recv_handshake() below
def queue_output(self, message):
self.output_queue.append(message)
def recv_handshake(self, proto):
del self.message_output
proto = min(proto, self.protocol_version)
Connection.recv_handshake(self, proto) # Raise error if wrong proto
self.message_output(proto)
queue = self.output_queue
del self.output_queue
for message in queue:
self.message_output(message)
# Defer the ThreadedAsync work to the manager.
def close_trigger(self):
# the manager should actually close the trigger
del self.trigger
def set_async(self, map):
pass
def _prepare_async(self):
# Don't do the register_loop_callback that the superclass does
pass
def check_mgr_async(self):
if not self.thr_async and self.mgr.thr_async:
assert self.mgr.trigger is not None, \
"manager (%s) has no trigger" % self.mgr
self.thr_async = True
self.trigger = self.mgr.trigger
return 1
return 0
def is_async(self):
# XXX could the check_mgr_async() be avoided on each test?
if self.thr_async:
return 1
return self.check_mgr_async()
def close(self):
self.mgr.close_conn(self)
self.__super_close()
lib/python/ZEO/zrpc/error.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from ZODB import POSException
from ZEO.Exceptions import ClientDisconnected
class ZRPCError(POSException.StorageError):
pass
class DisconnectedError(ZRPCError, ClientDisconnected):
"""The database storage is disconnected from the storage server.
The error occurred because a problem in the low-level RPC connection,
or because the connection was closed.
"""
# This subclass is raised when zrpc catches the error.
lib/python/ZEO/zrpc/log.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
import threading
import logging
from ZODB.loglevels import BLATHER
LOG_THREAD_ID = 0 # Set this to 1 during heavy debugging
logger = logging.getLogger('ZEO.zrpc')
_label = "%s" % os.getpid()
def new_label():
global _label
_label = str(os.getpid())
def log(message, level=BLATHER, label=None, exc_info=False):
label = label or _label
if LOG_THREAD_ID:
label = label + ':' + threading.currentThread().getName()
logger.log(level, '(%s) %s' % (label, message), exc_info=exc_info)
REPR_LIMIT = 60
def short_repr(obj):
"Return an object repr limited to REPR_LIMIT bytes."
# Some of the objects being repr'd are large strings. A lot of memory
# would be wasted to repr them and then truncate, so they are treated
# specially in this function.
# Also handle short repr of a tuple containing a long string.
# This strategy works well for arguments to StorageServer methods.
# The oid is usually first and will get included in its entirety.
# The pickle is near the beginning, too, and you can often fit the
# module name in the pickle.
if isinstance(obj, str):
if len(obj) > REPR_LIMIT:
r = repr(obj[:REPR_LIMIT])
else:
r = repr(obj)
if len(r) > REPR_LIMIT:
r = r[:REPR_LIMIT-4] + '...' + r[-1]
return r
elif isinstance(obj, (list, tuple)):
elts = []
size = 0
for elt in obj:
r = short_repr(elt)
elts.append(r)
size += len(r)
if size > REPR_LIMIT:
break
if isinstance(obj, tuple):
r = "(%s)" % (", ".join(elts))
else:
r = "[%s]" % (", ".join(elts))
else:
r = repr(obj)
if len(r) > REPR_LIMIT:
return r[:REPR_LIMIT] + '...'
else:
return r
lib/python/ZEO/zrpc/marshal.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import cPickle
from cStringIO import StringIO
import types
import logging
from ZEO.zrpc.error import ZRPCError
from ZEO.zrpc.log import log, short_repr
class Marshaller:
"""Marshal requests and replies to second across network"""
def encode(self, msgid, flags, name, args):
"""Returns an encoded message"""
# (We used to have a global pickler, but that's not thread-safe. :-( )
pickler = cPickle.Pickler()
pickler.fast = 1
return pickler.dump((msgid, flags, name, args), 1)
def decode(self, msg):
"""Decodes msg and returns its parts"""
unpickler = cPickle.Unpickler(StringIO(msg))
unpickler.find_global = find_global
try:
return unpickler.load() # msgid, flags, name, args
except:
log("can't decode message: %s" % short_repr(msg),
level=logging.ERROR)
raise
_globals = globals()
_silly = ('__doc__',)
def find_global(module, name):
"""Helper for message unpickler"""
try:
m = __import__(module, _globals, _globals, _silly)
except ImportError, msg:
raise ZRPCError("import error %s: %s" % (module, msg))
try:
r = getattr(m, name)
except AttributeError:
raise ZRPCError("module %s has no global %s" % (module, name))
safe = getattr(r, '__no_side_effects__', 0)
if safe:
return r
# XXX what's a better way to do this? esp w/ 2.1 & 2.2
if type(r) == types.ClassType and issubclass(r, Exception):
return r
raise ZRPCError("Unsafe global: %s.%s" % (module, name))
lib/python/ZEO/zrpc/server.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import socket
import types
from ZEO.zrpc.connection import Connection
from ZEO.zrpc.log import log
import logging
import ThreadedAsync.LoopCallback
# Export the main asyncore loop
loop = ThreadedAsync.LoopCallback.loop
class Dispatcher(asyncore.dispatcher):
"""A server that accepts incoming RPC connections"""
__super_init = asyncore.dispatcher.__init__
def __init__(self, addr, factory=Connection):
self.__super_init()
self.addr = addr
self.factory = factory
self._open_socket()
def _open_socket(self):
if type(self.addr) == types.TupleType:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
log("listening on %s" % str(self.addr), logging.INFO)
self.bind(self.addr)
self.listen(5)
def writable(self):
return 0
def readable(self):
return 1
def handle_accept(self):
try:
sock, addr = self.accept()
except socket.error, msg:
log("accepted failed: %s" % msg)
return
c = self.factory(sock, addr)
log("connect from %s: %s" % (repr(addr), c))
lib/python/ZEO/zrpc/smac.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Sized Message Async Connections.
This class extends the basic asyncore layer with a record-marking
layer. The message_output() method accepts an arbitrary sized string
as its argument. It sends over the wire the length of the string
encoded using struct.pack('>I') and the string itself. The receiver
passes the original string to message_input().
This layer also supports an optional message authentication code
(MAC). If a session key is present, it uses HMAC-SHA-1 to generate a
20-byte MAC. If a MAC is present, the high-order bit of the length
is set to 1 and the MAC immediately follows the length.
"""
import asyncore
import errno
try:
import hmac
except ImportError:
import _hmac as hmac
import sha
import socket
import struct
import threading
import logging
from types import StringType
from ZODB.loglevels import TRACE
from ZEO.zrpc.log import log, short_repr
from ZEO.zrpc.error import DisconnectedError
# Use the dictionary to make sure we get the minimum number of errno
# entries. We expect that EWOULDBLOCK == EAGAIN on most systems --
# or that only one is actually used.
tmp_dict = {errno.EWOULDBLOCK: 0,
errno.EAGAIN: 0,
errno.EINTR: 0,
}
expected_socket_read_errors = tuple(tmp_dict.keys())
tmp_dict = {errno.EAGAIN: 0,
errno.EWOULDBLOCK: 0,
errno.ENOBUFS: 0,
errno.EINTR: 0,
}
expected_socket_write_errors = tuple(tmp_dict.keys())
del tmp_dict
# We chose 60000 as the socket limit by looking at the largest strings
# that we could pass to send() without blocking.
SEND_SIZE = 60000
MAC_BIT = 0x80000000L
class SizedMessageAsyncConnection(asyncore.dispatcher):
__super_init = asyncore.dispatcher.__init__
__super_close = asyncore.dispatcher.close
__closed = True # Marker indicating that we're closed
socket = None # to outwit Sam's getattr
def __init__(self, sock, addr, map=None, debug=None):
self.addr = addr
if debug is not None:
self._debug = debug
elif not hasattr(self, '_debug'):
self._debug = __debug__
# __input_lock protects __inp, __input_len, __state, __msg_size
self.__input_lock = threading.Lock()
self.__inp = None # None, a single String, or a list
self.__input_len = 0
# Instance variables __state, __msg_size and __has_mac work together:
# when __state == 0:
# __msg_size == 4, and the next thing read is a message size;
# __has_mac is set according to the MAC_BIT in the header
# when __state == 1:
# __msg_size is variable, and the next thing read is a message.
# __has_mac indicates if we're in MAC mode or not (and
# therefore, if we need to check the mac header)
# The next thing read is always of length __msg_size.
# The state alternates between 0 and 1.
self.__state = 0
self.__has_mac = 0
self.__msg_size = 4
self.__output_lock = threading.Lock() # Protects __output
self.__output = []
self.__closed = False
# Each side of the connection sends and receives messages. A
# MAC is generated for each message and depends on each
# previous MAC; the state of the MAC generator depends on the
# history of operations it has performed. So the MACs must be
# generated in the same order they are verified.
# Each side is guaranteed to receive messages in the order
# they are sent, but there is no ordering constraint between
# message sends and receives. If the two sides are A and B
# and message An indicates the nth message sent by A, then
# A1 A2 B1 B2 and A1 B1 B2 A2 are both legitimate total
# orderings of the messages.
# As a result, there must be seperate MAC generators for each
# side of the connection. If not, the generator state would
# be different after A1 A2 B1 B2 than it would be after
# A1 B1 B2 A2; if the generator state was different, the MAC
# could not be verified.
self.__hmac_send = None
self.__hmac_recv = None
self.__super_init(sock, map)
def setSessionKey(self, sesskey):
log("set session key %r" % sesskey)
self.__hmac_send = hmac.HMAC(sesskey, digestmod=sha)
self.__hmac_recv = hmac.HMAC(sesskey, digestmod=sha)
def get_addr(self):
return self.addr
# XXX avoid expensive getattr calls? Can't remember exactly what
# this comment was supposed to mean, but it has something to do
# with the way asyncore uses getattr and uses if sock:
def __nonzero__(self):
return 1
def handle_read(self):
self.__input_lock.acquire()
try:
# Use a single __inp buffer and integer indexes to make this fast.
try:
d = self.recv(8192)
except socket.error, err:
if err[0] in expected_socket_read_errors:
return
raise
if not d:
return
input_len = self.__input_len + len(d)
msg_size = self.__msg_size
state = self.__state
has_mac = self.__has_mac
inp = self.__inp
if msg_size > input_len:
if inp is None:
self.__inp = d
elif type(self.__inp) is StringType:
self.__inp = [self.__inp, d]
else:
self.__inp.append(d)
self.__input_len = input_len
return # keep waiting for more input
# load all previous input and d into single string inp
if isinstance(inp, StringType):
inp = inp + d
elif inp is None:
inp = d
else:
inp.append(d)
inp = "".join(inp)
offset = 0
while (offset + msg_size) <= input_len:
msg = inp[offset:offset + msg_size]
offset = offset + msg_size
if not state:
msg_size = struct.unpack(">I", msg)[0]
has_mac = msg_size & MAC_BIT
if has_mac:
msg_size ^= MAC_BIT
msg_size += 20
elif self.__hmac_send:
raise ValueError("Received message without MAC")
state = 1
else:
msg_size = 4
state = 0
# XXX We call message_input() with __input_lock
# held!!! And message_input() may end up calling
# message_output(), which has its own lock. But
# message_output() cannot call message_input(), so
# the locking order is always consistent, which
# prevents deadlock. Also, message_input() may
# take a long time, because it can cause an
# incoming call to be handled. During all this
# time, the __input_lock is held. That's a good
# thing, because it serializes incoming calls.
if has_mac:
mac = msg[:20]
msg = msg[20:]
if self.__hmac_recv:
self.__hmac_recv.update(msg)
_mac = self.__hmac_recv.digest()
if mac != _mac:
raise ValueError("MAC failed: %r != %r"
% (_mac, mac))
else:
log("Received MAC but no session key set")
elif self.__hmac_send:
raise ValueError("Received message without MAC")
self.message_input(msg)
self.__state = state
self.__has_mac = has_mac
self.__msg_size = msg_size
self.__inp = inp[offset:]
self.__input_len = input_len - offset
finally:
self.__input_lock.release()
def readable(self):
return True
def writable(self):
if len(self.__output) == 0:
return False
else:
return True
def handle_write(self):
self.__output_lock.acquire()
try:
output = self.__output
while output:
# Accumulate output into a single string so that we avoid
# multiple send() calls, but avoid accumulating too much
# data. If we send a very small string and have more data
# to send, we will likely incur delays caused by the
# unfortunate interaction between the Nagle algorithm and
# delayed acks. If we send a very large string, only a
# portion of it will actually be delivered at a time.
l = 0
for i in range(len(output)):
l += len(output[i])
if l > SEND_SIZE:
break
i += 1
# It is very unlikely that i will be 1.
v = "".join(output[:i])
del output[:i]
try:
n = self.send(v)
except socket.error, err:
if err[0] in expected_socket_write_errors:
break # we couldn't write anything
raise
if n < len(v):
output.insert(0, v[n:])
break # we can't write any more
finally:
self.__output_lock.release()
def handle_close(self):
self.close()
def message_output(self, message):
if __debug__:
if self._debug:
log("message_output %d bytes: %s hmac=%d" %
(len(message), short_repr(message),
self.__hmac_send and 1 or 0),
level=TRACE)
if self.__closed:
raise DisconnectedError(
"This action is temporarily unavailable.<p>")
self.__output_lock.acquire()
try:
# do two separate appends to avoid copying the message string
if self.__hmac_send:
self.__output.append(struct.pack(">I", len(message) | MAC_BIT))
self.__hmac_send.update(message)
self.__output.append(self.__hmac_send.digest())
else:
self.__output.append(struct.pack(">I", len(message)))
if len(message) <= SEND_SIZE:
self.__output.append(message)
else:
for i in range(0, len(message), SEND_SIZE):
self.__output.append(message[i:i+SEND_SIZE])
finally:
self.__output_lock.release()
def close(self):
if not self.__closed:
self.__closed = True
self.__super_close()
lib/python/ZEO/zrpc/trigger.py 0 → 100644
View file @ 96172040
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import os
import socket
import thread
if os.name == 'posix':
class trigger(asyncore.file_dispatcher):
"Wake up a call to select() running in the main thread"
# This is useful in a context where you are using Medusa's I/O
# subsystem to deliver data, but the data is generated by another
# thread. Normally, if Medusa is in the middle of a call to
# select(), new output data generated by another thread will have
# to sit until the call to select() either times out or returns.
# If the trigger is 'pulled' by another thread, it should immediately
# generate a READ event on the trigger object, which will force the
# select() invocation to return.
# A common use for this facility: letting Medusa manage I/O for a
# large number of connections; but routing each request through a
# thread chosen from a fixed-size thread pool. When a thread is
# acquired, a transaction is performed, but output data is
# accumulated into buffers that will be emptied more efficiently
# by Medusa. [picture a server that can process database queries
# rapidly, but doesn't want to tie up threads waiting to send data
# to low-bandwidth connections]
# The other major feature provided by this class is the ability to
# move work back into the main thread: if you call pull_trigger()
# with a thunk argument, when select() wakes up and receives the
# event it will call your thunk from within that thread. The main
# purpose of this is to remove the need to wrap thread locks around
# Medusa's data structures, which normally do not need them. [To see
# why this is true, imagine this scenario: A thread tries to push some
# new data onto a channel's outgoing data queue at the same time that
# the main thread is trying to remove some]
def __init__(self):
r, w = self._fds = os.pipe()
self.trigger = w
asyncore.file_dispatcher.__init__(self, r)
self.lock = thread.allocate_lock()
self.thunks = []
self._closed = 0
# Override the asyncore close() method, because it seems that
# it would only close the r file descriptor and not w. The
# constructor calls file_dispatcher.__init__ and passes r,
# which would get stored in a file_wrapper and get closed by
# the default close. But that would leave w open...
def close(self):
if not self._closed:
self._closed = 1
self.del_channel()
for fd in self._fds:
os.close(fd)
self._fds = []
def __repr__(self):
return '<select-trigger (pipe) at %x>' % id(self)
def readable(self):
return 1
def writable(self):
return 0
def handle_connect(self):
pass
def handle_close(self):
self.close()
def pull_trigger(self, thunk=None):
if thunk:
self.lock.acquire()
try:
self.thunks.append(thunk)
finally:
self.lock.release()
os.write(self.trigger, 'x')
def handle_read(self):
try:
self.recv(8192)
except socket.error:
return
self.lock.acquire()
try:
for thunk in self.thunks:
try:
thunk()
except:
nil, t, v, tbinfo = asyncore.compact_traceback()
print ('exception in trigger thunk:'
' (%s:%s %s)' % (t, v, tbinfo))
self.thunks = []
finally:
self.lock.release()
else:
# XXX Should define a base class that has the common methods and
# then put the platform-specific in a subclass named trigger.
# win32-safe version
HOST = '127.0.0.1'
MINPORT = 19950
NPORTS = 50
class trigger(asyncore.dispatcher):
portoffset = 0
def __init__(self):
a = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
w = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# set TCP_NODELAY to true to avoid buffering
w.setsockopt(socket.IPPROTO_TCP, 1, 1)
# tricky: get a pair of connected sockets
for i in range(NPORTS):
trigger.portoffset = (trigger.portoffset + 1) % NPORTS
port = MINPORT + trigger.portoffset
address = (HOST, port)
try:
a.bind(address)
except socket.error:
continue
else:
break
else:
raise RuntimeError, 'Cannot bind trigger!'
a.listen(1)
w.setblocking(0)
try:
w.connect(address)
except:
pass
r, addr = a.accept()
a.close()
w.setblocking(1)
self.trigger = w
asyncore.dispatcher.__init__(self, r)
self.lock = thread.allocate_lock()
self.thunks = []
self._trigger_connected = 0
self._closed = 0
def close(self):
if not self._closed:
self._closed = 1
self.del_channel()
# self.socket is a, self.trigger is w from __init__
self.socket.close()
self.trigger.close()
def __repr__(self):
return '<select-trigger (loopback) at %x>' % id(self)
def readable(self):
return 1
def writable(self):
return 0
def handle_connect(self):
pass
def pull_trigger(self, thunk=None):
if thunk:
self.lock.acquire()
try:
self.thunks.append(thunk)
finally:
self.lock.release()
self.trigger.send('x')
def handle_read(self):
try:
self.recv(8192)
except socket.error:
return
self.lock.acquire()
try:
for thunk in self.thunks:
try:
thunk()
except:
nil, t, v, tbinfo = asyncore.compact_traceback()
print ('exception in trigger thunk:'
' (%s:%s %s)' % (t, v, tbinfo))
self.thunks = []
finally:
self.lock.release()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7