Skip to content

Z
ZODB
Commit e3082da7 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Split ZEO into separate project

parent 56c7dd30
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 21209 deletions
setup.py
View file @ e3082da7
...@@ -20,7 +20,7 @@ to application logic. ZODB includes features such as a plugable storage ...@@ -20,7 +20,7 @@ to application logic. ZODB includes features such as a plugable storage
interface, rich transaction support, and undo. interface, rich transaction support, and undo.
""" """
VERSION = "3.11dev" VERSION = "4.0.0dev"
from ez_setup import use_setuptools from ez_setup import use_setuptools
use_setuptools() use_setuptools()
......
src/ZEO/ClientStorage.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The ClientStorage class and the exceptions that it may raise.
Public contents of this module:
ClientStorage -- the main class, implementing the Storage API
"""
from persistent.TimeStamp import TimeStamp
from ZEO.auth import get_module
from ZEO.cache import ClientCache
from ZEO.Exceptions import ClientStorageError, ClientDisconnected, AuthError
from ZEO import ServerStub
from ZEO.TransactionBuffer import TransactionBuffer
from ZEO.zrpc.client import ConnectionManager
from ZODB import POSException
from ZODB import utils
import BTrees.IOBTree
import cPickle
import logging
import os
import re
import socket
import stat
import sys
import tempfile
import thread
import threading
import time
import weakref
import zc.lockfile
import ZEO.interfaces
import ZODB
import ZODB.BaseStorage
import ZODB.interfaces
import ZODB.event
import zope.interface
logger = logging.getLogger(__name__)
try:
from ZODB.ConflictResolution import ResolvedSerial
except ImportError:
ResolvedSerial = 'rs'
def tid2time(tid):
return str(TimeStamp(tid))
def get_timestamp(prev_ts=None):
"""Internal helper to return a unique TimeStamp instance.
If the optional argument is not None, it must be a TimeStamp; the
return value is then guaranteed to be at least 1 microsecond later
the argument.
"""
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5] + (t % 60,))
if prev_ts is not None:
t = t.laterThan(prev_ts)
return t
class DisconnectedServerStub:
"""Internal helper class used as a faux RPC stub when disconnected.
This raises ClientDisconnected on all attribute accesses.
This is a singleton class -- there should be only one instance,
the global disconnected_stub, so it can be tested by identity.
"""
def __getattr__(self, attr):
raise ClientDisconnected()
# Singleton instance of DisconnectedServerStub
disconnected_stub = DisconnectedServerStub()
MB = 1024**2
class ClientStorage(object):
"""A storage class that is a network client to a remote storage.
This is a faithful implementation of the Storage API.
This class is thread-safe; transactions are serialized in
tpc_begin().
"""
# ClientStorage does not declare any interfaces here. Interfaces are
# declared according to the server's storage once a connection is
# established.
# Classes we instantiate. A subclass might override.
TransactionBufferClass = TransactionBuffer
ClientCacheClass = ClientCache
ConnectionManagerClass = ConnectionManager
StorageServerStubClass = ServerStub.stub
def __init__(self, addr, storage='1', cache_size=20 * MB,
name='', client=None, var=None,
min_disconnect_poll=1, max_disconnect_poll=30,
wait_for_server_on_startup=None, # deprecated alias for wait
wait=None, wait_timeout=None,
read_only=0, read_only_fallback=0,
drop_cache_rather_verify=False,
username='', password='', realm=None,
blob_dir=None, shared_blob_dir=False,
blob_cache_size=None, blob_cache_size_check=10,
client_label=None,
):
"""ClientStorage constructor.
This is typically invoked from a custom_zodb.py file.
All arguments except addr should be keyword arguments.
Arguments:
addr
The server address(es). This is either a list of
addresses or a single address. Each address can be a
(hostname, port) tuple to signify a TCP/IP connection or
a pathname string to signify a Unix domain socket
connection. A hostname may be a DNS name or a dotted IP
address. Required.
storage
The storage name, defaulting to '1'. The name must
match one of the storage names supported by the server(s)
specified by the addr argument. The storage name is
displayed in the Zope control panel.
cache_size
The disk cache size, defaulting to 20 megabytes.
This is passed to the ClientCache constructor.
name
The storage name, defaulting to ''. If this is false,
str(addr) is used as the storage name.
client
A name used to construct persistent cache filenames.
Defaults to None, in which case the cache is not persistent.
See ClientCache for more info.
var
When client is not None, this specifies the directory
where the persistent cache files are created. It defaults
to None, in whichcase the current directory is used.
min_disconnect_poll
The minimum delay in seconds between
attempts to connect to the server, in seconds. Defaults
to 5 seconds.
max_disconnect_poll
The maximum delay in seconds between
attempts to connect to the server, in seconds. Defaults
to 300 seconds.
wait_for_server_on_startup
A backwards compatible alias for
the wait argument.
wait
A flag indicating whether to wait until a connection
with a server is made, defaulting to true.
wait_timeout
Maximum time to wait for a connection before
giving up. Only meaningful if wait is True.
read_only
A flag indicating whether this should be a
read-only storage, defaulting to false (i.e. writing is
allowed by default).
read_only_fallback
A flag indicating whether a read-only
remote storage should be acceptable as a fallback when no
writable storages are available. Defaults to false. At
most one of read_only and read_only_fallback should be
true.
username
string with username to be used when authenticating.
These only need to be provided if you are connecting to an
authenticated server storage.
password
string with plaintext password to be used when authenticated.
realm
not documented.
drop_cache_rather_verify
a flag indicating that the cache should be dropped rather
than expensively verified.
blob_dir
directory path for blob data. 'blob data' is data that
is retrieved via the loadBlob API.
shared_blob_dir
Flag whether the blob_dir is a server-shared filesystem
that should be used instead of transferring blob data over
zrpc.
blob_cache_size
Maximum size of the ZEO blob cache, in bytes. If not set, then
the cache size isn't checked and the blob directory will
grow without bound.
This option is ignored if shared_blob_dir is true.
blob_cache_size_check
ZEO check size as percent of blob_cache_size. The ZEO
cache size will be checked when this many bytes have been
loaded into the cache. Defaults to 10% of the blob cache
size. This option is ignored if shared_blob_dir is true.
client_label
A label to include in server log messages for the client.
Note that the authentication protocol is defined by the server
and is detected by the ClientStorage upon connecting (see
testConnection() and doAuth() for details).
"""
if isinstance(addr, int):
addr = '127.0.0.1', addr
self.__name__ = name or str(addr) # Standard convention for storages
logger.info(
"%s %s (pid=%d) created %s/%s for storage: %r",
self.__name__,
self.__class__.__name__,
os.getpid(),
read_only and "RO" or "RW",
read_only_fallback and "fallback" or "normal",
storage,
)
self._drop_cache_rather_verify = drop_cache_rather_verify
# wait defaults to True, but wait_for_server_on_startup overrides
# if not None
if wait_for_server_on_startup is not None:
if wait is not None and wait != wait_for_server_on_startup:
logger.warning(
"%s ClientStorage(): conflicting values for wait and "
"wait_for_server_on_startup; wait prevails",
self.__name__)
else:
logger.info(
"%s ClientStorage(): wait_for_server_on_startup "
"is deprecated; please use wait instead",
self.__name__)
wait = wait_for_server_on_startup
elif wait is None:
wait = 1
self._addr = addr # For tests
# A ZEO client can run in disconnected mode, using data from
# its cache, or in connected mode. Several instance variables
# are related to whether the client is connected.
# _server: All method calls are invoked through the server
# stub. When not connect, set to disconnected_stub an
# object that raises ClientDisconnected errors.
# _ready: A threading Event that is set only if _server
# is set to a real stub.
# _connection: The current zrpc connection or None.
# _connection is set as soon as a connection is established,
# but _server is set only after cache verification has finished
# and clients can safely use the server. _pending_server holds
# a server stub while it is being verified.
self._server = disconnected_stub
self._connection = None
self._pending_server = None
self._ready = threading.Event()
# _is_read_only stores the constructor argument
self._is_read_only = read_only
self._storage = storage
self._read_only_fallback = read_only_fallback
self._username = username
self._password = password
self._realm = realm
self._iterators = weakref.WeakValueDictionary()
self._iterator_ids = set()
# Flag tracking disconnections in the middle of a transaction. This
# is reset in tpc_begin() and set in notifyDisconnected().
self._midtxn_disconnect = 0
# _server_addr is used by sortKey()
self._server_addr = None
self._client_label = client_label
self._pickler = self._tfile = None
self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client',
'supportsUndo': 0, 'interfaces': ()}
self._tbuf = self.TransactionBufferClass()
self._db = None
self._ltid = None # the last committed transaction
# _serials: stores (oid, serialno) as returned by server
# _seriald: _check_serials() moves from _serials to _seriald,
# which maps oid to serialno
# TODO: If serial number matches transaction id, then there is
# no need to have all this extra infrastructure for handling
# serial numbers. The vote call can just return the tid.
# If there is a conflict error, we can't have a special method
# called just to propagate the error.
self._serials = []
self._seriald = {}
# A ClientStorage only allows one thread to commit at a time.
# Mutual exclusion is achieved using _tpc_cond, which
# protects _transaction. A thread that wants to assign to
# self._transaction must acquire _tpc_cond first. A thread
# that decides it's done with a transaction (whether via success
# or failure) must set _transaction to None and do
# _tpc_cond.notify() before releasing _tpc_cond.
self._tpc_cond = threading.Condition()
self._transaction = None
# Prevent multiple new_oid calls from going out. The _oids
# variable should only be modified while holding the
# _oid_lock.
self._oid_lock = threading.Lock()
self._oids = [] # Object ids retrieved from new_oids()
# load() and tpc_finish() must be serialized to guarantee
# that cache modifications from each occur atomically.
# It also prevents multiple load calls occuring simultaneously,
# which simplifies the cache logic.
self._load_lock = threading.Lock()
# _load_oid and _load_status are protected by _lock
self._load_oid = None
self._load_status = None
# Can't read data in one thread while writing data
# (tpc_finish) in another thread. In general, the lock
# must prevent access to the cache while _update_cache
# is executing.
self._lock = threading.Lock()
# XXX need to check for POSIX-ness here
self.blob_dir = blob_dir
self.shared_blob_dir = shared_blob_dir
if blob_dir is not None:
# Avoid doing this import unless we need it, as it
# currently requires pywin32 on Windows.
import ZODB.blob
if shared_blob_dir:
self.fshelper = ZODB.blob.FilesystemHelper(blob_dir)
else:
if 'zeocache' not in ZODB.blob.LAYOUTS:
ZODB.blob.LAYOUTS['zeocache'] = BlobCacheLayout()
self.fshelper = ZODB.blob.FilesystemHelper(
blob_dir, layout_name='zeocache')
self.fshelper.create()
self.fshelper.checkSecure()
else:
self.fshelper = None
if client is not None:
dir = var or os.getcwd()
cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
else:
cache_path = None
self._cache = self.ClientCacheClass(cache_path, size=cache_size)
self._blob_cache_size = blob_cache_size
self._blob_data_bytes_loaded = 0
if blob_cache_size is not None:
assert blob_cache_size_check < 100
self._blob_cache_size_check = (
blob_cache_size * blob_cache_size_check / 100)
self._check_blob_size()
self._rpc_mgr = self.ConnectionManagerClass(addr, self,
tmin=min_disconnect_poll,
tmax=max_disconnect_poll)
if wait:
self._wait(wait_timeout)
else:
# attempt_connect() will make an attempt that doesn't block
# "too long," for a very vague notion of too long. If that
# doesn't succeed, call connect() to start a thread.
if not self._rpc_mgr.attempt_connect():
self._rpc_mgr.connect()
def new_addr(self, addr):
self._addr = addr
self._rpc_mgr.new_addrs(addr)
def _wait(self, timeout=None):
if timeout is not None:
deadline = time.time() + timeout
logger.debug("%s Setting deadline to %f", self.__name__, deadline)
else:
deadline = None
# Wait for a connection to be established.
self._rpc_mgr.connect(sync=1)
# When a synchronous connect() call returns, there is
# a valid _connection object but cache validation may
# still be going on. This code must wait until validation
# finishes, but if the connection isn't a zrpc async
# connection it also needs to poll for input.
while 1:
self._ready.wait(30)
if self._ready.isSet():
break
if timeout and time.time() > deadline:
logger.warning("%s Timed out waiting for connection",
self.__name__)
break
logger.info("%s Waiting for cache verification to finish",
self.__name__)
def close(self):
"Storage API: finalize the storage, releasing external resources."
_rpc_mgr = self._rpc_mgr
self._rpc_mgr = None
if _rpc_mgr is None:
return # already closed
if self._connection is not None:
self._connection.register_object(None) # Don't call me!
self._connection = None
_rpc_mgr.close()
self._tbuf.close()
if self._cache is not None:
self._cache.close()
self._cache = None
if self._tfile is not None:
self._tfile.close()
if self._check_blob_size_thread is not None:
self._check_blob_size_thread.join()
_check_blob_size_thread = None
def _check_blob_size(self, bytes=None):
if self._blob_cache_size is None:
return
if self.shared_blob_dir or not self.blob_dir:
return
if (bytes is not None) and (bytes < self._blob_cache_size_check):
return
self._blob_data_bytes_loaded = 0
target = max(self._blob_cache_size - self._blob_cache_size_check, 0)
check_blob_size_thread = threading.Thread(
target=_check_blob_cache_size,
args=(self.blob_dir, target),
)
check_blob_size_thread.setDaemon(True)
check_blob_size_thread.start()
self._check_blob_size_thread = check_blob_size_thread
def registerDB(self, db):
"""Storage API: register a database for invalidation messages.
This is called by ZODB.DB (and by some tests).
The storage isn't really ready to use until after this call.
"""
self._db = db
def is_connected(self, test=False):
"""Return whether the storage is currently connected to a server."""
# This function is used by clients, so we only report that a
# connection exists when the connection is ready to use.
if test:
try:
self._server.lastTransaction()
except Exception:
pass
return self._ready.isSet()
def sync(self):
# The separate async thread should keep us up to date
pass
def doAuth(self, protocol, stub):
if not (self._username and self._password):
raise AuthError("empty username or password")
module = get_module(protocol)
if not module:
logger.error("%s %s: no such an auth protocol: %s",
self.__name__, self.__class__.__name__, protocol)
return
storage_class, client, db_class = module
if not client:
logger.error(
"%s %s: %s isn't a valid protocol, must have a Client class",
self.__name__, self.__class__.__name__, protocol)
raise AuthError("invalid protocol")
c = client(stub)
# Initiate authentication, returns boolean specifying whether OK
return c.start(self._username, self._realm, self._password)
def testConnection(self, conn):
"""Internal: test the given connection.
This returns: 1 if the connection is an optimal match,
0 if it is a suboptimal but acceptable match.
It can also raise DisconnectedError or ReadOnlyError.
This is called by ZEO.zrpc.ConnectionManager to decide which
connection to use in case there are multiple, and some are
read-only and others are read-write.
This works by calling register() on the server. In read-only
mode, register() is called with the read_only flag set. In
writable mode and in read-only fallback mode, register() is
called with the read_only flag cleared. In read-only fallback
mode only, if the register() call raises ReadOnlyError, it is
retried with the read-only flag set, and if this succeeds,
this is deemed a suboptimal match. In all other cases, a
succeeding register() call is deemed an optimal match, and any
exception raised by register() is passed through.
"""
logger.info("%s Testing connection %r", self.__name__, conn)
# TODO: Should we check the protocol version here?
conn._is_read_only = self._is_read_only
stub = self.StorageServerStubClass(conn)
auth = stub.getAuthProtocol()
logger.info("%s Server authentication protocol %r", self.__name__, auth)
if auth:
skey = self.doAuth(auth, stub)
if skey:
logger.info("%s Client authentication successful",
self.__name__)
conn.setSessionKey(skey)
else:
logger.info("%s Authentication failed",
self.__name__)
raise AuthError("Authentication failed")
try:
stub.register(str(self._storage), self._is_read_only)
return 1
except POSException.ReadOnlyError:
if not self._read_only_fallback:
raise
logger.info("%s Got ReadOnlyError; trying again with read_only=1",
self.__name__)
stub.register(str(self._storage), read_only=1)
conn._is_read_only = True
return 0
def notifyConnected(self, conn):
"""Internal: start using the given connection.
This is called by ConnectionManager after it has decided which
connection should be used.
"""
if self._cache is None:
# the storage was closed, but the connect thread called
# this method before it was stopped.
return
if self._connection is not None:
# If we are upgrading from a read-only fallback connection,
# we must close the old connection to prevent it from being
# used while the cache is verified against the new connection.
self._connection.register_object(None) # Don't call me!
self._connection.close()
self._connection = None
self._ready.clear()
reconnect = 1
else:
reconnect = 0
self.set_server_addr(conn.get_addr())
self._connection = conn
# invalidate our db cache
if self._db is not None:
self._db.invalidateCache()
if reconnect:
logger.info("%s Reconnected to storage: %s",
self.__name__, self._server_addr)
else:
logger.info("%s Connected to storage: %s",
self.__name__, self._server_addr)
stub = self.StorageServerStubClass(conn)
if self._client_label and conn.peer_protocol_version >= "Z310":
stub.set_client_label(self._client_label)
if conn.peer_protocol_version < "Z3101":
logger.warning("Old server doesn't suppport "
"checkCurrentSerialInTransaction")
self.checkCurrentSerialInTransaction = lambda *args: None
self._oids = []
self.verify_cache(stub)
# It's important to call get_info after calling verify_cache.
# If we end up doing a full-verification, we need to wait till
# it's done. By doing a synchonous call, we are guarenteed
# that the verification will be done because operations are
# handled in order.
self._info.update(stub.get_info())
self._handle_extensions()
for iface in (
ZODB.interfaces.IStorageRestoreable,
ZODB.interfaces.IStorageIteration,
ZODB.interfaces.IStorageUndoable,
ZODB.interfaces.IStorageCurrentRecordIteration,
ZODB.interfaces.IBlobStorage,
ZODB.interfaces.IExternalGC,
):
if (iface.__module__, iface.__name__) in self._info.get(
'interfaces', ()):
zope.interface.alsoProvides(self, iface)
def _handle_extensions(self):
for name in self.getExtensionMethods().keys():
if not hasattr(self, name):
def mklambda(mname):
return (lambda *args, **kw:
self._server.rpc.call(mname, *args, **kw))
setattr(self, name, mklambda(name))
def set_server_addr(self, addr):
# Normalize server address and convert to string
if isinstance(addr, str):
self._server_addr = addr
else:
assert isinstance(addr, tuple)
# If the server is on a remote host, we need to guarantee
# that all clients used the same name for the server. If
# they don't, the sortKey() may be different for each client.
# The best solution seems to be the official name reported
# by gethostbyaddr().
host = addr[0]
try:
canonical, aliases, addrs = socket.gethostbyaddr(host)
except socket.error, err:
logger.debug("%s Error resolving host: %s (%s)",
self.__name__, host, err)
canonical = host
self._server_addr = str((canonical, addr[1]))
def sortKey(self):
# If the client isn't connected to anything, it can't have a
# valid sortKey(). Raise an error to stop the transaction early.
if self._server_addr is None:
raise ClientDisconnected
else:
return '%s:%s' % (self._storage, self._server_addr)
### Is there a race condition between notifyConnected and
### notifyDisconnected? In Particular, what if we get
### notifyDisconnected in the middle of notifyConnected?
### The danger is that we'll proceed as if we were connected
### without worrying if we were, but this would happen any way if
### notifyDisconnected had to get the instance lock. There's
### nothing to gain by getting the instance lock.
def notifyDisconnected(self):
"""Internal: notify that the server connection was terminated.
This is called by ConnectionManager when the connection is
closed or when certain problems with the connection occur.
"""
logger.info("%s Disconnected from storage: %r",
self.__name__, self._server_addr)
self._connection = None
self._ready.clear()
self._server = disconnected_stub
self._midtxn_disconnect = 1
self._iterator_gc(True)
def __len__(self):
"""Return the size of the storage."""
# TODO: Is this method used?
return self._info['length']
def getName(self):
"""Storage API: return the storage name as a string.
The return value consists of two parts: the name as determined
by the name and addr argments to the ClientStorage
constructor, and the string 'connected' or 'disconnected' in
parentheses indicating whether the storage is (currently)
connected.
"""
return "%s (%s)" % (
self.__name__,
self.is_connected() and "connected" or "disconnected")
def getSize(self):
"""Storage API: an approximate size of the database, in bytes."""
return self._info['size']
def getExtensionMethods(self):
"""getExtensionMethods
This returns a dictionary whose keys are names of extra methods
provided by this storage. Storage proxies (such as ZEO) should
call this method to determine the extra methods that they need
to proxy in addition to the standard storage methods.
Dictionary values should be None; this will be a handy place
for extra marshalling information, should we need it
"""
return self._info.get('extensionMethods', {})
def supportsUndo(self):
"""Storage API: return whether we support undo."""
return self._info['supportsUndo']
def isReadOnly(self):
"""Storage API: return whether we are in read-only mode."""
if self._is_read_only:
return True
else:
# If the client is configured for a read-write connection
# but has a read-only fallback connection, conn._is_read_only
# will be True. If self._connection is None, we'll behave as
# read_only
try:
return self._connection._is_read_only
except AttributeError:
return True
def _check_trans(self, trans):
"""Internal helper to check a transaction argument for sanity."""
if self._is_read_only:
raise POSException.ReadOnlyError()
if self._transaction is not trans:
raise POSException.StorageTransactionError(self._transaction,
trans)
def history(self, oid, size=1):
"""Storage API: return a sequence of HistoryEntry objects.
"""
return self._server.history(oid, size)
def record_iternext(self, next=None):
"""Storage API: get the next database record.
This is part of the conversion-support API.
"""
return self._server.record_iternext(next)
def getTid(self, oid):
"""Storage API: return current serial number for oid."""
return self._server.getTid(oid)
def loadSerial(self, oid, serial):
"""Storage API: load a historical revision of an object."""
return self._server.loadSerial(oid, serial)
def load(self, oid, version=''):
"""Storage API: return the data for a given object.
This returns the pickle data and serial number for the object
specified by the given object id, if they exist;
otherwise a KeyError is raised.
"""
self._lock.acquire() # for atomic processing of invalidations
try:
t = self._cache.load(oid)
if t:
return t
finally:
self._lock.release()
if self._server is None:
raise ClientDisconnected()
self._load_lock.acquire()
try:
self._lock.acquire()
try:
self._load_oid = oid
self._load_status = 1
finally:
self._lock.release()
data, tid = self._server.loadEx(oid)
self._lock.acquire() # for atomic processing of invalidations
try:
if self._load_status:
self._cache.store(oid, tid, None, data)
self._load_oid = None
finally:
self._lock.release()
finally:
self._load_lock.release()
return data, tid
def loadBefore(self, oid, tid):
self._lock.acquire()
try:
t = self._cache.loadBefore(oid, tid)
if t is not None:
return t
finally:
self._lock.release()
t = self._server.loadBefore(oid, tid)
if t is None:
return None
data, start, end = t
if end is None:
# This method should not be used to get current data. It
# doesn't use the _load_lock, so it is possble to overlap
# this load with an invalidation for the same object.
# If we call again, we're guaranteed to get the
# post-invalidation data. But if the data is still
# current, we'll still get end == None.
# Maybe the best thing to do is to re-run the test with
# the load lock in the case. That's slow performance, but
# I don't think real application code will ever care about
# it.
return data, start, end
self._lock.acquire()
try:
self._cache.store(oid, start, end, data)
finally:
self._lock.release()
return data, start, end
def new_oid(self):
"""Storage API: return a new object identifier."""
if self._is_read_only:
raise POSException.ReadOnlyError()
# avoid multiple oid requests to server at the same time
self._oid_lock.acquire()
try:
if not self._oids:
self._oids = self._server.new_oids()
self._oids.reverse()
return self._oids.pop()
finally:
self._oid_lock.release()
def pack(self, t=None, referencesf=None, wait=1, days=0):
"""Storage API: pack the storage.
Deviations from the Storage API: the referencesf argument is
ignored; two additional optional arguments wait and days are
provided:
wait -- a flag indicating whether to wait for the pack to
complete; defaults to true.
days -- a number of days to subtract from the pack time;
defaults to zero.
"""
# TODO: Is it okay that read-only connections allow pack()?
# rf argument ignored; server will provide its own implementation
if t is None:
t = time.time()
t = t - (days * 86400)
return self._server.pack(t, wait)
def _check_serials(self):
"""Internal helper to move data from _serials to _seriald."""
# serials are always going to be the same, the only
# question is whether an exception has been raised.
if self._serials:
l = len(self._serials)
r = self._serials[:l]
del self._serials[:l]
for oid, s in r:
if isinstance(s, Exception):
self._cache.invalidate(oid, None)
raise s
self._seriald[oid] = s
return r
def store(self, oid, serial, data, version, txn):
"""Storage API: store data for an object."""
assert not version
self._check_trans(txn)
self._server.storea(oid, serial, data, id(txn))
self._tbuf.store(oid, data)
return self._check_serials()
def checkCurrentSerialInTransaction(self, oid, serial, transaction):
self._check_trans(transaction)
self._server.checkCurrentSerialInTransaction(oid, serial,
id(transaction))
def storeBlob(self, oid, serial, data, blobfilename, version, txn):
"""Storage API: store a blob object."""
assert not version
# Grab the file right away. That way, if we don't have enough
# room for a copy, we'll know now rather than in tpc_finish.
# Also, this releaves the client of having to manage the file
# (or the directory contianing it).
self.fshelper.getPathForOID(oid, create=True)
fd, target = self.fshelper.blob_mkstemp(oid, serial)
os.close(fd)
# It's a bit odd (and impossible on windows) to rename over
# an existing file. We'll use the temporary file name as a base.
target += '-'
ZODB.blob.rename_or_copy_blob(blobfilename, target)
os.remove(target[:-1])
serials = self.store(oid, serial, data, '', txn)
if self.shared_blob_dir:
self._server.storeBlobShared(
oid, serial, data, os.path.basename(target), id(txn))
else:
self._server.storeBlob(oid, serial, data, target, txn)
self._tbuf.storeBlob(oid, target)
return serials
def receiveBlobStart(self, oid, serial):
blob_filename = self.fshelper.getBlobFilename(oid, serial)
assert not os.path.exists(blob_filename)
lockfilename = os.path.join(os.path.dirname(blob_filename), '.lock')
assert os.path.exists(lockfilename)
blob_filename += '.dl'
assert not os.path.exists(blob_filename)
f = open(blob_filename, 'wb')
f.close()
def receiveBlobChunk(self, oid, serial, chunk):
blob_filename = self.fshelper.getBlobFilename(oid, serial)+'.dl'
assert os.path.exists(blob_filename)
f = open(blob_filename, 'r+b')
f.seek(0, 2)
f.write(chunk)
f.close()
self._blob_data_bytes_loaded += len(chunk)
self._check_blob_size(self._blob_data_bytes_loaded)
def receiveBlobStop(self, oid, serial):
blob_filename = self.fshelper.getBlobFilename(oid, serial)
os.rename(blob_filename+'.dl', blob_filename)
os.chmod(blob_filename, stat.S_IREAD)
def deleteObject(self, oid, serial, txn):
self._check_trans(txn)
self._server.deleteObject(oid, serial, id(txn))
self._tbuf.store(oid, None)
def loadBlob(self, oid, serial):
# Load a blob. If it isn't present and we have a shared blob
# directory, then assume that it doesn't exist on the server
# and return None.
if self.fshelper is None:
raise POSException.Unsupported("No blob cache directory is "
"configured.")
blob_filename = self.fshelper.getBlobFilename(oid, serial)
if self.shared_blob_dir:
if os.path.exists(blob_filename):
return blob_filename
else:
# We're using a server shared cache. If the file isn't
# here, it's not anywhere.
raise POSException.POSKeyError("No blob file", oid, serial)
if os.path.exists(blob_filename):
return _accessed(blob_filename)
# First, we'll create the directory for this oid, if it doesn't exist.
self.fshelper.createPathForOID(oid)
# OK, it's not here and we (or someone) needs to get it. We
# want to avoid getting it multiple times. We want to avoid
# getting it multiple times even accross separate client
# processes on the same machine. We'll use file locking.
lock = _lock_blob(blob_filename)
try:
# We got the lock, so it's our job to download it. First,
# we'll double check that someone didn't download it while we
# were getting the lock:
if os.path.exists(blob_filename):
return _accessed(blob_filename)
# Ask the server to send it to us. When this function
# returns, it will have been sent. (The recieving will
# have been handled by the asyncore thread.)
self._server.sendBlob(oid, serial)
if os.path.exists(blob_filename):
return _accessed(blob_filename)
raise POSException.POSKeyError("No blob file", oid, serial)
finally:
lock.close()
def openCommittedBlobFile(self, oid, serial, blob=None):
blob_filename = self.loadBlob(oid, serial)
try:
if blob is None:
return open(blob_filename, 'rb')
else:
return ZODB.blob.BlobFile(blob_filename, 'r', blob)
except (IOError):
# The file got removed while we were opening.
# Fall through and try again with the protection of the lock.
pass
lock = _lock_blob(blob_filename)
try:
blob_filename = self.fshelper.getBlobFilename(oid, serial)
if not os.path.exists(blob_filename):
if self.shared_blob_dir:
# We're using a server shared cache. If the file isn't
# here, it's not anywhere.
raise POSException.POSKeyError("No blob file", oid, serial)
self._server.sendBlob(oid, serial)
if not os.path.exists(blob_filename):
raise POSException.POSKeyError("No blob file", oid, serial)
_accessed(blob_filename)
if blob is None:
return open(blob_filename, 'rb')
else:
return ZODB.blob.BlobFile(blob_filename, 'r', blob)
finally:
lock.close()
def temporaryDirectory(self):
return self.fshelper.temp_dir
def tpc_vote(self, txn):
"""Storage API: vote on a transaction."""
if txn is not self._transaction:
raise POSException.StorageTransactionError(
"tpc_vote called with wrong transaction")
self._server.vote(id(txn))
return self._check_serials()
def tpc_transaction(self):
return self._transaction
def tpc_begin(self, txn, tid=None, status=' '):
"""Storage API: begin a transaction."""
if self._is_read_only:
raise POSException.ReadOnlyError()
self._tpc_cond.acquire()
self._midtxn_disconnect = 0
while self._transaction is not None:
# It is allowable for a client to call two tpc_begins in a
# row with the same transaction, and the second of these
# must be ignored.
if self._transaction == txn:
self._tpc_cond.release()
raise POSException.StorageTransactionError(
"Duplicate tpc_begin calls for same transaction")
self._tpc_cond.wait(30)
self._transaction = txn
self._tpc_cond.release()
try:
self._server.tpc_begin(id(txn), txn.user, txn.description,
txn._extension, tid, status)
except:
# Client may have disconnected during the tpc_begin().
if self._server is not disconnected_stub:
self.end_transaction()
raise
self._tbuf.clear()
self._seriald.clear()
del self._serials[:]
def end_transaction(self):
"""Internal helper to end a transaction."""
# the right way to set self._transaction to None
# calls notify() on _tpc_cond in case there are waiting threads
self._tpc_cond.acquire()
self._transaction = None
self._tpc_cond.notify()
self._tpc_cond.release()
def lastTransaction(self):
return self._cache.getLastTid()
def tpc_abort(self, txn):
"""Storage API: abort a transaction."""
if txn is not self._transaction:
return
try:
# Caution: Are there any exceptions that should prevent an
# abort from occurring? It seems wrong to swallow them
# all, yet you want to be sure that other abort logic is
# executed regardless.
try:
self._server.tpc_abort(id(txn))
except ClientDisconnected:
logger.debug("%s ClientDisconnected in tpc_abort() ignored",
self.__name__)
finally:
self._tbuf.clear()
self._seriald.clear()
del self._serials[:]
self._iterator_gc()
self.end_transaction()
def tpc_finish(self, txn, f=None):
"""Storage API: finish a transaction."""
if txn is not self._transaction:
raise POSException.StorageTransactionError(
"tpc_finish called with wrong transaction")
self._load_lock.acquire()
try:
if self._midtxn_disconnect:
raise ClientDisconnected(
'Calling tpc_finish() on a disconnected transaction')
finished = 0
try:
self._lock.acquire() # for atomic processing of invalidations
try:
tid = self._server.tpc_finish(id(txn))
finished = 1
self._update_cache(tid)
if f is not None:
f(tid)
finally:
self._lock.release()
r = self._check_serials()
assert r is None or len(r) == 0, "unhandled serialnos: %s" % r
except:
if finished:
# The server successfully committed. If we get a failure
# here, our own state will be in question, so reconnect.
self._connection.close()
raise
self.end_transaction()
finally:
self._load_lock.release()
self._iterator_gc()
def _update_cache(self, tid):
"""Internal helper to handle objects modified by a transaction.
This iterates over the objects in the transaction buffer and
update or invalidate the cache.
"""
# Must be called with _lock already acquired.
# Not sure why _update_cache() would be called on a closed storage.
if self._cache is None:
return
for oid, _ in self._seriald.iteritems():
self._cache.invalidate(oid, tid)
for oid, data in self._tbuf:
# If data is None, we just invalidate.
if data is not None:
s = self._seriald[oid]
if s != ResolvedSerial:
assert s == tid, (s, tid)
self._cache.store(oid, s, None, data)
else:
# object deletion
self._cache.invalidate(oid, tid)
if self.fshelper is not None:
blobs = self._tbuf.blobs
had_blobs = False
while blobs:
oid, blobfilename = blobs.pop()
self._blob_data_bytes_loaded += os.stat(blobfilename).st_size
targetpath = self.fshelper.getPathForOID(oid, create=True)
target_blob_file_name = self.fshelper.getBlobFilename(oid, tid)
lock = _lock_blob(target_blob_file_name)
try:
ZODB.blob.rename_or_copy_blob(
blobfilename,
target_blob_file_name,
)
finally:
lock.close()
had_blobs = True
if had_blobs:
self._check_blob_size(self._blob_data_bytes_loaded)
self._cache.setLastTid(tid)
self._tbuf.clear()
def undo(self, trans_id, txn):
"""Storage API: undo a transaction.
This is executed in a transactional context. It has no effect
until the transaction is committed. It can be undone itself.
Zope uses this to implement undo unless it is not supported by
a storage.
"""
self._check_trans(txn)
self._server.undoa(trans_id, id(txn))
def undoInfo(self, first=0, last=-20, specification=None):
"""Storage API: return undo information."""
return self._server.undoInfo(first, last, specification)
def undoLog(self, first=0, last=-20, filter=None):
"""Storage API: return a sequence of TransactionDescription objects.
The filter argument should be None or left unspecified, since
it is impossible to pass the filter function to the server to
be executed there. If filter is not None, an empty sequence
is returned.
"""
if filter is not None:
return []
return self._server.undoLog(first, last)
# Recovery support
def copyTransactionsFrom(self, other, verbose=0):
"""Copy transactions from another storage.
This is typically used for converting data from one storage to
another. `other` must have an .iterator() method.
"""
ZODB.BaseStorage.copy(other, self, verbose)
def restore(self, oid, serial, data, version, prev_txn, transaction):
"""Write data already committed in a separate database."""
assert not version
self._check_trans(transaction)
self._server.restorea(oid, serial, data, prev_txn, id(transaction))
# Don't update the transaction buffer, because current data are
# unaffected.
return self._check_serials()
# Below are methods invoked by the StorageServer
def serialnos(self, args):
"""Server callback to pass a list of changed (oid, serial) pairs."""
self._serials.extend(args)
def info(self, dict):
"""Server callback to update the info dictionary."""
self._info.update(dict)
def verify_cache(self, server):
"""Internal routine called to verify the cache.
The return value (indicating which path we took) is used by
the test suite.
"""
self._pending_server = server
# setup tempfile to hold zeoVerify results and interim
# invalidation results
self._tfile = tempfile.TemporaryFile(suffix=".inv")
self._pickler = cPickle.Pickler(self._tfile, 1)
self._pickler.fast = 1 # Don't use the memo
if self._connection.peer_protocol_version < 'Z309':
client = ClientStorage308Adapter(self)
else:
client = self
# allow incoming invalidations:
self._connection.register_object(client)
# If verify_cache() finishes the cache verification process,
# it should set self._server. If it goes through full cache
# verification, then endVerify() should self._server.
server_tid = server.lastTransaction()
if not self._cache:
logger.info("%s No verification necessary -- empty cache",
self.__name__)
if server_tid != utils.z64:
self._cache.setLastTid(server_tid)
self.finish_verification()
return "empty cache"
cache_tid = self._cache.getLastTid()
if cache_tid != utils.z64:
if server_tid == cache_tid:
logger.info(
"%s No verification necessary"
" (cache_tid up-to-date %r)",
self.__name__, server_tid)
self.finish_verification()
return "no verification"
elif server_tid < cache_tid:
message = ("%s Client has seen newer transactions than server!"
% self.__name__)
logger.critical(message)
raise ClientStorageError(message)
# log some hints about last transaction
logger.info("%s last inval tid: %r %s\n",
self.__name__, cache_tid,
tid2time(cache_tid))
logger.info("%s last transaction: %r %s",
self.__name__, server_tid,
server_tid and tid2time(server_tid))
pair = server.getInvalidations(cache_tid)
if pair is not None:
logger.info("%s Recovering %d invalidations",
self.__name__, len(pair[1]))
self.finish_verification(pair)
return "quick verification"
elif server_tid != utils.z64:
# Hm, to have gotten here, the cache is non-empty, but
# it has no last tid. This doesn't seem like good situation.
# We'll have to verify the cache, if we're willing.
self._cache.setLastTid(server_tid)
ZODB.event.notify(ZEO.interfaces.StaleCache(self))
# From this point on, we do not have complete information about
# the missed transactions. The reason is that cache
# verification only checks objects in the client cache and
# there may be objects in the object caches that aren't in the
# client cach that would need verification too. We avoid that
# problem by just invalidating the objects in the object caches.
if self._db is not None:
self._db.invalidateCache()
if self._cache and self._drop_cache_rather_verify:
logger.critical("%s dropping stale cache", self.__name__)
self._cache.clear()
if server_tid:
self._cache.setLastTid(server_tid)
self.finish_verification()
return "cache dropped"
logger.info("%s Verifying cache", self.__name__)
for oid, tid in self._cache.contents():
server.verify(oid, tid)
server.endZeoVerify()
return "full verification"
def invalidateVerify(self, oid):
"""Server callback to invalidate an oid pair.
This is called as part of cache validation.
"""
# Invalidation as result of verify_cache().
# Queue an invalidate for the end the verification procedure.
if self._pickler is None:
# This should never happen.
logger.error("%s invalidateVerify with no _pickler", self.__name__)
return
self._pickler.dump((None, [oid]))
def endVerify(self):
"""Server callback to signal end of cache validation."""
logger.info("%s endVerify finishing", self.__name__)
self.finish_verification()
logger.info("%s endVerify finished", self.__name__)
def finish_verification(self, catch_up=None):
self._lock.acquire()
try:
if catch_up:
# process catch-up invalidations
self._process_invalidations(*catch_up)
if self._pickler is None:
return
# write end-of-data marker
self._pickler.dump((None, None))
self._pickler = None
self._tfile.seek(0)
unpickler = cPickle.Unpickler(self._tfile)
min_tid = self._cache.getLastTid()
while 1:
tid, oids = unpickler.load()
logger.debug('pickled inval %r %r', tid, min_tid)
if oids is None:
break
if ((tid is None)
or (min_tid is None)
or (tid > min_tid)
):
self._process_invalidations(tid, oids)
self._tfile.close()
self._tfile = None
finally:
self._lock.release()
self._server = self._pending_server
self._ready.set()
self._pending_server = None
def invalidateTransaction(self, tid, oids):
"""Server callback: Invalidate objects modified by tid."""
self._lock.acquire()
try:
if self._pickler is not None:
logger.debug(
"%s Transactional invalidation during cache verification",
self.__name__)
self._pickler.dump((tid, oids))
else:
self._process_invalidations(tid, oids)
finally:
self._lock.release()
def _process_invalidations(self, tid, oids):
for oid in oids:
if oid == self._load_oid:
self._load_status = 0
self._cache.invalidate(oid, tid)
self._cache.setLastTid(tid)
if self._db is not None:
self._db.invalidate(tid, oids)
# The following are for compatibility with protocol version 2.0.0
def invalidateTrans(self, oids):
return self.invalidateTransaction(None, oids)
invalidate = invalidateVerify
end = endVerify
Invalidate = invalidateTrans
# IStorageIteration
def iterator(self, start=None, stop=None):
"""Return an IStorageTransactionInformation iterator."""
# iids are "iterator IDs" that can be used to query an iterator whose
# status is held on the server.
iid = self._server.iterator_start(start, stop)
return self._setup_iterator(TransactionIterator, iid)
def _setup_iterator(self, factory, iid, *args):
self._iterators[iid] = iterator = factory(self, iid, *args)
self._iterator_ids.add(iid)
return iterator
def _forget_iterator(self, iid):
self._iterators.pop(iid, None)
self._iterator_ids.remove(iid)
def _iterator_gc(self, disconnected=False):
if not self._iterator_ids:
return
if disconnected:
for i in self._iterators.values():
i._iid = -1
self._iterators.clear()
self._iterator_ids.clear()
return
iids = self._iterator_ids - set(self._iterators)
if iids:
try:
self._server.iterator_gc(list(iids))
except ClientDisconnected:
# If we get disconnected, all of the iterators on the
# server are thrown away. We should clear ours too:
return self._iterator_gc(True)
self._iterator_ids -= iids
def server_status(self):
return self._server.server_status()
class TransactionIterator(object):
def __init__(self, storage, iid, *args):
self._storage = storage
self._iid = iid
self._ended = False
def __iter__(self):
return self
def next(self):
if self._ended:
raise StopIteration()
if self._iid < 0:
raise ClientDisconnected("Disconnected iterator")
tx_data = self._storage._server.iterator_next(self._iid)
if tx_data is None:
# The iterator is exhausted, and the server has already
# disposed it.
self._ended = True
self._storage._forget_iterator(self._iid)
raise StopIteration()
return ClientStorageTransactionInformation(
self._storage, self, *tx_data)
class ClientStorageTransactionInformation(ZODB.BaseStorage.TransactionRecord):
def __init__(self, storage, txiter, tid, status, user, description,
extension):
self._storage = storage
self._txiter = txiter
self._completed = False
self._riid = None
self.tid = tid
self.status = status
self.user = user
self.description = description
self.extension = extension
def __iter__(self):
riid = self._storage._server.iterator_record_start(self._txiter._iid,
self.tid)
return self._storage._setup_iterator(RecordIterator, riid)
class RecordIterator(object):
def __init__(self, storage, riid):
self._riid = riid
self._completed = False
self._storage = storage
def __iter__(self):
return self
def next(self):
if self._completed:
# We finished iteration once already and the server can't know
# about the iteration anymore.
raise StopIteration()
item = self._storage._server.iterator_record_next(self._riid)
if item is None:
# The iterator is exhausted, and the server has already
# disposed it.
self._completed = True
raise StopIteration()
return ZODB.BaseStorage.DataRecord(*item)
class ClientStorage308Adapter:
def __init__(self, client):
self.client = client
def invalidateTransaction(self, tid, args):
self.client.invalidateTransaction(tid, [arg[0] for arg in args])
def invalidateVerify(self, arg):
self.client.invalidateVerify(arg[0])
def __getattr__(self, name):
return getattr(self.client, name)
class BlobCacheLayout(object):
size = 997
def oid_to_path(self, oid):
return str(utils.u64(oid) % self.size)
def getBlobFilePath(self, oid, tid):
base, rem = divmod(utils.u64(oid), self.size)
return os.path.join(
str(rem),
"%s.%s%s" % (base, tid.encode('hex'), ZODB.blob.BLOB_SUFFIX)
)
def _accessed(filename):
try:
os.utime(filename, (time.time(), os.stat(filename).st_mtime))
except OSError:
pass # We tried. :)
return filename
cache_file_name = re.compile(r'\d+$').match
def _check_blob_cache_size(blob_dir, target):
logger = logging.getLogger(__name__+'.check_blob_cache')
layout = open(os.path.join(blob_dir, ZODB.blob.LAYOUT_MARKER)
).read().strip()
if not layout == 'zeocache':
logger.critical("Invalid blob directory layout %s", layout)
raise ValueError("Invalid blob directory layout", layout)
attempt_path = os.path.join(blob_dir, 'check_size.attempt')
try:
check_lock = zc.lockfile.LockFile(
os.path.join(blob_dir, 'check_size.lock'))
except zc.lockfile.LockError:
try:
time.sleep(1)
check_lock = zc.lockfile.LockFile(
os.path.join(blob_dir, 'check_size.lock'))
except zc.lockfile.LockError:
# Someone is already cleaning up, so don't bother
logger.debug("%s Another thread is checking the blob cache size.",
thread.get_ident())
open(attempt_path, 'w').close() # Mark that we tried
return
logger.debug("%s Checking blob cache size. (target: %s)",
thread.get_ident(), target)
try:
while 1:
size = 0
blob_suffix = ZODB.blob.BLOB_SUFFIX
files_by_atime = BTrees.OOBTree.BTree()
for dirname in os.listdir(blob_dir):
if not cache_file_name(dirname):
continue
base = os.path.join(blob_dir, dirname)
if not os.path.isdir(base):
continue
for file_name in os.listdir(base):
if not file_name.endswith(blob_suffix):
continue
file_path = os.path.join(base, file_name)
if not os.path.isfile(file_path):
continue
stat = os.stat(file_path)
size += stat.st_size
t = stat.st_atime
if t not in files_by_atime:
files_by_atime[t] = []
files_by_atime[t].append(os.path.join(dirname, file_name))
logger.debug("%s blob cache size: %s", thread.get_ident(), size)
if size <= target:
if os.path.isfile(attempt_path):
try:
os.remove(attempt_path)
except OSError:
pass # Sigh, windows
continue
logger.debug("%s -->", thread.get_ident())
break
while size > target and files_by_atime:
for file_name in files_by_atime.pop(files_by_atime.minKey()):
file_name = os.path.join(blob_dir, file_name)
lockfilename = os.path.join(os.path.dirname(file_name),
'.lock')
try:
lock = zc.lockfile.LockFile(lockfilename)
except zc.lockfile.LockError:
logger.debug("%s Skipping locked %s",
thread.get_ident(),
os.path.basename(file_name))
continue # In use, skip
try:
fsize = os.stat(file_name).st_size
try:
ZODB.blob.remove_committed(file_name)
except OSError, v:
pass # probably open on windows
else:
size -= fsize
finally:
lock.close()
if size <= target:
break
logger.debug("%s reduced blob cache size: %s",
thread.get_ident(), size)
finally:
check_lock.close()
def check_blob_size_script(args=None):
if args is None:
args = sys.argv[1:]
blob_dir, target = args
_check_blob_cache_size(blob_dir, int(target))
def _lock_blob(path):
lockfilename = os.path.join(os.path.dirname(path), '.lock')
n = 0
while 1:
try:
return zc.lockfile.LockFile(lockfilename)
except zc.lockfile.LockError:
time.sleep(0.01)
n += 1
if n > 60000:
raise
else:
break
src/ZEO/Exceptions.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Exceptions for ZEO."""
from ZODB.POSException import StorageError
class ClientStorageError(StorageError):
"""An error occurred in the ZEO Client Storage."""
class UnrecognizedResult(ClientStorageError):
"""A server call returned an unrecognized result."""
class ClientDisconnected(ClientStorageError):
"""The database storage is disconnected from the storage."""
class AuthError(StorageError):
"""The client provided invalid authentication credentials."""
src/ZEO/ServerStub.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""RPC stubs for interface exported by StorageServer."""
import time
from ZODB.utils import z64
##
# ZEO storage server.
# <p>
# Remote method calls can be synchronous or asynchronous. If the call
# is synchronous, the client thread blocks until the call returns. A
# single client can only have one synchronous request outstanding. If
# several threads share a single client, threads other than the caller
# will block only if the attempt to make another synchronous call.
# An asynchronous call does not cause the client thread to block. An
# exception raised by an asynchronous method is logged on the server,
# but is not returned to the client.
class StorageServer:
"""An RPC stub class for the interface exported by ClientStorage.
This is the interface presented by the StorageServer to the
ClientStorage; i.e. the ClientStorage calls these methods and they
are executed in the StorageServer.
See the StorageServer module for documentation on these methods,
with the exception of _update(), which is documented here.
"""
def __init__(self, rpc):
"""Constructor.
The argument is a connection: an instance of the
zrpc.connection.Connection class.
"""
self.rpc = rpc
def extensionMethod(self, name):
return ExtensionMethodWrapper(self.rpc, name).call
##
# Register current connection with a storage and a mode.
# In effect, it is like an open call.
# @param storage_name a string naming the storage. This argument
# is primarily for backwards compatibility with servers
# that supported multiple storages.
# @param read_only boolean
# @exception ValueError unknown storage_name or already registered
# @exception ReadOnlyError storage is read-only and a read-write
# connectio was requested
def register(self, storage_name, read_only):
self.rpc.call('register', storage_name, read_only)
##
# Return dictionary of meta-data about the storage.
# @defreturn dict
def get_info(self):
return self.rpc.call('get_info')
##
# Check whether the server requires authentication. Returns
# the name of the protocol.
# @defreturn string
def getAuthProtocol(self):
return self.rpc.call('getAuthProtocol')
##
# Return id of the last committed transaction
# @defreturn string
def lastTransaction(self):
# Not in protocol version 2.0.0; see __init__()
return self.rpc.call('lastTransaction') or z64
##
# Return invalidations for all transactions after tid.
# @param tid transaction id
# @defreturn 2-tuple, (tid, list)
# @return tuple containing the last committed transaction
# and a list of oids that were invalidated. Returns
# None and an empty list if the server does not have
# the list of oids available.
def getInvalidations(self, tid):
# Not in protocol version 2.0.0; see __init__()
return self.rpc.call('getInvalidations', tid)
##
# Check whether a serial number is current for oid.
# If the serial number is not current, the
# server will make an asynchronous invalidateVerify() call.
# @param oid object id
# @param s serial number
# @defreturn async
def zeoVerify(self, oid, s):
self.rpc.callAsync('zeoVerify', oid, s)
##
# Check whether current serial number is valid for oid.
# If the serial number is not current, the server will make an
# asynchronous invalidateVerify() call.
# @param oid object id
# @param serial client's current serial number
# @defreturn async
def verify(self, oid, serial):
self.rpc.callAsync('verify', oid, serial)
##
# Signal to the server that cache verification is done.
# @defreturn async
def endZeoVerify(self):
self.rpc.callAsync('endZeoVerify')
##
# Generate a new set of oids.
# @param n number of new oids to return
# @defreturn list
# @return list of oids
def new_oids(self, n=None):
if n is None:
return self.rpc.call('new_oids')
else:
return self.rpc.call('new_oids', n)
##
# Pack the storage.
# @param t pack time
# @param wait optional, boolean. If true, the call will not
# return until the pack is complete.
def pack(self, t, wait=None):
if wait is None:
self.rpc.call('pack', t)
else:
self.rpc.call('pack', t, wait)
##
# Return current data for oid.
# @param oid object id
# @defreturn 2-tuple
# @return 2-tuple, current non-version data, serial number
# @exception KeyError if oid is not found
def zeoLoad(self, oid):
return self.rpc.call('zeoLoad', oid)[:2]
##
# Return current data for oid, and the tid of the
# transaction that wrote the most recent revision.
# @param oid object id
# @defreturn 2-tuple
# @return data, transaction id
# @exception KeyError if oid is not found
def loadEx(self, oid):
return self.rpc.call("loadEx", oid)
##
# Return non-current data along with transaction ids that identify
# the lifetime of the specific revision.
# @param oid object id
# @param tid a transaction id that provides an upper bound on
# the lifetime of the revision. That is, loadBefore
# returns the revision that was current before tid committed.
# @defreturn 4-tuple
# @return data, serial numbr, start transaction id, end transaction id
def loadBefore(self, oid, tid):
return self.rpc.call("loadBefore", oid, tid)
##
# Storage new revision of oid.
# @param oid object id
# @param serial serial number that this transaction read
# @param data new data record for oid
# @param id id of current transaction
# @defreturn async
def storea(self, oid, serial, data, id):
self.rpc.callAsync('storea', oid, serial, data, id)
def checkCurrentSerialInTransaction(self, oid, serial, id):
self.rpc.callAsync('checkCurrentSerialInTransaction', oid, serial, id)
def restorea(self, oid, serial, data, prev_txn, id):
self.rpc.callAsync('restorea', oid, serial, data, prev_txn, id)
def storeBlob(self, oid, serial, data, blobfilename, txn):
# Store a blob to the server. We don't want to real all of
# the data into memory, so we use a message iterator. This
# allows us to read the blob data as needed.
def store():
yield ('storeBlobStart', ())
f = open(blobfilename, 'rb')
while 1:
chunk = f.read(59000)
if not chunk:
break
yield ('storeBlobChunk', (chunk, ))
f.close()
yield ('storeBlobEnd', (oid, serial, data, id(txn)))
self.rpc.callAsyncIterator(store())
def storeBlobShared(self, oid, serial, data, filename, id):
self.rpc.callAsync('storeBlobShared', oid, serial, data, filename, id)
def deleteObject(self, oid, serial, id):
self.rpc.callAsync('deleteObject', oid, serial, id)
##
# Start two-phase commit for a transaction
# @param id id used by client to identify current transaction. The
# only purpose of this argument is to distinguish among multiple
# threads using a single ClientStorage.
# @param user name of user committing transaction (can be "")
# @param description string containing transaction metadata (can be "")
# @param ext dictionary of extended metadata (?)
# @param tid optional explicit tid to pass to underlying storage
# @param status optional status character, e.g "p" for pack
# @defreturn async
def tpc_begin(self, id, user, descr, ext, tid, status):
self.rpc.callAsync('tpc_begin', id, user, descr, ext, tid, status)
def vote(self, trans_id):
return self.rpc.call('vote', trans_id)
def tpc_finish(self, id):
return self.rpc.call('tpc_finish', id)
def tpc_abort(self, id):
self.rpc.call('tpc_abort', id)
def history(self, oid, length=None):
if length is None:
return self.rpc.call('history', oid)
else:
return self.rpc.call('history', oid, length)
def record_iternext(self, next):
return self.rpc.call('record_iternext', next)
def sendBlob(self, oid, serial):
return self.rpc.call('sendBlob', oid, serial)
def getTid(self, oid):
return self.rpc.call('getTid', oid)
def loadSerial(self, oid, serial):
return self.rpc.call('loadSerial', oid, serial)
def new_oid(self):
return self.rpc.call('new_oid')
def undoa(self, trans_id, trans):
self.rpc.callAsync('undoa', trans_id, trans)
def undoLog(self, first, last):
return self.rpc.call('undoLog', first, last)
def undoInfo(self, first, last, spec):
return self.rpc.call('undoInfo', first, last, spec)
def iterator_start(self, start, stop):
return self.rpc.call('iterator_start', start, stop)
def iterator_next(self, iid):
return self.rpc.call('iterator_next', iid)
def iterator_record_start(self, txn_iid, tid):
return self.rpc.call('iterator_record_start', txn_iid, tid)
def iterator_record_next(self, iid):
return self.rpc.call('iterator_record_next', iid)
def iterator_gc(self, iids):
return self.rpc.callAsync('iterator_gc', iids)
def server_status(self):
return self.rpc.call("server_status")
def set_client_label(self, label):
return self.rpc.callAsync('set_client_label', label)
class StorageServer308(StorageServer):
def __init__(self, rpc):
if rpc.peer_protocol_version == 'Z200':
self.lastTransaction = lambda: z64
self.getInvalidations = lambda tid: None
self.getAuthProtocol = lambda: None
StorageServer.__init__(self, rpc)
def history(self, oid, length=None):
if length is None:
return self.rpc.call('history', oid, '')
else:
return self.rpc.call('history', oid, '', length)
def getInvalidations(self, tid):
# Not in protocol version 2.0.0; see __init__()
result = self.rpc.call('getInvalidations', tid)
if result is not None:
result = result[0], [oid for (oid, version) in result[1]]
return result
def verify(self, oid, serial):
self.rpc.callAsync('verify', oid, '', serial)
def loadEx(self, oid):
return self.rpc.call("loadEx", oid, '')[:2]
def storea(self, oid, serial, data, id):
self.rpc.callAsync('storea', oid, serial, data, '', id)
def storeBlob(self, oid, serial, data, blobfilename, txn):
# Store a blob to the server. We don't want to real all of
# the data into memory, so we use a message iterator. This
# allows us to read the blob data as needed.
def store():
yield ('storeBlobStart', ())
f = open(blobfilename, 'rb')
while 1:
chunk = f.read(59000)
if not chunk:
break
yield ('storeBlobChunk', (chunk, ))
f.close()
yield ('storeBlobEnd', (oid, serial, data, '', id(txn)))
self.rpc.callAsyncIterator(store())
def storeBlobShared(self, oid, serial, data, filename, id):
self.rpc.callAsync('storeBlobShared', oid, serial, data, filename,
'', id)
def zeoVerify(self, oid, s):
self.rpc.callAsync('zeoVerify', oid, s, None)
def iterator_start(self, start, stop):
raise NotImplementedError
def iterator_next(self, iid):
raise NotImplementedError
def iterator_record_start(self, txn_iid, tid):
raise NotImplementedError
def iterator_record_next(self, iid):
raise NotImplementedError
def iterator_gc(self, iids):
raise NotImplementedError
def stub(client, connection):
start = time.time()
# Wait until we know what version the other side is using.
while connection.peer_protocol_version is None:
if time.time()-start > 10:
raise ValueError("Timeout waiting for protocol handshake")
time.sleep(0.1)
if connection.peer_protocol_version < 'Z309':
return StorageServer308(connection)
return StorageServer(connection)
class ExtensionMethodWrapper:
def __init__(self, rpc, name):
self.rpc = rpc
self.name = name
def call(self, *a, **kwa):
return self.rpc.call(self.name, *a, **kwa)
src/ZEO/StorageServer.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The StorageServer class and the exception that it may raise.
This server acts as a front-end for one or more real storages, like
file storage or Berkeley storage.
TODO: Need some basic access control-- a declaration of the methods
exported for invocation by the server.
"""
from __future__ import with_statement
from ZEO.Exceptions import AuthError
from ZEO.monitor import StorageStats, StatsServer
from ZEO.zrpc.connection import ManagedServerConnection, Delay, MTDelay, Result
from ZEO.zrpc.server import Dispatcher
from ZODB.ConflictResolution import ResolvedSerial
from ZODB.loglevels import BLATHER
from ZODB.POSException import StorageError, StorageTransactionError
from ZODB.POSException import TransactionError, ReadOnlyError, ConflictError
from ZODB.serialize import referencesf
from ZODB.utils import oid_repr, p64, u64, z64
import asyncore
import cPickle
import itertools
import logging
import os
import sys
import tempfile
import threading
import time
import transaction
import warnings
import ZEO.zrpc.error
import ZODB.blob
import ZODB.event
import ZODB.serialize
import ZODB.TimeStamp
import zope.interface
logger = logging.getLogger('ZEO.StorageServer')
def log(message, level=logging.INFO, label='', exc_info=False):
"""Internal helper to log a message."""
if label:
message = "(%s) %s" % (label, message)
logger.log(level, message, exc_info=exc_info)
class StorageServerError(StorageError):
"""Error reported when an unpicklable exception is raised."""
class ZEOStorage:
"""Proxy to underlying storage for a single remote client."""
# A list of extension methods. A subclass with extra methods
# should override.
extensions = []
def __init__(self, server, read_only=0, auth_realm=None):
self.server = server
# timeout and stats will be initialized in register()
self.stats = None
self.connection = None
self.client = None
self.storage = None
self.storage_id = "uninitialized"
self.transaction = None
self.read_only = read_only
self.log_label = 'unconnected'
self.locked = False # Don't have storage lock
self.verifying = 0
self.store_failed = 0
self.authenticated = 0
self.auth_realm = auth_realm
self.blob_tempfile = None
# The authentication protocol may define extra methods.
self._extensions = {}
for func in self.extensions:
self._extensions[func.func_name] = None
self._iterators = {}
self._iterator_ids = itertools.count()
# Stores the last item that was handed out for a
# transaction iterator.
self._txn_iterators_last = {}
def _finish_auth(self, authenticated):
if not self.auth_realm:
return 1
self.authenticated = authenticated
return authenticated
def set_database(self, database):
self.database = database
def notifyConnected(self, conn):
self.connection = conn
assert conn.peer_protocol_version is not None
if conn.peer_protocol_version < 'Z309':
self.client = ClientStub308(conn)
conn.register_object(ZEOStorage308Adapter(self))
else:
self.client = ClientStub(conn)
self.log_label = _addr_label(conn.addr)
def notifyDisconnected(self):
# When this storage closes, we must ensure that it aborts
# any pending transaction.
if self.transaction is not None:
self.log("disconnected during %s transaction"
% (self.locked and 'locked' or 'unlocked'))
self.tpc_abort(self.transaction.id)
else:
self.log("disconnected")
self.connection = None
def __repr__(self):
tid = self.transaction and repr(self.transaction.id)
if self.storage:
stid = (self.tpc_transaction() and
repr(self.tpc_transaction().id))
else:
stid = None
name = self.__class__.__name__
return "<%s %X trans=%s s_trans=%s>" % (name, id(self), tid, stid)
def log(self, msg, level=logging.INFO, exc_info=False):
log(msg, level=level, label=self.log_label, exc_info=exc_info)
def setup_delegation(self):
"""Delegate several methods to the storage
"""
# Called from register
storage = self.storage
info = self.get_info()
if not info['supportsUndo']:
self.undoLog = self.undoInfo = lambda *a,**k: ()
self.getTid = storage.getTid
self.load = storage.load
self.loadSerial = storage.loadSerial
record_iternext = getattr(storage, 'record_iternext', None)
if record_iternext is not None:
self.record_iternext = record_iternext
try:
fn = storage.getExtensionMethods
except AttributeError:
pass # no extension methods
else:
d = fn()
self._extensions.update(d)
for name in d:
assert not hasattr(self, name)
setattr(self, name, getattr(storage, name))
self.lastTransaction = storage.lastTransaction
try:
self.tpc_transaction = storage.tpc_transaction
except AttributeError:
if hasattr(storage, '_transaction'):
log("Storage %r doesn't have a tpc_transaction method.\n"
"See ZEO.interfaces.IServeable."
"Falling back to using _transaction attribute, which\n."
"is icky.",
logging.ERROR)
self.tpc_transaction = lambda : storage._transaction
else:
raise
def history(self,tid,size=1):
# This caters for storages which still accept
# a version parameter.
return self.storage.history(tid,size=size)
def _check_tid(self, tid, exc=None):
if self.read_only:
raise ReadOnlyError()
if self.transaction is None:
caller = sys._getframe().f_back.f_code.co_name
self.log("no current transaction: %s()" % caller,
level=logging.WARNING)
if exc is not None:
raise exc(None, tid)
else:
return 0
if self.transaction.id != tid:
caller = sys._getframe().f_back.f_code.co_name
self.log("%s(%s) invalid; current transaction = %s" %
(caller, repr(tid), repr(self.transaction.id)),
logging.WARNING)
if exc is not None:
raise exc(self.transaction.id, tid)
else:
return 0
return 1
def getAuthProtocol(self):
"""Return string specifying name of authentication module to use.
The module name should be auth_%s where %s is auth_protocol."""
protocol = self.server.auth_protocol
if not protocol or protocol == 'none':
return None
return protocol
def register(self, storage_id, read_only):
"""Select the storage that this client will use
This method must be the first one called by the client.
For authenticated storages this method will be called by the client
immediately after authentication is finished.
"""
if self.auth_realm and not self.authenticated:
raise AuthError("Client was never authenticated with server!")
if self.storage is not None:
self.log("duplicate register() call")
raise ValueError("duplicate register() call")
storage = self.server.storages.get(storage_id)
if storage is None:
self.log("unknown storage_id: %s" % storage_id)
raise ValueError("unknown storage: %s" % storage_id)
if not read_only and (self.read_only or storage.isReadOnly()):
raise ReadOnlyError()
self.read_only = self.read_only or read_only
self.storage_id = storage_id
self.storage = storage
self.setup_delegation()
self.stats = self.server.register_connection(storage_id, self)
def get_info(self):
storage = self.storage
supportsUndo = (getattr(storage, 'supportsUndo', lambda : False)()
and self.connection.peer_protocol_version >= 'Z310')
# Communicate the backend storage interfaces to the client
storage_provides = zope.interface.providedBy(storage)
interfaces = []
for candidate in storage_provides.__iro__:
interfaces.append((candidate.__module__, candidate.__name__))
return {'length': len(storage),
'size': storage.getSize(),
'name': storage.getName(),
'supportsUndo': supportsUndo,
'extensionMethods': self.getExtensionMethods(),
'supports_record_iternext': hasattr(self, 'record_iternext'),
'interfaces': tuple(interfaces),
}
def get_size_info(self):
return {'length': len(self.storage),
'size': self.storage.getSize(),
}
def getExtensionMethods(self):
return self._extensions
def loadEx(self, oid):
self.stats.loads += 1
return self.storage.load(oid, '')
def loadBefore(self, oid, tid):
self.stats.loads += 1
return self.storage.loadBefore(oid, tid)
def getInvalidations(self, tid):
invtid, invlist = self.server.get_invalidations(self.storage_id, tid)
if invtid is None:
return None
self.log("Return %d invalidations up to tid %s"
% (len(invlist), u64(invtid)))
return invtid, invlist
def verify(self, oid, tid):
try:
t = self.getTid(oid)
except KeyError:
self.client.invalidateVerify(oid)
else:
if tid != t:
self.client.invalidateVerify(oid)
def zeoVerify(self, oid, s):
if not self.verifying:
self.verifying = 1
self.stats.verifying_clients += 1
try:
os = self.getTid(oid)
except KeyError:
self.client.invalidateVerify((oid, ''))
# It's not clear what we should do now. The KeyError
# could be caused by an object uncreation, in which case
# invalidation is right. It could be an application bug
# that left a dangling reference, in which case it's bad.
else:
if s != os:
self.client.invalidateVerify((oid, ''))
def endZeoVerify(self):
if self.verifying:
self.stats.verifying_clients -= 1
self.verifying = 0
self.client.endVerify()
def pack(self, time, wait=1):
# Yes, you can pack a read-only server or storage!
if wait:
return run_in_thread(self._pack_impl, time)
else:
# If the client isn't waiting for a reply, start a thread
# and forget about it.
t = threading.Thread(target=self._pack_impl, args=(time,))
t.start()
return None
def _pack_impl(self, time):
self.log("pack(time=%s) started..." % repr(time))
self.storage.pack(time, referencesf)
self.log("pack(time=%s) complete" % repr(time))
# Broadcast new size statistics
self.server.invalidate(0, self.storage_id, None,
(), self.get_size_info())
def new_oids(self, n=100):
"""Return a sequence of n new oids, where n defaults to 100"""
n = min(n, 100)
if self.read_only:
raise ReadOnlyError()
if n <= 0:
n = 1
return [self.storage.new_oid() for i in range(n)]
# undoLog and undoInfo are potentially slow methods
def undoInfo(self, first, last, spec):
return run_in_thread(self.storage.undoInfo, first, last, spec)
def undoLog(self, first, last):
return run_in_thread(self.storage.undoLog, first, last)
def tpc_begin(self, id, user, description, ext, tid=None, status=" "):
if self.read_only:
raise ReadOnlyError()
if self.transaction is not None:
if self.transaction.id == id:
self.log("duplicate tpc_begin(%s)" % repr(id))
return
else:
raise StorageTransactionError("Multiple simultaneous tpc_begin"
" requests from one client.")
t = transaction.Transaction()
t.id = id
t.user = user
t.description = description
t._extension = ext
self.serials = []
self.invalidated = []
self.txnlog = CommitLog()
self.blob_log = []
self.tid = tid
self.status = status
self.store_failed = 0
self.stats.active_txns += 1
# Assign the transaction attribute last. This is so we don't
# think we've entered TPC until everything is set. Why?
# Because if we have an error after this, the server will
# think it is in TPC and the client will think it isn't. At
# that point, the client will keep trying to enter TPC and
# server won't let it. Errors *after* the tpc_begin call will
# cause the client to abort the transaction.
# (Also see https://bugs.launchpad.net/zodb/+bug/374737.)
self.transaction = t
def tpc_finish(self, id):
if not self._check_tid(id):
return
assert self.locked, "finished called wo lock"
self.stats.commits += 1
self.storage.tpc_finish(self.transaction, self._invalidate)
# Note that the tid is still current because we still hold the
# commit lock. We'll relinquish it in _clear_transaction.
tid = self.storage.lastTransaction()
# Return the tid, for cache invalidation optimization
return Result(tid, self._clear_transaction)
def _invalidate(self, tid):
if self.invalidated:
self.server.invalidate(self, self.storage_id, tid,
self.invalidated, self.get_size_info())
def tpc_abort(self, tid):
if not self._check_tid(tid):
return
self.stats.aborts += 1
self.storage.tpc_abort(self.transaction)
self._clear_transaction()
def _clear_transaction(self):
# Common code at end of tpc_finish() and tpc_abort()
if self.locked:
self.server.unlock_storage(self)
self.locked = 0
if self.transaction is not None:
self.server.stop_waiting(self)
self.transaction = None
self.stats.active_txns -= 1
if self.txnlog is not None:
self.txnlog.close()
self.txnlog = None
for oid, oldserial, data, blobfilename in self.blob_log:
ZODB.blob.remove_committed(blobfilename)
del self.blob_log
def vote(self, tid):
self._check_tid(tid, exc=StorageTransactionError)
if self.locked or self.server.already_waiting(self):
raise StorageTransactionError(
'Already voting (%s)' % (self.locked and 'locked' or 'waiting')
)
return self._try_to_vote()
def _try_to_vote(self, delay=None):
if self.connection is None:
return # We're disconnected
if delay is not None and delay.sent:
# as a consequence of the unlocking strategy, _try_to_vote
# may be called multiple times for delayed
# transactions. The first call will mark the delay as
# sent. We should skip if the delay was already sent.
return
self.locked, delay = self.server.lock_storage(self, delay)
if self.locked:
try:
self.log(
"Preparing to commit transaction: %d objects, %d bytes"
% (self.txnlog.stores, self.txnlog.size()),
level=BLATHER)
if (self.tid is not None) or (self.status != ' '):
self.storage.tpc_begin(self.transaction,
self.tid, self.status)
else:
self.storage.tpc_begin(self.transaction)
for op, args in self.txnlog:
if not getattr(self, op)(*args):
break
# Blob support
while self.blob_log and not self.store_failed:
oid, oldserial, data, blobfilename = self.blob_log.pop()
self._store(oid, oldserial, data, blobfilename)
if not self.store_failed:
# Only call tpc_vote of no store call failed,
# otherwise the serialnos() call will deliver an
# exception that will be handled by the client in
# its tpc_vote() method.
serials = self.storage.tpc_vote(self.transaction)
if serials:
self.serials.extend(serials)
self.client.serialnos(self.serials)
except Exception:
self.storage.tpc_abort(self.transaction)
self._clear_transaction()
if delay is not None:
delay.error()
else:
raise
else:
if delay is not None:
delay.reply(None)
else:
return None
else:
return delay
def _unlock_callback(self, delay):
connection = self.connection
if connection is None:
self.server.stop_waiting(self)
else:
connection.call_from_thread(self._try_to_vote, delay)
# The public methods of the ZEO client API do not do the real work.
# They defer work until after the storage lock has been acquired.
# Most of the real implementations are in methods beginning with
# an _.
def deleteObject(self, oid, serial, id):
self._check_tid(id, exc=StorageTransactionError)
self.stats.stores += 1
self.txnlog.delete(oid, serial)
def storea(self, oid, serial, data, id):
self._check_tid(id, exc=StorageTransactionError)
self.stats.stores += 1
self.txnlog.store(oid, serial, data)
def checkCurrentSerialInTransaction(self, oid, serial, id):
self._check_tid(id, exc=StorageTransactionError)
self.txnlog.checkread(oid, serial)
def restorea(self, oid, serial, data, prev_txn, id):
self._check_tid(id, exc=StorageTransactionError)
self.stats.stores += 1
self.txnlog.restore(oid, serial, data, prev_txn)
def storeBlobStart(self):
assert self.blob_tempfile is None
self.blob_tempfile = tempfile.mkstemp(
dir=self.storage.temporaryDirectory())
def storeBlobChunk(self, chunk):
os.write(self.blob_tempfile[0], chunk)
def storeBlobEnd(self, oid, serial, data, id):
self._check_tid(id, exc=StorageTransactionError)
assert self.txnlog is not None # effectively not allowed after undo
fd, tempname = self.blob_tempfile
self.blob_tempfile = None
os.close(fd)
self.blob_log.append((oid, serial, data, tempname))
def storeBlobShared(self, oid, serial, data, filename, id):
self._check_tid(id, exc=StorageTransactionError)
assert self.txnlog is not None # effectively not allowed after undo
# Reconstruct the full path from the filename in the OID directory
if (os.path.sep in filename
or not (filename.endswith('.tmp')
or filename[:-1].endswith('.tmp')
)
):
logger.critical(
"We're under attack! (bad filename to storeBlobShared, %r)",
filename)
raise ValueError(filename)
filename = os.path.join(self.storage.fshelper.getPathForOID(oid),
filename)
self.blob_log.append((oid, serial, data, filename))
def sendBlob(self, oid, serial):
self.client.storeBlob(oid, serial, self.storage.loadBlob(oid, serial))
def undo(*a, **k):
raise NotImplementedError
def undoa(self, trans_id, tid):
self._check_tid(tid, exc=StorageTransactionError)
self.txnlog.undo(trans_id)
def _op_error(self, oid, err, op):
self.store_failed = 1
if isinstance(err, ConflictError):
self.stats.conflicts += 1
self.log("conflict error oid=%s msg=%s" %
(oid_repr(oid), str(err)), BLATHER)
if not isinstance(err, TransactionError):
# Unexpected errors are logged and passed to the client
self.log("%s error: %s, %s" % ((op,)+ sys.exc_info()[:2]),
logging.ERROR, exc_info=True)
err = self._marshal_error(err)
# The exception is reported back as newserial for this oid
self.serials.append((oid, err))
def _delete(self, oid, serial):
err = None
try:
self.storage.deleteObject(oid, serial, self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
self._op_error(oid, err, 'delete')
return err is None
def _checkread(self, oid, serial):
err = None
try:
self.storage.checkCurrentSerialInTransaction(
oid, serial, self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
self._op_error(oid, err, 'checkCurrentSerialInTransaction')
return err is None
def _store(self, oid, serial, data, blobfile=None):
err = None
try:
if blobfile is None:
newserial = self.storage.store(
oid, serial, data, '', self.transaction)
else:
newserial = self.storage.storeBlob(
oid, serial, data, blobfile, '', self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
self._op_error(oid, err, 'store')
else:
if serial != "\0\0\0\0\0\0\0\0":
self.invalidated.append(oid)
if isinstance(newserial, str):
newserial = [(oid, newserial)]
for oid, s in newserial or ():
if s == ResolvedSerial:
self.stats.conflicts_resolved += 1
self.log("conflict resolved oid=%s"
% oid_repr(oid), BLATHER)
self.serials.append((oid, s))
return err is None
def _restore(self, oid, serial, data, prev_txn):
err = None
try:
self.storage.restore(oid, serial, data, '', prev_txn,
self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
self._op_error(oid, err, 'restore')
return err is None
def _undo(self, trans_id):
err = None
try:
tid, oids = self.storage.undo(trans_id, self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
self._op_error(z64, err, 'undo')
else:
self.invalidated.extend(oids)
self.serials.extend((oid, ResolvedSerial) for oid in oids)
return err is None
def _marshal_error(self, error):
# Try to pickle the exception. If it can't be pickled,
# the RPC response would fail, so use something that can be pickled.
pickler = cPickle.Pickler()
pickler.fast = 1
try:
pickler.dump(error, 1)
except:
msg = "Couldn't pickle storage exception: %s" % repr(error)
self.log(msg, logging.ERROR)
error = StorageServerError(msg)
return error
# IStorageIteration support
def iterator_start(self, start, stop):
iid = self._iterator_ids.next()
self._iterators[iid] = iter(self.storage.iterator(start, stop))
return iid
def iterator_next(self, iid):
iterator = self._iterators[iid]
try:
info = iterator.next()
except StopIteration:
del self._iterators[iid]
item = None
if iid in self._txn_iterators_last:
del self._txn_iterators_last[iid]
else:
item = (info.tid,
info.status,
info.user,
info.description,
info.extension)
# Keep a reference to the last iterator result to allow starting a
# record iterator off it.
self._txn_iterators_last[iid] = info
return item
def iterator_record_start(self, txn_iid, tid):
record_iid = self._iterator_ids.next()
txn_info = self._txn_iterators_last[txn_iid]
if txn_info.tid != tid:
raise Exception(
'Out-of-order request for record iterator for transaction %r'
% tid)
self._iterators[record_iid] = iter(txn_info)
return record_iid
def iterator_record_next(self, iid):
iterator = self._iterators[iid]
try:
info = iterator.next()
except StopIteration:
del self._iterators[iid]
item = None
else:
item = (info.oid,
info.tid,
info.data,
info.data_txn)
return item
def iterator_gc(self, iids):
for iid in iids:
self._iterators.pop(iid, None)
def server_status(self):
return self.server.server_status(self)
def set_client_label(self, label):
self.log_label = str(label)+' '+_addr_label(self.connection.addr)
class StorageServerDB:
def __init__(self, server, storage_id):
self.server = server
self.storage_id = storage_id
self.references = ZODB.serialize.referencesf
def invalidate(self, tid, oids, version=''):
if version:
raise StorageServerError("Versions aren't supported.")
storage_id = self.storage_id
self.server.invalidate(None, storage_id, tid, oids)
def invalidateCache(self):
self.server._invalidateCache(self.storage_id)
transform_record_data = untransform_record_data = lambda self, data: data
class StorageServer:
"""The server side implementation of ZEO.
The StorageServer is the 'manager' for incoming connections. Each
connection is associated with its own ZEOStorage instance (defined
below). The StorageServer may handle multiple storages; each
ZEOStorage instance only handles a single storage.
"""
# Classes we instantiate. A subclass might override.
DispatcherClass = ZEO.zrpc.server.Dispatcher
ZEOStorageClass = ZEOStorage
ManagedServerConnectionClass = ManagedServerConnection
def __init__(self, addr, storages,
read_only=0,
invalidation_queue_size=100,
invalidation_age=None,
transaction_timeout=None,
monitor_address=None,
auth_protocol=None,
auth_database=None,
auth_realm=None,
):
"""StorageServer constructor.
This is typically invoked from the start.py script.
Arguments (the first two are required and positional):
addr -- the address at which the server should listen. This
can be a tuple (host, port) to signify a TCP/IP connection
or a pathname string to signify a Unix domain socket
connection. A hostname may be a DNS name or a dotted IP
address.
storages -- a dictionary giving the storage(s) to handle. The
keys are the storage names, the values are the storage
instances, typically FileStorage or Berkeley storage
instances. By convention, storage names are typically
strings representing small integers starting at '1'.
read_only -- an optional flag saying whether the server should
operate in read-only mode. Defaults to false. Note that
even if the server is operating in writable mode,
individual storages may still be read-only. But if the
server is in read-only mode, no write operations are
allowed, even if the storages are writable. Note that
pack() is considered a read-only operation.
invalidation_queue_size -- The storage server keeps a queue
of the objects modified by the last N transactions, where
N == invalidation_queue_size. This queue is used to
speed client cache verification when a client disconnects
for a short period of time.
invalidation_age --
If the invalidation queue isn't big enough to support a
quick verification, but the last transaction seen by a
client is younger than the invalidation age, then
invalidations will be computed by iterating over
transactions later than the given transaction.
transaction_timeout -- The maximum amount of time to wait for
a transaction to commit after acquiring the storage lock.
If the transaction takes too long, the client connection
will be closed and the transaction aborted.
monitor_address -- The address at which the monitor server
should listen. If specified, a monitor server is started.
The monitor server provides server statistics in a simple
text format.
auth_protocol -- The name of the authentication protocol to use.
Examples are "digest" and "srp".
auth_database -- The name of the password database filename.
It should be in a format compatible with the authentication
protocol used; for instance, "sha" and "srp" require different
formats.
Note that to implement an authentication protocol, a server
and client authentication mechanism must be implemented in a
auth_* module, which should be stored inside the "auth"
subdirectory. This module may also define a DatabaseClass
variable that should indicate what database should be used
by the authenticator.
"""
self.addr = addr
self.storages = storages
msg = ", ".join(
["%s:%s:%s" % (name, storage.isReadOnly() and "RO" or "RW",
storage.getName())
for name, storage in storages.items()])
log("%s created %s with storages: %s" %
(self.__class__.__name__, read_only and "RO" or "RW", msg))
self._lock = threading.Lock()
self._commit_locks = {}
self._waiting = dict((name, []) for name in storages)
self.read_only = read_only
self.auth_protocol = auth_protocol
self.auth_database = auth_database
self.auth_realm = auth_realm
self.database = None
if auth_protocol:
self._setup_auth(auth_protocol)
# A list, by server, of at most invalidation_queue_size invalidations.
# The list is kept in sorted order with the most recent
# invalidation at the front. The list never has more than
# self.invq_bound elements.
self.invq_bound = invalidation_queue_size
self.invq = {}
for name, storage in storages.items():
self._setup_invq(name, storage)
storage.registerDB(StorageServerDB(self, name))
self.invalidation_age = invalidation_age
self.connections = {}
self.socket_map = {}
self.dispatcher = self.DispatcherClass(
addr, factory=self.new_connection, map=self.socket_map)
if len(self.addr) == 2 and self.addr[1] == 0 and self.addr[0]:
self.addr = self.dispatcher.socket.getsockname()
ZODB.event.notify(
Serving(self, address=self.dispatcher.socket.getsockname()))
self.stats = {}
self.timeouts = {}
for name in self.storages.keys():
self.connections[name] = []
self.stats[name] = StorageStats(self.connections[name])
if transaction_timeout is None:
# An object with no-op methods
timeout = StubTimeoutThread()
else:
timeout = TimeoutThread(transaction_timeout)
timeout.start()
self.timeouts[name] = timeout
if monitor_address:
warnings.warn(
"The monitor server is deprecated. Use the server_status\n"
"ZEO method instead.",
DeprecationWarning)
self.monitor = StatsServer(monitor_address, self.stats)
else:
self.monitor = None
def _setup_invq(self, name, storage):
lastInvalidations = getattr(storage, 'lastInvalidations', None)
if lastInvalidations is None:
# Using None below doesn't look right, but the first
# element in invq is never used. See get_invalidations.
# (If it was used, it would generate an error, which would
# be good. :) Doing this allows clients that were up to
# date when a server was restarted to pick up transactions
# it subsequently missed.
self.invq[name] = [(storage.lastTransaction() or z64, None)]
else:
self.invq[name] = list(lastInvalidations(self.invq_bound))
self.invq[name].reverse()
def _setup_auth(self, protocol):
# Can't be done in global scope, because of cyclic references
from ZEO.auth import get_module
name = self.__class__.__name__
module = get_module(protocol)
if not module:
log("%s: no such an auth protocol: %s" % (name, protocol))
return
storage_class, client, db_class = module
if not storage_class or not issubclass(storage_class, ZEOStorage):
log(("%s: %s isn't a valid protocol, must have a StorageClass" %
(name, protocol)))
self.auth_protocol = None
return
self.ZEOStorageClass = storage_class
log("%s: using auth protocol: %s" % (name, protocol))
# We create a Database instance here for use with the authenticator
# modules. Having one instance allows it to be shared between multiple
# storages, avoiding the need to bloat each with a new authenticator
# Database that would contain the same info, and also avoiding any
# possibly synchronization issues between them.
self.database = db_class(self.auth_database)
if self.database.realm != self.auth_realm:
raise ValueError("password database realm %r "
"does not match storage realm %r"
% (self.database.realm, self.auth_realm))
def new_connection(self, sock, addr):
"""Internal: factory to create a new connection.
This is called by the Dispatcher class in ZEO.zrpc.server
whenever accept() returns a socket for a new incoming
connection.
"""
if self.auth_protocol and self.database:
zstorage = self.ZEOStorageClass(self, self.read_only,
auth_realm=self.auth_realm)
zstorage.set_database(self.database)
else:
zstorage = self.ZEOStorageClass(self, self.read_only)
c = self.ManagedServerConnectionClass(sock, addr, zstorage, self)
log("new connection %s: %s" % (addr, repr(c)))
return c
def register_connection(self, storage_id, conn):
"""Internal: register a connection with a particular storage.
This is called by ZEOStorage.register().
The dictionary self.connections maps each storage name to a
list of current connections for that storage; this information
is needed to handle invalidation. This function updates this
dictionary.
Returns the timeout and stats objects for the appropriate storage.
"""
self.connections[storage_id].append(conn)
return self.stats[storage_id]
def _invalidateCache(self, storage_id):
"""We need to invalidate any caches we have.
This basically means telling our clients to
invalidate/revalidate their caches. We do this by closing them
and making them reconnect.
"""
# This method can be called from foreign threads. We have to
# worry about interaction with the main thread.
# 1. We modify self.invq which is read by get_invalidations
# below. This is why get_invalidations makes a copy of
# self.invq.
# 2. We access connections. There are two dangers:
#
# a. We miss a new connection. This is not a problem because
# if a client connects after we get the list of connections,
# then it will have to read the invalidation queue, which
# has already been reset.
#
# b. A connection is closes while we are iterating. This
# doesn't matter, bacause we can call should_close on a closed
# connection.
# Rebuild invq
self._setup_invq(storage_id, self.storages[storage_id])
# Make a copy since we are going to be mutating the
# connections indirectoy by closing them. We don't care about
# later transactions since they will have to validate their
# caches anyway.
for p in self.connections[storage_id][:]:
try:
p.connection.should_close()
p.connection.trigger.pull_trigger()
except ZEO.zrpc.error.DisconnectedError:
pass
def invalidate(self, conn, storage_id, tid, invalidated=(), info=None):
"""Internal: broadcast info and invalidations to clients.
This is called from several ZEOStorage methods.
invalidated is a sequence of oids.
This can do three different things:
- If the invalidated argument is non-empty, it broadcasts
invalidateTransaction() messages to all clients of the given
storage except the current client (the conn argument).
- If the invalidated argument is empty and the info argument
is a non-empty dictionary, it broadcasts info() messages to
all clients of the given storage, including the current
client.
- If both the invalidated argument and the info argument are
non-empty, it broadcasts invalidateTransaction() messages to all
clients except the current, and sends an info() message to
the current client.
"""
# This method can be called from foreign threads. We have to
# worry about interaction with the main thread.
# 1. We modify self.invq which is read by get_invalidations
# below. This is why get_invalidations makes a copy of
# self.invq.
# 2. We access connections. There are two dangers:
#
# a. We miss a new connection. This is not a problem because
# we are called while the storage lock is held. A new
# connection that tries to read data won't read committed
# data without first recieving an invalidation. Also, if a
# client connects after getting the list of connections,
# then it will have to read the invalidation queue, which
# has been updated to reflect the invalidations.
#
# b. A connection is closes while we are iterating. We'll need
# to cactch and ignore Disconnected errors.
if invalidated:
invq = self.invq[storage_id]
if len(invq) >= self.invq_bound:
invq.pop()
invq.insert(0, (tid, invalidated))
for p in self.connections[storage_id]:
try:
if invalidated and p is not conn:
p.client.invalidateTransaction(tid, invalidated)
elif info is not None:
p.client.info(info)
except ZEO.zrpc.error.DisconnectedError:
pass
def get_invalidations(self, storage_id, tid):
"""Return a tid and list of all objects invalidation since tid.
The tid is the most recent transaction id seen by the client.
Returns None if it is unable to provide a complete list
of invalidations for tid. In this case, client should
do full cache verification.
"""
# We make a copy of invq because it might be modified by a
# foreign (other than main thread) calling invalidate above.
invq = self.invq[storage_id][:]
oids = set()
latest_tid = None
if invq and invq[-1][0] <= tid:
# We have needed data in the queue
for _tid, L in invq:
if _tid <= tid:
break
oids.update(L)
latest_tid = invq[0][0]
elif (self.invalidation_age and
(self.invalidation_age >
(time.time()-ZODB.TimeStamp.TimeStamp(tid).timeTime())
)
):
for t in self.storages[storage_id].iterator(p64(u64(tid)+1)):
for r in t:
oids.add(r.oid)
latest_tid = t.tid
elif not invq:
log("invq empty")
else:
log("tid to old for invq %s < %s" % (u64(tid), u64(invq[-1][0])))
return latest_tid, list(oids)
def loop(self):
try:
asyncore.loop(map=self.socket_map)
except Exception:
if not self.__closed:
raise # Unexpected exc
__thread = None
def start_thread(self, daemon=True):
self.__thread = thread = threading.Thread(target=self.loop)
thread.setDaemon(daemon)
thread.start()
__closed = False
def close(self, join_timeout=1):
"""Close the dispatcher so that there are no new connections.
This is only called from the test suite, AFAICT.
"""
if self.__closed:
return
self.__closed = True
# Stop accepting connections
self.dispatcher.close()
if self.monitor is not None:
self.monitor.close()
ZODB.event.notify(Closed(self))
# Close open client connections
for sid, connections in self.connections.items():
for conn in connections[:]:
try:
conn.connection.close()
except:
pass
for name, storage in self.storages.iteritems():
logger.info("closing storage %r", name)
storage.close()
if self.__thread is not None:
self.__thread.join(join_timeout)
def close_conn(self, conn):
"""Internal: remove the given connection from self.connections.
This is the inverse of register_connection().
"""
for cl in self.connections.values():
if conn.obj in cl:
cl.remove(conn.obj)
def lock_storage(self, zeostore, delay):
storage_id = zeostore.storage_id
waiting = self._waiting[storage_id]
with self._lock:
if storage_id in self._commit_locks:
# The lock is held by another zeostore
locked = self._commit_locks[storage_id]
assert locked is not zeostore, (storage_id, delay)
if locked.connection is None:
locked.log("Still locked after disconnected. Unlocking.",
logging.CRITICAL)
if locked.transaction:
locked.storage.tpc_abort(locked.transaction)
del self._commit_locks[storage_id]
# yuck: have to manipulate lock to appease with :(
self._lock.release()
try:
return self.lock_storage(zeostore, delay)
finally:
self._lock.acquire()
if delay is None:
# New request, queue it
assert not [i for i in waiting if i[0] is zeostore
], "already waiting"
delay = Delay()
waiting.append((zeostore, delay))
zeostore.log("(%r) queue lock: transactions waiting: %s"
% (storage_id, len(waiting)),
_level_for_waiting(waiting)
)
return False, delay
else:
self._commit_locks[storage_id] = zeostore
self.timeouts[storage_id].begin(zeostore)
self.stats[storage_id].lock_time = time.time()
if delay is not None:
# we were waiting, stop
waiting[:] = [i for i in waiting if i[0] is not zeostore]
zeostore.log("(%r) lock: transactions waiting: %s"
% (storage_id, len(waiting)),
_level_for_waiting(waiting)
)
return True, delay
def unlock_storage(self, zeostore):
storage_id = zeostore.storage_id
waiting = self._waiting[storage_id]
with self._lock:
assert self._commit_locks[storage_id] is zeostore
del self._commit_locks[storage_id]
self.timeouts[storage_id].end(zeostore)
self.stats[storage_id].lock_time = None
callbacks = waiting[:]
if callbacks:
assert not [i for i in waiting if i[0] is zeostore
], "waiting while unlocking"
zeostore.log("(%r) unlock: transactions waiting: %s"
% (storage_id, len(callbacks)),
_level_for_waiting(callbacks)
)
for zeostore, delay in callbacks:
try:
zeostore._unlock_callback(delay)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
logger.exception("Calling unlock callback")
def stop_waiting(self, zeostore):
storage_id = zeostore.storage_id
waiting = self._waiting[storage_id]
with self._lock:
new_waiting = [i for i in waiting if i[0] is not zeostore]
if len(new_waiting) == len(waiting):
return
waiting[:] = new_waiting
zeostore.log("(%r) dequeue lock: transactions waiting: %s"
% (storage_id, len(waiting)),
_level_for_waiting(waiting)
)
def already_waiting(self, zeostore):
storage_id = zeostore.storage_id
waiting = self._waiting[storage_id]
with self._lock:
return bool([i for i in waiting if i[0] is zeostore])
def server_status(self, zeostore):
storage_id = zeostore.storage_id
status = self.stats[storage_id].__dict__.copy()
status['connections'] = len(status['connections'])
status['waiting'] = len(self._waiting[storage_id])
status['timeout-thread-is-alive'] = self.timeouts[storage_id].isAlive()
return status
def _level_for_waiting(waiting):
if len(waiting) > 9:
return logging.CRITICAL
if len(waiting) > 3:
return logging.WARNING
else:
return logging.DEBUG
class StubTimeoutThread:
def begin(self, client):
pass
def end(self, client):
pass
isAlive = lambda self: 'stub'
class TimeoutThread(threading.Thread):
"""Monitors transaction progress and generates timeouts."""
# There is one TimeoutThread per storage, because there's one
# transaction lock per storage.
def __init__(self, timeout):
threading.Thread.__init__(self)
self.setDaemon(1)
self._timeout = timeout
self._client = None
self._deadline = None
self._cond = threading.Condition() # Protects _client and _deadline
def begin(self, client):
# Called from the restart code the "main" thread, whenever the
# storage lock is being acquired. (Serialized by asyncore.)
with self._cond:
assert self._client is None
self._client = client
self._deadline = time.time() + self._timeout
self._cond.notify()
def end(self, client):
# Called from the "main" thread whenever the storage lock is
# being released. (Serialized by asyncore.)
with self._cond:
assert self._client is not None
assert self._client is client
self._client = None
self._deadline = None
def run(self):
# Code running in the thread.
while 1:
with self._cond:
while self._deadline is None:
self._cond.wait()
howlong = self._deadline - time.time()
if howlong <= 0:
# Prevent reporting timeout more than once
self._deadline = None
client = self._client # For the howlong <= 0 branch below
if howlong <= 0:
client.log("Transaction timeout after %s seconds" %
self._timeout, logging.CRITICAL)
try:
client.connection.call_from_thread(client.connection.close)
except:
client.log("Timeout failure", logging.CRITICAL,
exc_info=sys.exc_info())
self.end(client)
else:
time.sleep(howlong)
def run_in_thread(method, *args):
t = SlowMethodThread(method, args)
t.start()
return t.delay
class SlowMethodThread(threading.Thread):
"""Thread to run potentially slow storage methods.
Clients can use the delay attribute to access the MTDelay object
used to send a zrpc response at the right time.
"""
# Some storage methods can take a long time to complete. If we
# run these methods via a standard asyncore read handler, they
# will block all other server activity until they complete. To
# avoid blocking, we spawn a separate thread, return an MTDelay()
# object, and have the thread reply() when it finishes.
def __init__(self, method, args):
threading.Thread.__init__(self)
self._method = method
self._args = args
self.delay = MTDelay()
def run(self):
try:
result = self._method(*self._args)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
self.delay.error(sys.exc_info())
else:
self.delay.reply(result)
class ClientStub:
def __init__(self, rpc):
self.rpc = rpc
def beginVerify(self):
self.rpc.callAsync('beginVerify')
def invalidateVerify(self, args):
self.rpc.callAsync('invalidateVerify', args)
def endVerify(self):
self.rpc.callAsync('endVerify')
def invalidateTransaction(self, tid, args):
# Note that this method is *always* called from a different
# thread than self.rpc's async thread. It is the only method
# for which this is true and requires special consideration!
# callAsyncNoSend is important here because:
# - callAsyncNoPoll isn't appropriate because
# the network thread may not wake up for a long time,
# delaying invalidations for too long. (This is demonstrateed
# by a test failure.)
# - callAsync isn't appropriate because (on the server) it tries
# to write to the socket. If self.rpc's network thread also
# tries to write at the ame time, we can run into problems
# because handle_write isn't thread safe.
self.rpc.callAsyncNoSend('invalidateTransaction', tid, args)
def serialnos(self, arg):
self.rpc.callAsyncNoPoll('serialnos', arg)
def info(self, arg):
self.rpc.callAsyncNoPoll('info', arg)
def storeBlob(self, oid, serial, blobfilename):
def store():
yield ('receiveBlobStart', (oid, serial))
f = open(blobfilename, 'rb')
while 1:
chunk = f.read(59000)
if not chunk:
break
yield ('receiveBlobChunk', (oid, serial, chunk, ))
f.close()
yield ('receiveBlobStop', (oid, serial))
self.rpc.callAsyncIterator(store())
class ClientStub308(ClientStub):
def invalidateTransaction(self, tid, args):
ClientStub.invalidateTransaction(
self, tid, [(arg, '') for arg in args])
def invalidateVerify(self, oid):
ClientStub.invalidateVerify(self, (oid, ''))
class ZEOStorage308Adapter:
def __init__(self, storage):
self.storage = storage
def __eq__(self, other):
return self is other or self.storage is other
def getSerial(self, oid):
return self.storage.loadEx(oid)[1] # Z200
def history(self, oid, version, size=1):
if version:
raise ValueError("Versions aren't supported.")
return self.storage.history(oid, size=size)
def getInvalidations(self, tid):
result = self.storage.getInvalidations(tid)
if result is not None:
result = result[0], [(oid, '') for oid in result[1]]
return result
def verify(self, oid, version, tid):
if version:
raise StorageServerError("Versions aren't supported.")
return self.storage.verify(oid, tid)
def loadEx(self, oid, version=''):
if version:
raise StorageServerError("Versions aren't supported.")
data, serial = self.storage.loadEx(oid)
return data, serial, ''
def storea(self, oid, serial, data, version, id):
if version:
raise StorageServerError("Versions aren't supported.")
self.storage.storea(oid, serial, data, id)
def storeBlobEnd(self, oid, serial, data, version, id):
if version:
raise StorageServerError("Versions aren't supported.")
self.storage.storeBlobEnd(oid, serial, data, id)
def storeBlobShared(self, oid, serial, data, filename, version, id):
if version:
raise StorageServerError("Versions aren't supported.")
self.storage.storeBlobShared(oid, serial, data, filename, id)
def getInfo(self):
result = self.storage.getInfo()
result['supportsVersions'] = False
return result
def zeoVerify(self, oid, s, sv=None):
if sv:
raise StorageServerError("Versions aren't supported.")
self.storage.zeoVerify(oid, s)
def modifiedInVersion(self, oid):
return ''
def versions(self):
return ()
def versionEmpty(self, version):
return True
def commitVersion(self, *a, **k):
raise NotImplementedError
abortVersion = commitVersion
def zeoLoad(self, oid): # Z200
p, s = self.storage.loadEx(oid)
return p, s, '', None, None
def __getattr__(self, name):
return getattr(self.storage, name)
def _addr_label(addr):
if isinstance(addr, type("")):
return addr
else:
host, port = addr
return str(host) + ":" + str(port)
class CommitLog:
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".comit-log")
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
self.stores = 0
def size(self):
return self.file.tell()
def delete(self, oid, serial):
self.pickler.dump(('_delete', (oid, serial)))
self.stores += 1
def checkread(self, oid, serial):
self.pickler.dump(('_checkread', (oid, serial)))
self.stores += 1
def store(self, oid, serial, data):
self.pickler.dump(('_store', (oid, serial, data)))
self.stores += 1
def restore(self, oid, serial, data, prev_txn):
self.pickler.dump(('_restore', (oid, serial, data, prev_txn)))
self.stores += 1
def undo(self, transaction_id):
self.pickler.dump(('_undo', (transaction_id, )))
self.stores += 1
def __iter__(self):
self.file.seek(0)
unpickler = cPickle.Unpickler(self.file)
for i in range(self.stores):
yield unpickler.load()
def close(self):
if self.file:
self.file.close()
self.file = None
class ServerEvent:
def __init__(self, server, **kw):
self.__dict__.update(kw)
self.server = server
class Serving(ServerEvent):
pass
class Closed(ServerEvent):
pass
src/ZEO/TransactionBuffer.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A TransactionBuffer store transaction updates until commit or abort.
A transaction may generate enough data that it is not practical to
always hold pending updates in memory. Instead, a TransactionBuffer
is used to store the data until a commit or abort.
"""
# A faster implementation might store trans data in memory until it
# reaches a certain size.
from threading import Lock
import os
import cPickle
import tempfile
import ZODB.blob
class TransactionBuffer:
# Valid call sequences:
#
# ((store | invalidate)* begin_iterate next* clear)* close
#
# get_size can be called any time
# The TransactionBuffer is used by client storage to hold update
# data until the tpc_finish(). It is normally used by a single
# thread, because only one thread can be in the two-phase commit
# at one time.
# It is possible, however, for one thread to close the storage
# while another thread is in the two-phase commit. We must use
# a lock to guard against this race, because unpredictable things
# can happen in Python if one thread closes a file that another
# thread is reading. In a debug build, an assert() can fail.
# Caution: If an operation is performed on a closed TransactionBuffer,
# it has no effect and does not raise an exception. The only time
# this should occur is when a ClientStorage is closed in one
# thread while another thread is in its tpc_finish(). It's not
# clear what should happen in this case. If the tpc_finish()
# completes without error, the Connection using it could have
# inconsistent data. This should have minimal effect, though,
# because the Connection is connected to a closed storage.
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".tbuf")
self.lock = Lock()
self.closed = 0
self.count = 0
self.size = 0
self.blobs = []
# It's safe to use a fast pickler because the only objects
# stored are builtin types -- strings or None.
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
def close(self):
self.clear()
self.lock.acquire()
try:
self.closed = 1
try:
self.file.close()
except OSError:
pass
finally:
self.lock.release()
def store(self, oid, data):
"""Store oid, version, data for later retrieval"""
self.lock.acquire()
try:
if self.closed:
return
self.pickler.dump((oid, data))
self.count += 1
# Estimate per-record cache size
self.size = self.size + (data and len(data) or 0) + 31
finally:
self.lock.release()
def storeBlob(self, oid, blobfilename):
self.blobs.append((oid, blobfilename))
def invalidate(self, oid):
self.lock.acquire()
try:
if self.closed:
return
self.pickler.dump((oid, None))
self.count += 1
finally:
self.lock.release()
def clear(self):
"""Mark the buffer as empty"""
self.lock.acquire()
try:
if self.closed:
return
self.file.seek(0)
self.count = 0
self.size = 0
while self.blobs:
oid, blobfilename = self.blobs.pop()
if os.path.exists(blobfilename):
ZODB.blob.remove_committed(blobfilename)
finally:
self.lock.release()
def __iter__(self):
self.lock.acquire()
try:
if self.closed:
return
self.file.flush()
self.file.seek(0)
return TBIterator(self.file, self.count)
finally:
self.lock.release()
class TBIterator(object):
def __init__(self, f, count):
self.file = f
self.count = count
self.unpickler = cPickle.Unpickler(f)
def __iter__(self):
return self
def next(self):
"""Return next tuple of data or None if EOF"""
if self.count == 0:
self.file.seek(0)
self.size = 0
raise StopIteration
oid_ver_data = self.unpickler.load()
self.count -= 1
return oid_ver_data
src/ZEO/__init__.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZEO -- Zope Enterprise Objects.
See the file README.txt in this directory for an overview.
ZEO is now part of ZODB; ZODB's home on the web is
http://wiki.zope.org/ZODB
"""
def client(*args, **kw):
import ZEO.ClientStorage
return ZEO.ClientStorage.ClientStorage(*args, **kw)
def DB(*args, **kw):
import ZODB
return ZODB.DB(client(*args, **kw))
def connection(*args, **kw):
return DB(*args, **kw).open_then_close_db_when_connection_closes()
def server(path=None, blob_dir=None, storage_conf=None, zeo_conf=None,
port=None):
"""Convenience function to start a server for interactive exploration
This fuction starts a ZEO server, given a storage configuration or
a file-storage path and blob directory. You can also supply a ZEO
configuration string or a port. If neither a ZEO port or
configuration is supplied, a port is chosen randomly.
The server address and a stop function are returned. The address
can be passed to ZEO.ClientStorage.ClientStorage or ZEO.DB to
create a client to the server. The stop function can be called
without arguments to stop the server.
Arguments:
path
A file-storage path. This argument is ignored if a storage
configuration is supplied.
blob_dir
A blob directory path. This argument is ignored if a storage
configuration is supplied.
storage_conf
A storage configuration string. If none is supplied, then at
least a file-storage path must be supplied and the storage
configuration will be generated from the file-storage path and
the blob directory.
zeo_conf
A ZEO server configuration string.
port
If no ZEO configuration is supplied, the one will be computed
from the port. If no port is supplied, one will be chosedn
randomly.
"""
import os, ZEO.tests.forker
if storage_conf is None and path is None:
storage_conf = '<mappingstorage>\n</mappingstorage>'
if port is None and zeo_conf is None:
port = ZEO.tests.forker.get_port()
addr, admin, pid, config = ZEO.tests.forker.start_zeo_server(
storage_conf, zeo_conf, port, keep=True, path=path,
blob_dir=blob_dir, suicide=False)
os.remove(config)
def stop_server():
ZEO.tests.forker.shutdown_zeo_server(admin)
os.waitpid(pid, 0)
return addr, stop_server
src/ZEO/auth/__init__.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
_auth_modules = {}
def get_module(name):
if name == 'sha':
from auth_sha import StorageClass, SHAClient, Database
return StorageClass, SHAClient, Database
elif name == 'digest':
from auth_digest import StorageClass, DigestClient, DigestDatabase
return StorageClass, DigestClient, DigestDatabase
else:
return _auth_modules.get(name)
def register_module(name, storage_class, client, db):
if _auth_modules.has_key(name):
raise TypeError("%s is already registred" % name)
_auth_modules[name] = storage_class, client, db
src/ZEO/auth/auth_digest.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Digest authentication for ZEO
This authentication mechanism follows the design of HTTP digest
authentication (RFC 2069). It is a simple challenge-response protocol
that does not send passwords in the clear, but does not offer strong
security. The RFC discusses many of the limitations of this kind of
protocol.
Guard the password database as if it contained plaintext passwords.
It stores the hash of a username and password. This does not expose
the plaintext password, but it is sensitive nonetheless. An attacker
with the hash can impersonate the real user. This is a limitation of
the simple digest scheme.
HTTP is a stateless protocol, and ZEO is a stateful protocol. The
security requirements are quite different as a result. The HTTP
protocol uses a nonce as a challenge. The ZEO protocol requires a
separate session key that is used for message authentication. We
generate a second nonce for this purpose; the hash of nonce and
user/realm/password is used as the session key.
TODO: I'm not sure if this is a sound approach; SRP would be preferred.
"""
import os
import random
import struct
import time
from ZEO.auth.base import Database, Client
from ZEO.StorageServer import ZEOStorage
from ZEO.Exceptions import AuthError
from ZEO.hash import sha1
def get_random_bytes(n=8):
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
s = f.read(n)
f.close()
else:
L = [chr(random.randint(0, 255)) for i in range(n)]
s = "".join(L)
return s
def hexdigest(s):
return sha1(s).hexdigest()
class DigestDatabase(Database):
def __init__(self, filename, realm=None):
Database.__init__(self, filename, realm)
# Initialize a key used to build the nonce for a challenge.
# We need one key for the lifetime of the server, so it
# is convenient to store in on the database.
self.noncekey = get_random_bytes(8)
def _store_password(self, username, password):
dig = hexdigest("%s:%s:%s" % (username, self.realm, password))
self._users[username] = dig
def session_key(h_up, nonce):
# The hash itself is a bit too short to be a session key.
# HMAC wants a 64-byte key. We don't want to use h_up
# directly because it would never change over time. Instead
# use the hash plus part of h_up.
return sha1("%s:%s" % (h_up, nonce)).digest() + h_up[:44]
class StorageClass(ZEOStorage):
def set_database(self, database):
assert isinstance(database, DigestDatabase)
self.database = database
self.noncekey = database.noncekey
def _get_time(self):
# Return a string representing the current time.
t = int(time.time())
return struct.pack("i", t)
def _get_nonce(self):
# RFC 2069 recommends a nonce of the form
# H(client-IP ":" time-stamp ":" private-key)
dig = sha1()
dig.update(str(self.connection.addr))
dig.update(self._get_time())
dig.update(self.noncekey)
return dig.hexdigest()
def auth_get_challenge(self):
"""Return realm, challenge, and nonce."""
self._challenge = self._get_nonce()
self._key_nonce = self._get_nonce()
return self.auth_realm, self._challenge, self._key_nonce
def auth_response(self, resp):
# verify client response
user, challenge, response = resp
# Since zrpc is a stateful protocol, we just store the nonce
# we sent to the client. It will need to generate a new
# nonce for a new connection anyway.
if self._challenge != challenge:
raise ValueError("invalid challenge")
# lookup user in database
h_up = self.database.get_password(user)
# regeneration resp from user, password, and nonce
check = hexdigest("%s:%s" % (h_up, challenge))
if check == response:
self.connection.setSessionKey(session_key(h_up, self._key_nonce))
return self._finish_auth(check == response)
extensions = [auth_get_challenge, auth_response]
class DigestClient(Client):
extensions = ["auth_get_challenge", "auth_response"]
def start(self, username, realm, password):
_realm, challenge, nonce = self.stub.auth_get_challenge()
if _realm != realm:
raise AuthError("expected realm %r, got realm %r"
% (_realm, realm))
h_up = hexdigest("%s:%s:%s" % (username, realm, password))
resp_dig = hexdigest("%s:%s" % (h_up, challenge))
result = self.stub.auth_response((username, challenge, resp_dig))
if result:
return session_key(h_up, nonce)
else:
return None
src/ZEO/auth/base.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Base classes for defining an authentication protocol.
Database -- abstract base class for password database
Client -- abstract base class for authentication client
"""
import os
from ZEO.hash import sha1
class Client:
# Subclass should override to list the names of methods that
# will be called on the server.
extensions = []
def __init__(self, stub):
self.stub = stub
for m in self.extensions:
setattr(self.stub, m, self.stub.extensionMethod(m))
def sort(L):
"""Sort a list in-place and return it."""
L.sort()
return L
class Database:
"""Abstracts a password database.
This class is used both in the authentication process (via
get_password()) and by client scripts that manage the password
database file.
The password file is a simple, colon-separated text file mapping
usernames to password hashes. The hashes are SHA hex digests
produced from the password string.
"""
realm = None
def __init__(self, filename, realm=None):
"""Creates a new Database
filename: a string containing the full pathname of
the password database file. Must be readable by the user
running ZEO. Must be writeable by any client script that
accesses the database.
realm: the realm name (a string)
"""
self._users = {}
self.filename = filename
self.load()
if realm:
if self.realm and self.realm != realm:
raise ValueError("Specified realm %r differs from database "
"realm %r" % (realm or '', self.realm))
else:
self.realm = realm
def save(self, fd=None):
filename = self.filename
if not fd:
fd = open(filename, 'w')
if self.realm:
print >> fd, "realm", self.realm
for username in sort(self._users.keys()):
print >> fd, "%s: %s" % (username, self._users[username])
def load(self):
filename = self.filename
if not filename:
return
if not os.path.exists(filename):
return
fd = open(filename)
L = fd.readlines()
if not L:
return
if L[0].startswith("realm "):
line = L.pop(0).strip()
self.realm = line[len("realm "):]
for line in L:
username, hash = line.strip().split(":", 1)
self._users[username] = hash.strip()
def _store_password(self, username, password):
self._users[username] = self.hash(password)
def get_password(self, username):
"""Returns password hash for specified username.
Callers must check for LookupError, which is raised in
the case of a non-existent user specified."""
if not self._users.has_key(username):
raise LookupError("No such user: %s" % username)
return self._users[username]
def hash(self, s):
return sha1(s).hexdigest()
def add_user(self, username, password):
if self._users.has_key(username):
raise LookupError("User %s already exists" % username)
self._store_password(username, password)
def del_user(self, username):
if not self._users.has_key(username):
raise LookupError("No such user: %s" % username)
del self._users[username]
def change_password(self, username, password):
if not self._users.has_key(username):
raise LookupError("No such user: %s" % username)
self._store_password(username, password)
src/ZEO/auth/hmac.py deleted 100644 → 0
View file @ 56c7dd30
"""HMAC (Keyed-Hashing for Message Authentication) Python module.
Implements the HMAC algorithm as described by RFC 2104.
"""
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
"""
return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
# The size of the digests returned by HMAC depends on the underlying
# hashing module used.
digest_size = None
class HMAC:
"""RFC2104 HMAC class.
This supports the API for Cryptographic Hash Functions (PEP 247).
"""
def __init__(self, key, msg = None, digestmod = None):
"""Create a new HMAC object.
key: key for the keyed hash object.
msg: Initial input for the hash, if provided.
digestmod: A module supporting PEP 247. Defaults to the md5 module.
"""
if digestmod is None:
import md5
digestmod = md5
self.digestmod = digestmod
self.outer = digestmod.new()
self.inner = digestmod.new()
self.digest_size = digestmod.digest_size
blocksize = 64
ipad = "\x36" * blocksize
opad = "\x5C" * blocksize
if len(key) > blocksize:
key = digestmod.new(key).digest()
key = key + chr(0) * (blocksize - len(key))
self.outer.update(_strxor(key, opad))
self.inner.update(_strxor(key, ipad))
if msg is not None:
self.update(msg)
## def clear(self):
## raise NotImplementedError("clear() method not available in HMAC.")
def update(self, msg):
"""Update this hashing object with the string msg.
"""
self.inner.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
An update to this copy won't affect the original object.
"""
other = HMAC("")
other.digestmod = self.digestmod
other.inner = self.inner.copy()
other.outer = self.outer.copy()
return other
def digest(self):
"""Return the hash value of this hashing object.
This returns a string containing 8-bit data. The object is
not altered in any way by this function; you can continue
updating the object after calling this function.
"""
h = self.outer.copy()
h.update(self.inner.digest())
return h.digest()
def hexdigest(self):
"""Like digest(), but returns a string of hexadecimal digits instead.
"""
return "".join([hex(ord(x))[2:].zfill(2)
for x in tuple(self.digest())])
def new(key, msg = None, digestmod = None):
"""Create a new hashing object and return it.
key: The starting key for the hash.
msg: if available, will immediately be hashed into the object's starting
state.
You can now feed arbitrary strings into the object using its update()
method, and can ask for the hash value at any time by calling its digest()
method.
"""
return HMAC(key, msg, digestmod)
src/ZEO/cache.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Disk-based client cache for ZEO.
ClientCache exposes an API used by the ZEO client storage. FileCache stores
objects on disk using a 2-tuple of oid and tid as key.
ClientCache's API is similar to a storage API, with methods like load(),
store(), and invalidate(). It manages in-memory data structures that allow
it to map this richer API onto the simple key-based API of the lower-level
FileCache.
"""
from struct import pack, unpack
import BTrees.LLBTree
import BTrees.LOBTree
import logging
import os
import tempfile
import threading
import time
import ZODB.fsIndex
import zc.lockfile
from ZODB.utils import p64, u64, z64
logger = logging.getLogger("ZEO.cache")
# A disk-based cache for ZEO clients.
#
# This class provides an interface to a persistent, disk-based cache
# used by ZEO clients to store copies of database records from the
# server.
#
# The details of the constructor as unspecified at this point.
#
# Each entry in the cache is valid for a particular range of transaction
# ids. The lower bound is the transaction that wrote the data. The
# upper bound is the next transaction that wrote a revision of the
# object. If the data is current, the upper bound is stored as None;
# the data is considered current until an invalidate() call is made.
#
# It is an error to call store() twice with the same object without an
# intervening invalidate() to set the upper bound on the first cache
# entry. Perhaps it will be necessary to have a call the removes
# something from the cache outright, without keeping a non-current
# entry.
# Cache verification
#
# When the client is connected to the server, it receives
# invalidations every time an object is modified. When the client is
# disconnected then reconnects, it must perform cache verification to make
# sure its cached data is synchronized with the storage's current state.
#
# quick verification
# full verification
#
# FileCache stores a cache in a single on-disk file.
#
# On-disk cache structure.
#
# The file begins with a 12-byte header. The first four bytes are the
# file's magic number - ZEC3 - indicating zeo cache version 4. The
# next eight bytes are the last transaction id.
magic = "ZEC3"
ZEC_HEADER_SIZE = 12
# Maximum block size. Note that while we are doing a store, we may
# need to write a free block that is almost twice as big. If we die
# in the middle of a store, then we need to split the large free records
# while opening.
max_block_size = (1<<31) - 1
# After the header, the file contains a contiguous sequence of blocks. All
# blocks begin with a one-byte status indicator:
#
# 'a'
# Allocated. The block holds an object; the next 4 bytes are >I
# format total block size.
#
# 'f'
# Free. The block is free; the next 4 bytes are >I format total
# block size.
#
# '1', '2', '3', '4'
# The block is free, and consists of 1, 2, 3 or 4 bytes total.
#
# "Total" includes the status byte, and size bytes. There are no
# empty (size 0) blocks.
# Allocated blocks have more structure:
#
# 1 byte allocation status ('a').
# 4 bytes block size, >I format.
# 8 byte oid
# 8 byte start_tid
# 8 byte end_tid
# 2 byte version length must be 0
# 4 byte data size
# data
# 8 byte redundant oid for error detection.
allocated_record_overhead = 43
# The cache's currentofs goes around the file, circularly, forever.
# It's always the starting offset of some block.
#
# When a new object is added to the cache, it's stored beginning at
# currentofs, and currentofs moves just beyond it. As many contiguous
# blocks needed to make enough room for the new object are evicted,
# starting at currentofs. Exception: if currentofs is close enough
# to the end of the file that the new object can't fit in one
# contiguous chunk, currentofs is reset to ZEC_HEADER_SIZE first.
class locked(object):
def __init__(self, func):
self.func = func
def __get__(self, inst, class_):
if inst is None:
return self
def call(*args, **kw):
inst._lock.acquire()
try:
return self.func(inst, *args, **kw)
finally:
inst._lock.release()
return call
class ClientCache(object):
"""A simple in-memory cache."""
# The default size of 200MB makes a lot more sense than the traditional
# default of 20MB. The default here is misleading, though, since
# ClientStorage is the only user of ClientCache, and it always passes an
# explicit size of its own choosing.
def __init__(self, path=None, size=200*1024**2, rearrange=.8):
# - `path`: filepath for the cache file, or None (in which case
# a temp file will be created)
self.path = path
# - `maxsize`: total size of the cache file
# We set to the minimum size of less than the minimum.
size = max(size, ZEC_HEADER_SIZE)
self.maxsize = size
# rearrange: if we read a current record and it's more than
# rearrange*size from the end, then copy it forward to keep it
# from being evicted.
self.rearrange = rearrange * size
# The number of records in the cache.
self._len = 0
# {oid -> pos}
self.current = ZODB.fsIndex.fsIndex()
# {oid -> {tid->pos}}
# Note that caches in the wild seem to have very little non-current
# data, so this would seem to have little impact on memory consumption.
# I wonder if we even need to store non-current data in the cache.
self.noncurrent = BTrees.LOBTree.LOBTree()
# tid for the most recent transaction we know about. This is also
# stored near the start of the file.
self.tid = z64
# Always the offset into the file of the start of a block.
# New and relocated objects are always written starting at
# currentofs.
self.currentofs = ZEC_HEADER_SIZE
# self.f is the open file object.
# When we're not reusing an existing file, self.f is left None
# here -- the scan() method must be called then to open the file
# (and it sets self.f).
fsize = ZEC_HEADER_SIZE
if path:
self._lock_file = zc.lockfile.LockFile(path + '.lock')
if not os.path.exists(path):
# Create a small empty file. We'll make it bigger in _initfile.
self.f = open(path, 'wb+')
self.f.write(magic+z64)
logger.info("created persistent cache file %r", path)
else:
fsize = os.path.getsize(self.path)
self.f = open(path, 'rb+')
logger.info("reusing persistent cache file %r", path)
else:
# Create a small empty file. We'll make it bigger in _initfile.
self.f = tempfile.TemporaryFile()
self.f.write(magic+z64)
logger.info("created temporary cache file %r", self.f.name)
try:
self._initfile(fsize)
except:
self.f.close()
if not path:
raise # unrecoverable temp file error :(
badpath = path+'.bad'
if os.path.exists(badpath):
logger.critical(
'Removing bad cache file: %r (prev bad exists).',
path, exc_info=1)
os.remove(path)
else:
logger.critical('Moving bad cache file to %r.',
badpath, exc_info=1)
os.rename(path, badpath)
self.f = open(path, 'wb+')
self.f.write(magic+z64)
self._initfile(ZEC_HEADER_SIZE)
# Statistics: _n_adds, _n_added_bytes,
# _n_evicts, _n_evicted_bytes,
# _n_accesses
self.clearStats()
self._setup_trace(path)
self._lock = threading.RLock()
# Backward compatibility. Client code used to have to use the fc
# attr to get to the file cache to get cache stats.
@property
def fc(self):
return self
def clear(self):
self.f.seek(ZEC_HEADER_SIZE)
self.f.truncate()
self._initfile(ZEC_HEADER_SIZE)
##
# Scan the current contents of the cache file, calling `install`
# for each object found in the cache. This method should only
# be called once to initialize the cache from disk.
def _initfile(self, fsize):
maxsize = self.maxsize
f = self.f
read = f.read
seek = f.seek
write = f.write
seek(0)
if read(4) != magic:
seek(0)
raise ValueError("unexpected magic number: %r" % read(4))
self.tid = read(8)
if len(self.tid) != 8:
raise ValueError("cache file too small -- no tid at start")
# Populate .filemap and .key2entry to reflect what's currently in the
# file, and tell our parent about it too (via the `install` callback).
# Remember the location of the largest free block. That seems a
# decent place to start currentofs.
self.current = ZODB.fsIndex.fsIndex()
self.noncurrent = BTrees.LOBTree.LOBTree()
l = 0
last = ofs = ZEC_HEADER_SIZE
first_free_offset = 0
current = self.current
status = ' '
while ofs < fsize:
seek(ofs)
status = read(1)
if status == 'a':
size, oid, start_tid, end_tid, lver = unpack(
">I8s8s8sH", read(30))
if ofs+size <= maxsize:
if end_tid == z64:
assert oid not in current, (ofs, f.tell())
current[oid] = ofs
else:
assert start_tid < end_tid, (ofs, f.tell())
self._set_noncurrent(oid, start_tid, ofs)
assert lver == 0, "Versions aren't supported"
l += 1
else:
# free block
if first_free_offset == 0:
first_free_offset = ofs
if status == 'f':
size, = unpack(">I", read(4))
if size > max_block_size:
# Oops, we either have an old cache, or a we
# crashed while storing. Split this block into two.
assert size <= max_block_size*2
seek(ofs+max_block_size)
write('f'+pack(">I", size-max_block_size))
seek(ofs)
write('f'+pack(">I", max_block_size))
sync(f)
elif status in '1234':
size = int(status)
else:
raise ValueError("unknown status byte value %s in client "
"cache file" % 0, hex(ord(status)))
last = ofs
ofs += size
if ofs >= maxsize:
# Oops, the file was bigger before.
if ofs > maxsize:
# The last record is too big. Replace it with a smaller
# free record
size = maxsize-last
seek(last)
if size > 4:
write('f'+pack(">I", size))
else:
write("012345"[size])
sync(f)
ofs = maxsize
break
if fsize < maxsize:
assert ofs==fsize
# Make sure the OS really saves enough bytes for the file.
seek(self.maxsize - 1)
write('x')
# add as many free blocks as are needed to fill the space
seek(ofs)
nfree = maxsize - ZEC_HEADER_SIZE
for i in range(0, nfree, max_block_size):
block_size = min(max_block_size, nfree-i)
write('f' + pack(">I", block_size))
seek(block_size-5, 1)
sync(self.f)
# There is always data to read and
assert last and status in ' f1234'
first_free_offset = last
else:
assert ofs==maxsize
if maxsize < fsize:
seek(maxsize)
f.truncate()
# We use the first_free_offset because it is most likelyt the
# place where we last wrote.
self.currentofs = first_free_offset or ZEC_HEADER_SIZE
self._len = l
def _set_noncurrent(self, oid, tid, ofs):
noncurrent_for_oid = self.noncurrent.get(u64(oid))
if noncurrent_for_oid is None:
noncurrent_for_oid = BTrees.LLBTree.LLBucket()
self.noncurrent[u64(oid)] = noncurrent_for_oid
noncurrent_for_oid[u64(tid)] = ofs
def _del_noncurrent(self, oid, tid):
try:
noncurrent_for_oid = self.noncurrent[u64(oid)]
del noncurrent_for_oid[u64(tid)]
if not noncurrent_for_oid:
del self.noncurrent[u64(oid)]
except KeyError:
logger.error("Couldn't find non-current %r", (oid, tid))
def clearStats(self):
self._n_adds = self._n_added_bytes = 0
self._n_evicts = self._n_evicted_bytes = 0
self._n_accesses = 0
def getStats(self):
return (self._n_adds, self._n_added_bytes,
self._n_evicts, self._n_evicted_bytes,
self._n_accesses
)
##
# The number of objects currently in the cache.
def __len__(self):
return self._len
##
# Close the underlying file. No methods accessing the cache should be
# used after this.
def close(self):
self._unsetup_trace()
f = self.f
self.f = None
if f is not None:
sync(f)
f.close()
if hasattr(self,'_lock_file'):
self._lock_file.close()
##
# Evict objects as necessary to free up at least nbytes bytes,
# starting at currentofs. If currentofs is closer than nbytes to
# the end of the file, currentofs is reset to ZEC_HEADER_SIZE first.
# The number of bytes actually freed may be (and probably will be)
# greater than nbytes, and is _makeroom's return value. The file is not
# altered by _makeroom. filemap and key2entry are updated to reflect the
# evictions, and it's the caller's responsibility both to fiddle
# the file, and to update filemap, to account for all the space
# freed (starting at currentofs when _makeroom returns, and
# spanning the number of bytes retured by _makeroom).
def _makeroom(self, nbytes):
assert 0 < nbytes <= self.maxsize - ZEC_HEADER_SIZE, (
nbytes, self.maxsize)
if self.currentofs + nbytes > self.maxsize:
self.currentofs = ZEC_HEADER_SIZE
ofs = self.currentofs
seek = self.f.seek
read = self.f.read
current = self.current
while nbytes > 0:
seek(ofs)
status = read(1)
if status == 'a':
size, oid, start_tid, end_tid = unpack(">I8s8s8s", read(28))
self._n_evicts += 1
self._n_evicted_bytes += size
if end_tid == z64:
del current[oid]
else:
self._del_noncurrent(oid, start_tid)
self._len -= 1
else:
if status == 'f':
size = unpack(">I", read(4))[0]
else:
assert status in '1234'
size = int(status)
ofs += size
nbytes -= size
return ofs - self.currentofs
##
# Update our idea of the most recent tid. This is stored in the
# instance, and also written out near the start of the cache file. The
# new tid must be strictly greater than our current idea of the most
# recent tid.
@locked
def setLastTid(self, tid):
if (not tid) or (tid == z64):
return
if (tid <= self.tid) and self._len:
if tid == self.tid:
return # Be a little forgiving
raise ValueError("new last tid (%s) must be greater than "
"previous one (%s)"
% (u64(tid), u64(self.tid)))
assert isinstance(tid, str) and len(tid) == 8, tid
self.tid = tid
self.f.seek(len(magic))
self.f.write(tid)
self.f.flush()
##
# Return the last transaction seen by the cache.
# @return a transaction id
# @defreturn string, or 8 nulls if no transaction is yet known
def getLastTid(self):
return self.tid
##
# Return the current data record for oid.
# @param oid object id
# @return (data record, serial number, tid), or None if the object is not
# in the cache
# @defreturn 3-tuple: (string, string, string)
@locked
def load(self, oid):
ofs = self.current.get(oid)
if ofs is None:
self._trace(0x20, oid)
return None
self.f.seek(ofs)
read = self.f.read
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid)
size, saved_oid, tid, end_tid, lver, ldata = unpack(
">I8s8s8sHI", read(34))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
assert end_tid == z64, (ofs, self.f.tell(), oid, tid, end_tid)
assert lver == 0, "Versions aren't supported"
data = read(ldata)
assert len(data) == ldata, (ofs, self.f.tell(), oid, len(data), ldata)
# WARNING: The following assert changes the file position.
# We must not depend on this below or we'll fail in optimized mode.
assert read(8) == oid, (ofs, self.f.tell(), oid)
self._n_accesses += 1
self._trace(0x22, oid, tid, end_tid, ldata)
ofsofs = self.currentofs - ofs
if ofsofs < 0:
ofsofs += self.maxsize
if (ofsofs > self.rearrange and
self.maxsize > 10*len(data) and
size > 4):
# The record is far back and might get evicted, but it's
# valuable, so move it forward.
# Remove fromn old loc:
del self.current[oid]
self.f.seek(ofs)
self.f.write('f'+pack(">I", size))
# Write to new location:
self._store(oid, tid, None, data, size)
return data, tid
##
# Return a non-current revision of oid that was current before tid.
# @param oid object id
# @param tid id of transaction that wrote next revision of oid
# @return data record, serial number, start tid, and end tid
# @defreturn 4-tuple: (string, string, string, string)
@locked
def loadBefore(self, oid, before_tid):
noncurrent_for_oid = self.noncurrent.get(u64(oid))
if noncurrent_for_oid is None:
self._trace(0x24, oid, "", before_tid)
return None
items = noncurrent_for_oid.items(None, u64(before_tid)-1)
if not items:
self._trace(0x24, oid, "", before_tid)
return None
tid, ofs = items[-1]
self.f.seek(ofs)
read = self.f.read
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid, before_tid)
size, saved_oid, saved_tid, end_tid, lver, ldata = unpack(
">I8s8s8sHI", read(34))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
assert saved_tid == p64(tid), (ofs, self.f.tell(), oid, saved_tid, tid)
assert end_tid != z64, (ofs, self.f.tell(), oid)
assert lver == 0, "Versions aren't supported"
data = read(ldata)
assert len(data) == ldata, (ofs, self.f.tell())
# WARNING: The following assert changes the file position.
# We must not depend on this below or we'll fail in optimized mode.
assert read(8) == oid, (ofs, self.f.tell(), oid)
if end_tid < before_tid:
self._trace(0x24, oid, "", before_tid)
return None
self._n_accesses += 1
self._trace(0x26, oid, "", saved_tid)
return data, saved_tid, end_tid
##
# Store a new data record in the cache.
# @param oid object id
# @param start_tid the id of the transaction that wrote this revision
# @param end_tid the id of the transaction that created the next
# revision of oid. If end_tid is None, the data is
# current.
# @param data the actual data
@locked
def store(self, oid, start_tid, end_tid, data):
seek = self.f.seek
if end_tid is None:
ofs = self.current.get(oid)
if ofs:
seek(ofs)
read = self.f.read
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid)
size, saved_oid, saved_tid, end_tid = unpack(
">I8s8s8s", read(28))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
assert end_tid == z64, (ofs, self.f.tell(), oid)
if saved_tid == start_tid:
return
raise ValueError("already have current data for oid")
else:
noncurrent_for_oid = self.noncurrent.get(u64(oid))
if noncurrent_for_oid and (u64(start_tid) in noncurrent_for_oid):
return
size = allocated_record_overhead + len(data)
# A number of cache simulation experiments all concluded that the
# 2nd-level ZEO cache got a much higher hit rate if "very large"
# objects simply weren't cached. For now, we ignore the request
# only if the entire cache file is too small to hold the object.
if size >= min(max_block_size, self.maxsize - ZEC_HEADER_SIZE):
return
self._n_adds += 1
self._n_added_bytes += size
self._len += 1
self._store(oid, start_tid, end_tid, data, size)
if end_tid:
self._trace(0x54, oid, start_tid, end_tid, dlen=len(data))
else:
self._trace(0x52, oid, start_tid, dlen=len(data))
def _store(self, oid, start_tid, end_tid, data, size):
# Low-level store used by store and load
# In the next line, we ask for an extra to make sure we always
# have a free block after the new alocated block. This free
# block acts as a ring pointer, so that on restart, we start
# where we left off.
nfreebytes = self._makeroom(size+1)
assert size <= nfreebytes, (size, nfreebytes)
excess = nfreebytes - size
# If there's any excess (which is likely), we need to record a
# free block following the end of the data record. That isn't
# expensive -- it's all a contiguous write.
if excess == 0:
extra = ''
elif excess < 5:
extra = "01234"[excess]
else:
extra = 'f' + pack(">I", excess)
ofs = self.currentofs
seek = self.f.seek
seek(ofs)
write = self.f.write
# Before writing data, we'll write a free block for the space freed.
# We'll come back with a last atomic write to rewrite the start of the
# allocated-block header.
write('f'+pack(">I", nfreebytes))
# Now write the rest of the allocation block header and object data.
write(pack(">8s8s8sHI", oid, start_tid, end_tid or z64, 0, len(data)))
write(data)
write(oid)
write(extra)
# Now, we'll go back and rewrite the beginning of the
# allocated block header.
seek(ofs)
write('a'+pack(">I", size))
if end_tid:
self._set_noncurrent(oid, start_tid, ofs)
else:
self.current[oid] = ofs
self.currentofs += size
##
# If `tid` is None,
# forget all knowledge of `oid`. (`tid` can be None only for
# invalidations generated by startup cache verification.) If `tid`
# isn't None, and we had current
# data for `oid`, stop believing we have current data, and mark the
# data we had as being valid only up to `tid`. In all other cases, do
# nothing.
#
# Paramters:
#
# - oid object id
# - tid the id of the transaction that wrote a new revision of oid,
# or None to forget all cached info about oid.
@locked
def invalidate(self, oid, tid):
ofs = self.current.get(oid)
if ofs is None:
# 0x10 == invalidate (miss)
self._trace(0x10, oid, tid)
return
self.f.seek(ofs)
read = self.f.read
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid)
size, saved_oid, saved_tid, end_tid = unpack(">I8s8s8s", read(28))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
assert end_tid == z64, (ofs, self.f.tell(), oid)
del self.current[oid]
if tid is None:
self.f.seek(ofs)
self.f.write('f'+pack(">I", size))
# 0x1E = invalidate (hit, discarding current or non-current)
self._trace(0x1E, oid, tid)
self._len -= 1
else:
if tid == saved_tid:
logger.warning("Ignoring invalidation with same tid as current")
return
self.f.seek(ofs+21)
self.f.write(tid)
self._set_noncurrent(oid, saved_tid, ofs)
# 0x1C = invalidate (hit, saving non-current)
self._trace(0x1C, oid, tid)
##
# Generates (oid, serial) oairs for all objects in the
# cache. This generator is used by cache verification.
def contents(self):
# May need to materialize list instead of iterating;
# depends on whether the caller may change the cache.
seek = self.f.seek
read = self.f.read
for oid, ofs in self.current.iteritems():
self._lock.acquire()
try:
seek(ofs)
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid)
size, saved_oid, tid, end_tid = unpack(">I8s8s8s", read(28))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
assert end_tid == z64, (ofs, self.f.tell(), oid)
result = oid, tid
finally:
self._lock.release()
yield result
def dump(self):
from ZODB.utils import oid_repr
print "cache size", len(self)
L = list(self.contents())
L.sort()
for oid, tid in L:
print oid_repr(oid), oid_repr(tid)
print "dll contents"
L = list(self)
L.sort(lambda x, y: cmp(x.key, y.key))
for x in L:
end_tid = x.end_tid or z64
print oid_repr(x.key[0]), oid_repr(x.key[1]), oid_repr(end_tid)
print
# If `path` isn't None (== we're using a persistent cache file), and
# envar ZEO_CACHE_TRACE is set to a non-empty value, try to open
# path+'.trace' as a trace file, and store the file object in
# self._tracefile. If not, or we can't write to the trace file, disable
# tracing by setting self._trace to a dummy function, and set
# self._tracefile to None.
_tracefile = None
def _trace(self, *a, **kw):
pass
def _setup_trace(self, path):
_tracefile = None
if path and os.environ.get("ZEO_CACHE_TRACE"):
tfn = path + ".trace"
try:
_tracefile = open(tfn, "ab")
except IOError, msg:
logger.warning("cannot write tracefile %r (%s)", tfn, msg)
else:
logger.info("opened tracefile %r", tfn)
if _tracefile is None:
return
now = time.time
def _trace(code, oid="", tid=z64, end_tid=z64, dlen=0):
# The code argument is two hex digits; bits 0 and 7 must be zero.
# The first hex digit shows the operation, the second the outcome.
# This method has been carefully tuned to be as fast as possible.
# Note: when tracing is disabled, this method is hidden by a dummy.
encoded = (dlen << 8) + code
if tid is None:
tid = z64
if end_tid is None:
end_tid = z64
try:
_tracefile.write(
pack(">iiH8s8s",
now(), encoded, len(oid), tid, end_tid) + oid,
)
except:
print `tid`, `end_tid`
raise
self._trace = _trace
self._tracefile = _tracefile
_trace(0x00)
def _unsetup_trace(self):
if self._tracefile is not None:
del self._trace
self._tracefile.close()
del self._tracefile
def sync(f):
f.flush()
if hasattr(os, 'fsync'):
def sync(f):
f.flush()
os.fsync(f.fileno())
src/ZEO/component.xml deleted 100644 → 0
View file @ 56c7dd30
<component>
<sectiontype name="zeo">
<description>
The content of a ZEO section describe operational parameters
of a ZEO server except for the storage(s) to be served.
</description>
<key name="address" datatype="socket-binding-address"
required="yes">
<description>
The address at which the server should listen. This can be in
the form 'host:port' to signify a TCP/IP connection or a
pathname string to signify a Unix domain socket connection (at
least one '/' is required). A hostname may be a DNS name or a
dotted IP address. If the hostname is omitted, the platform's
default behavior is used when binding the listening socket (''
is passed to socket.bind() as the hostname portion of the
address).
</description>
</key>
<key name="read-only" datatype="boolean"
required="no"
default="false">
<description>
Flag indicating whether the server should operate in read-only
mode. Defaults to false. Note that even if the server is
operating in writable mode, individual storages may still be
read-only. But if the server is in read-only mode, no write
operations are allowed, even if the storages are writable. Note
that pack() is considered a read-only operation.
</description>
</key>
<key name="invalidation-queue-size" datatype="integer"
required="no"
default="100">
<description>
The storage server keeps a queue of the objects modified by the
last N transactions, where N == invalidation_queue_size. This
queue is used to speed client cache verification when a client
disconnects for a short period of time.
</description>
</key>
<key name="invalidation-age" datatype="float" required="no">
<description>
The maximum age of a client for which quick-verification
invalidations will be provided by iterating over the served
storage. This option should only be used if the served storage
supports efficient iteration from a starting point near the
end of the transaction history (e.g. end of file).
</description>
</key>
<key name="monitor-address" datatype="socket-binding-address"
required="no">
<description>
The address at which the monitor server should listen. If
specified, a monitor server is started. The monitor server
provides server statistics in a simple text format. This can
be in the form 'host:port' to signify a TCP/IP connection or a
pathname string to signify a Unix domain socket connection (at
least one '/' is required). A hostname may be a DNS name or a
dotted IP address. If the hostname is omitted, the platform's
default behavior is used when binding the listening socket (''
is passed to socket.bind() as the hostname portion of the
address).
</description>
</key>
<key name="transaction-timeout" datatype="integer"
required="no">
<description>
The maximum amount of time to wait for a transaction to commit
after acquiring the storage lock, specified in seconds. If the
transaction takes too long, the client connection will be closed
and the transaction aborted.
</description>
</key>
<key name="authentication-protocol" required="no">
<description>
The name of the protocol used for authentication. The
only protocol provided with ZEO is "digest," but extensions
may provide other protocols.
</description>
</key>
<key name="authentication-database" required="no">
<description>
The path of the database containing authentication credentials.
</description>
</key>
<key name="authentication-realm" required="no">
<description>
The authentication realm of the server. Some authentication
schemes use a realm to identify the logical set of usernames
that are accepted by this server.
</description>
</key>
<key name="pid-filename" datatype="existing-dirpath"
required="no">
<description>
The full path to the file in which to write the ZEO server's Process ID
at startup. If omitted, $INSTANCE/var/ZEO.pid is used.
</description>
<metadefault>$INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid)</metadefault>
</key>
<!-- DM 2006-06-12: added option -->
<key name="drop-cache-rather-verify" datatype="boolean"
required="no" default="false">
<description>
indicates that the cache should be dropped rather than
verified when the verification optimization is not
available (e.g. when the ZEO server restarted).
</description>
</key>
</sectiontype>
</component>
src/ZEO/hash.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2008 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""In Python 2.6, the "sha" and "md5" modules have been deprecated
in favor of using hashlib for both. This class allows for compatibility
between versions."""
try:
import hashlib
sha1 = hashlib.sha1
new = sha1
except ImportError:
import sha
sha1 = sha.new
new = sha1
digest_size = sha.digest_size
src/ZEO/interfaces.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2006 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import zope.interface
class StaleCache(object):
"""A ZEO cache is stale and requires verification.
"""
def __init__(self, storage):
self.storage = storage
class IServeable(zope.interface.Interface):
"""Interface provided by storages that can be served by ZEO
"""
def getTid(oid):
"""The last transaction to change an object
Return the transaction id of the last transaction that committed a
change to an object with the given object id.
"""
def tpc_transaction():
"""The current transaction being committed.
If a storage is participating in a two-phase commit, then
return the transaction (object) being committed. Otherwise
return None.
"""
def lastInvalidations(size):
"""Get recent transaction invalidations
This method is optional and is used to get invalidations
performed by the most recent transactions.
An iterable of up to size entries must be returned, where each
entry is a transaction id and a sequence of object-id/empty-string
pairs describing the objects written by the
transaction, in chronological order.
"""
src/ZEO/monitor.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Monitor behavior of ZEO server and record statistics.
$Id$
"""
import asyncore
import socket
import time
import logging
zeo_version = 'unknown'
try:
import pkg_resources
except ImportError:
pass
else:
zeo_dist = pkg_resources.working_set.find(
pkg_resources.Requirement.parse('ZODB3')
)
if zeo_dist is not None:
zeo_version = zeo_dist.version
class StorageStats:
"""Per-storage usage statistics."""
def __init__(self, connections=None):
self.connections = connections
self.loads = 0
self.stores = 0
self.commits = 0
self.aborts = 0
self.active_txns = 0
self.verifying_clients = 0
self.lock_time = None
self.conflicts = 0
self.conflicts_resolved = 0
self.start = time.ctime()
@property
def clients(self):
return len(self.connections)
def parse(self, s):
# parse the dump format
lines = s.split("\n")
for line in lines:
field, value = line.split(":", 1)
if field == "Server started":
self.start = value
elif field == "Clients":
# Hack because we use this both on the server and on
# the client where there are no connections.
self.connections = [0] * int(value)
elif field == "Clients verifying":
self.verifying_clients = int(value)
elif field == "Active transactions":
self.active_txns = int(value)
elif field == "Commit lock held for":
# This assumes
self.lock_time = time.time() - int(value)
elif field == "Commits":
self.commits = int(value)
elif field == "Aborts":
self.aborts = int(value)
elif field == "Loads":
self.loads = int(value)
elif field == "Stores":
self.stores = int(value)
elif field == "Conflicts":
self.conflicts = int(value)
elif field == "Conflicts resolved":
self.conflicts_resolved = int(value)
def dump(self, f):
print >> f, "Server started:", self.start
print >> f, "Clients:", self.clients
print >> f, "Clients verifying:", self.verifying_clients
print >> f, "Active transactions:", self.active_txns
if self.lock_time:
howlong = time.time() - self.lock_time
print >> f, "Commit lock held for:", int(howlong)
print >> f, "Commits:", self.commits
print >> f, "Aborts:", self.aborts
print >> f, "Loads:", self.loads
print >> f, "Stores:", self.stores
print >> f, "Conflicts:", self.conflicts
print >> f, "Conflicts resolved:", self.conflicts_resolved
class StatsClient(asyncore.dispatcher):
def __init__(self, sock, addr):
asyncore.dispatcher.__init__(self, sock)
self.buf = []
self.closed = 0
def close(self):
self.closed = 1
# The socket is closed after all the data is written.
# See handle_write().
def write(self, s):
self.buf.append(s)
def writable(self):
return len(self.buf)
def readable(self):
return 0
def handle_write(self):
s = "".join(self.buf)
self.buf = []
n = self.socket.send(s)
if n < len(s):
self.buf.append(s[:n])
if self.closed and not self.buf:
asyncore.dispatcher.close(self)
class StatsServer(asyncore.dispatcher):
StatsConnectionClass = StatsClient
def __init__(self, addr, stats):
asyncore.dispatcher.__init__(self)
self.addr = addr
self.stats = stats
if type(self.addr) == tuple:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
logger = logging.getLogger('ZEO.monitor')
logger.info("listening on %s", repr(self.addr))
self.bind(self.addr)
self.listen(5)
def writable(self):
return 0
def readable(self):
return 1
def handle_accept(self):
try:
sock, addr = self.accept()
except socket.error:
return
f = self.StatsConnectionClass(sock, addr)
self.dump(f)
f.close()
def dump(self, f):
print >> f, "ZEO monitor server version %s" % zeo_version
print >> f, time.ctime()
print >> f
L = self.stats.keys()
L.sort()
for k in L:
stats = self.stats[k]
print >> f, "Storage:", k
stats.dump(f)
print >> f
src/ZEO/protocol.txt deleted 100644 → 0
View file @ 56c7dd30
ZEO Network Protocol (sans authentication)
==========================================
This document describes the ZEO network protocol. It assumes that the
optional authentication protocol isn't used. At the lowest
level, the protocol consists of sized messages. All communication
between the client and server consists of sized messages. A sized
message consists of a 4-byte unsigned big-endian content length,
followed by the content. There are two subprotocols, for protocol
negotiation, and for normal operation. The normal operation protocol
is a basic RPC protocol.
In the protocol negotiation phase, the server sends a protocol
identifier to the client. The client chooses a protocol to use to the
server. The client or the server can fail if it doesn't like the
protocol string sent by the other party. After sending their protocol
strings, the client and server switch to RPC mode.
The RPC protocol uses messages that are pickled tuples consisting of:
message_id
The message id is used to match replies with requests, allowing
multiple outstanding synchronous requests.
async_flag
An integer 0 for a regular (2-way) request and 1 for a one-way
request. Two-way requests have a reply. One way requests don't.
ZRS tries to use as many one-way requests as possible to avoid
network round trips.
name
The name of a method to call. If this is the special string
".reply", then the message is interpreted as a return from a
synchronous call.
args
A tuple of positional arguments or returned values.
After making a connection and negotiating the protocol, the following
interactions occur:
- The client requests the authentication protocol by calling
getAuthProtocol. For this discussion, we'll assume the server
returns None. Note that if the server doesn't require
authentication, this step is optional.
- The client calls register passing a storage identifier and a
read-only flag. The server doesn't return a value, but it may raise
an exception either if the storage doesn't exist, or if the
storage is readonly and the read-only flag passed by the client is
false.
At this point, the client and server send each other messages as
needed. The client may make regular or one-way calls to the
server. The server sends replies and one-way calls to the client.
src/ZEO/runzeo.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Start the ZEO storage server.
Usage: %s [-C URL] [-a ADDRESS] [-f FILENAME] [-h]
Options:
-C/--configuration URL -- configuration file or URL
-a/--address ADDRESS -- server address of the form PORT, HOST:PORT, or PATH
(a PATH must contain at least one "/")
-f/--filename FILENAME -- filename for FileStorage
-t/--timeout TIMEOUT -- transaction timeout in seconds (default no timeout)
-h/--help -- print this usage message and exit
-m/--monitor ADDRESS -- address of monitor server ([HOST:]PORT or PATH)
--pid-file PATH -- relative path to output file containing this process's pid;
default $(INSTANCE_HOME)/var/ZEO.pid but only if envar
INSTANCE_HOME is defined
Unless -C is specified, -a and -f are required.
"""
# The code here is designed to be reused by other, similar servers.
# For the forseeable future, it must work under Python 2.1 as well as
# 2.2 and above.
import asyncore
import os
import sys
import signal
import socket
import logging
import ZConfig.datatypes
import ZEO
from zdaemon.zdoptions import ZDOptions
logger = logging.getLogger('ZEO.runzeo')
_pid = str(os.getpid())
def log(msg, level=logging.INFO, exc_info=False):
"""Internal: generic logging function."""
message = "(%s) %s" % (_pid, msg)
logger.log(level, message, exc_info=exc_info)
def parse_binding_address(arg):
# Caution: Not part of the official ZConfig API.
obj = ZConfig.datatypes.SocketBindingAddress(arg)
return obj.family, obj.address
def windows_shutdown_handler():
# Called by the signal mechanism on Windows to perform shutdown.
import asyncore
asyncore.close_all()
class ZEOOptionsMixin:
storages = None
def handle_address(self, arg):
self.family, self.address = parse_binding_address(arg)
def handle_monitor_address(self, arg):
self.monitor_family, self.monitor_address = parse_binding_address(arg)
def handle_filename(self, arg):
from ZODB.config import FileStorage # That's a FileStorage *opener*!
class FSConfig:
def __init__(self, name, path):
self._name = name
self.path = path
self.stop = None
def getSectionName(self):
return self._name
if not self.storages:
self.storages = []
name = str(1 + len(self.storages))
conf = FileStorage(FSConfig(name, arg))
self.storages.append(conf)
testing_exit_immediately = False
def handle_test(self, *args):
self.testing_exit_immediately = True
def add_zeo_options(self):
self.add(None, None, None, "test", self.handle_test)
self.add(None, None, "a:", "address=", self.handle_address)
self.add(None, None, "f:", "filename=", self.handle_filename)
self.add("family", "zeo.address.family")
self.add("address", "zeo.address.address",
required="no server address specified; use -a or -C")
self.add("read_only", "zeo.read_only", default=0)
self.add("invalidation_queue_size", "zeo.invalidation_queue_size",
default=100)
self.add("invalidation_age", "zeo.invalidation_age")
self.add("transaction_timeout", "zeo.transaction_timeout",
"t:", "timeout=", float)
self.add("monitor_address", "zeo.monitor_address.address",
"m:", "monitor=", self.handle_monitor_address)
self.add('auth_protocol', 'zeo.authentication_protocol',
None, 'auth-protocol=', default=None)
self.add('auth_database', 'zeo.authentication_database',
None, 'auth-database=')
self.add('auth_realm', 'zeo.authentication_realm',
None, 'auth-realm=')
self.add('pid_file', 'zeo.pid_filename',
None, 'pid-file=')
class ZEOOptions(ZDOptions, ZEOOptionsMixin):
__doc__ = __doc__
logsectionname = "eventlog"
schemadir = os.path.dirname(ZEO.__file__)
def __init__(self):
ZDOptions.__init__(self)
self.add_zeo_options()
self.add("storages", "storages",
required="no storages specified; use -f or -C")
def realize(self, *a, **k):
ZDOptions.realize(self, *a, **k)
nunnamed = [s for s in self.storages if s.name is None]
if nunnamed:
if len(nunnamed) > 1:
return self.usage("No more than one storage may be unnamed.")
if [s for s in self.storages if s.name == '1']:
return self.usage(
"Can't have an unnamed storage and a storage named 1.")
for s in self.storages:
if s.name is None:
s.name = '1'
break
class ZEOServer:
def __init__(self, options):
self.options = options
def main(self):
self.setup_default_logging()
self.check_socket()
self.clear_socket()
self.make_pidfile()
try:
self.open_storages()
self.setup_signals()
self.create_server()
self.loop_forever()
finally:
self.server.close()
self.clear_socket()
self.remove_pidfile()
def setup_default_logging(self):
if self.options.config_logger is not None:
return
# No log file is configured; default to stderr.
root = logging.getLogger()
root.setLevel(logging.INFO)
fmt = logging.Formatter(
"------\n%(asctime)s %(levelname)s %(name)s %(message)s",
"%Y-%m-%dT%H:%M:%S")
handler = logging.StreamHandler()
handler.setFormatter(fmt)
root.addHandler(handler)
def check_socket(self):
if (isinstance(self.options.address, tuple) and
self.options.address[1] is None):
self.options.address = self.options.address[0], 0
return
if self.can_connect(self.options.family, self.options.address):
self.options.usage("address %s already in use" %
repr(self.options.address))
def can_connect(self, family, address):
s = socket.socket(family, socket.SOCK_STREAM)
try:
s.connect(address)
except socket.error:
return 0
else:
s.close()
return 1
def clear_socket(self):
if isinstance(self.options.address, type("")):
try:
os.unlink(self.options.address)
except os.error:
pass
def open_storages(self):
self.storages = {}
for opener in self.options.storages:
log("opening storage %r using %s"
% (opener.name, opener.__class__.__name__))
self.storages[opener.name] = opener.open()
def setup_signals(self):
"""Set up signal handlers.
The signal handler for SIGFOO is a method handle_sigfoo().
If no handler method is defined for a signal, the signal
action is not changed from its initial value. The handler
method is called without additional arguments.
"""
if os.name != "posix":
if os.name == "nt":
self.setup_win32_signals()
return
if hasattr(signal, 'SIGXFSZ'):
signal.signal(signal.SIGXFSZ, signal.SIG_IGN) # Special case
init_signames()
for sig, name in signames.items():
method = getattr(self, "handle_" + name.lower(), None)
if method is not None:
def wrapper(sig_dummy, frame_dummy, method=method):
method()
signal.signal(sig, wrapper)
def setup_win32_signals(self):
# Borrow the Zope Signals package win32 support, if available.
# Signals does a check/log for the availability of pywin32.
try:
import Signals.Signals
except ImportError:
logger.debug("Signals package not found. "
"Windows-specific signal handler "
"will *not* be installed.")
return
SignalHandler = Signals.Signals.SignalHandler
if SignalHandler is not None: # may be None if no pywin32.
SignalHandler.registerHandler(signal.SIGTERM,
windows_shutdown_handler)
SignalHandler.registerHandler(signal.SIGINT,
windows_shutdown_handler)
SIGUSR2 = 12 # not in signal module on Windows.
SignalHandler.registerHandler(SIGUSR2, self.handle_sigusr2)
def create_server(self):
self.server = create_server(self.storages, self.options)
def loop_forever(self):
if self.options.testing_exit_immediately:
print "testing exit immediately"
else:
self.server.loop()
def handle_sigterm(self):
log("terminated by SIGTERM")
sys.exit(0)
def handle_sigint(self):
log("terminated by SIGINT")
sys.exit(0)
def handle_sighup(self):
log("restarted by SIGHUP")
sys.exit(1)
def handle_sigusr2(self):
# log rotation signal - do the same as Zope 2.7/2.8...
if self.options.config_logger is None or os.name not in ("posix", "nt"):
log("received SIGUSR2, but it was not handled!",
level=logging.WARNING)
return
loggers = [self.options.config_logger]
if os.name == "posix":
for l in loggers:
l.reopen()
log("Log files reopened successfully", level=logging.INFO)
else: # nt - same rotation code as in Zope's Signals/Signals.py
for l in loggers:
for f in l.handler_factories:
handler = f()
if hasattr(handler, 'rotate') and callable(handler.rotate):
handler.rotate()
log("Log files rotation complete", level=logging.INFO)
def _get_pidfile(self):
pidfile = self.options.pid_file
# 'pidfile' is marked as not required.
if not pidfile:
# Try to find a reasonable location if the pidfile is not
# set. If we are running in a Zope environment, we can
# safely assume INSTANCE_HOME.
instance_home = os.environ.get("INSTANCE_HOME")
if not instance_home:
# If all our attempts failed, just log a message and
# proceed.
logger.debug("'pidfile' option not set, and 'INSTANCE_HOME' "
"environment variable could not be found. "
"Cannot guess pidfile location.")
return
self.options.pid_file = os.path.join(instance_home,
"var", "ZEO.pid")
def make_pidfile(self):
if not self.options.read_only:
self._get_pidfile()
pidfile = self.options.pid_file
if pidfile is None:
return
pid = os.getpid()
try:
if os.path.exists(pidfile):
os.unlink(pidfile)
f = open(pidfile, 'w')
print >> f, pid
f.close()
log("created PID file '%s'" % pidfile)
except IOError:
logger.error("PID file '%s' cannot be opened" % pidfile)
def remove_pidfile(self):
if not self.options.read_only:
pidfile = self.options.pid_file
if pidfile is None:
return
try:
if os.path.exists(pidfile):
os.unlink(pidfile)
log("removed PID file '%s'" % pidfile)
except IOError:
logger.error("PID file '%s' could not be removed" % pidfile)
def create_server(storages, options):
from ZEO.StorageServer import StorageServer
return StorageServer(
options.address,
storages,
read_only = options.read_only,
invalidation_queue_size = options.invalidation_queue_size,
invalidation_age = options.invalidation_age,
transaction_timeout = options.transaction_timeout,
monitor_address = options.monitor_address,
auth_protocol = options.auth_protocol,
auth_database = options.auth_database,
auth_realm = options.auth_realm,
)
# Signal names
signames = None
def signame(sig):
"""Return a symbolic name for a signal.
Return "signal NNN" if there is no corresponding SIG name in the
signal module.
"""
if signames is None:
init_signames()
return signames.get(sig) or "signal %d" % sig
def init_signames():
global signames
signames = {}
for name, sig in signal.__dict__.items():
k_startswith = getattr(name, "startswith", None)
if k_startswith is None:
continue
if k_startswith("SIG") and not k_startswith("SIG_"):
signames[sig] = name
# Main program
def main(args=None):
options = ZEOOptions()
options.realize(args)
s = ZEOServer(options)
s.main()
if __name__ == "__main__":
main()
src/ZEO/schema.xml deleted 100644 → 0
View file @ 56c7dd30
<schema>
<!-- note that zeoctl.xml is a closely related schema which should
match this schema, but should require the "runner" section -->
<description>
This schema describes the configuration of the ZEO storage server
process.
</description>
<!-- Use the storage types defined by ZODB. -->
<import package="ZODB"/>
<!-- Use the ZEO server information structure. -->
<import package="ZEO"/>
<import package="ZConfig.components.logger"/>
<!-- runner control -->
<import package="zdaemon"/>
<section type="zeo" name="*" required="yes" attribute="zeo" />
<section type="runner" name="*" required="no" attribute="runner" />
<multisection name="*" type="ZODB.storage"
attribute="storages"
required="yes">
<description>
One or more storages that are provided by the ZEO server. The
section names are used as the storage names, and must be unique
within each ZEO storage server. Traditionally, these names
represent small integers starting at '1'.
</description>
</multisection>
<section name="*" type="eventlog" attribute="eventlog" required="no" />
</schema>
src/ZEO/scripts/README.txt deleted 100644 → 0
View file @ 56c7dd30
This directory contains a collection of utilities for working with
ZEO. Some are more useful than others. If you install ZODB using
distutils ("python setup.py install"), some of these will be
installed.
Unless otherwise noted, these scripts are invoked with the name of the
Data.fs file as their only argument. Example: checkbtrees.py data.fs.
parsezeolog.py -- parse BLATHER logs from ZEO server
This script may be obsolete. It has not been tested against the
current log output of the ZEO server.
Reports on the time and size of transactions committed by a ZEO
server, by inspecting log messages at BLATHER level.
timeout.py -- script to test transaction timeout
usage: timeout.py address delay [storage-name]
This script connects to a storage, begins a transaction, calls store()
and tpc_vote(), and then sleeps forever. This should trigger the
transaction timeout feature of the server.
zeopack.py -- pack a ZEO server
The script connects to a server and calls pack() on a specific
storage. See the script for usage details.
zeoreplay.py -- experimental script to replay transactions from a ZEO log
Like parsezeolog.py, this may be obsolete because it was written
against an earlier version of the ZEO server. See the script for
usage details.
zeoup.py
usage: zeoup.py [options]
The test will connect to a ZEO server, load the root object, and
attempt to update the zeoup counter in the root. It will report
success if it updates to counter or if it gets a ConflictError. A
ConflictError is considered a success, because the client was able to
start a transaction.
See the script for details about the options.
zeoserverlog.py -- analyze ZEO server log for performance statistics
See the module docstring for details; there are a large number of
options. New in ZODB3 3.1.4.
zeoqueue.py -- report number of clients currently waiting in the ZEO queue
See the module docstring for details.
src/ZEO/scripts/cache_simul.py deleted 100755 → 0
View file @ 56c7dd30
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001-2005 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Cache simulation.
Usage: simul.py [-s size] tracefile
Options:
-s size: cache size in MB (default 20 MB)
-i: summarizing interval in minutes (default 15; max 60)
-r: rearrange factor
Note:
- The simulation isn't perfect.
- The simulation will be far off if the trace file
was created starting with a non-empty cache
"""
import bisect
import getopt
import struct
import re
import sys
import ZEO.cache
from ZODB.utils import z64, u64
# we assign ctime locally to facilitate test replacement!
from time import ctime
def usage(msg):
print >> sys.stderr, msg
print >> sys.stderr, __doc__
def main(args=None):
if args is None:
args = sys.argv[1:]
# Parse options.
MB = 1<<20
cachelimit = 20*MB
rearrange = 0.8
simclass = CircularCacheSimulation
interval_step = 15
try:
opts, args = getopt.getopt(args, "s:i:r:")
except getopt.error, msg:
usage(msg)
return 2
for o, a in opts:
if o == '-s':
cachelimit = int(float(a)*MB)
elif o == '-i':
interval_step = int(a)
elif o == '-r':
rearrange = float(a)
else:
assert False, (o, a)
interval_step *= 60
if interval_step <= 0:
interval_step = 60
elif interval_step > 3600:
interval_step = 3600
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file.
if filename.endswith(".gz"):
# Open gzipped file.
try:
import gzip
except ImportError:
print >> sys.stderr, "can't read gzipped files (no module gzip)"
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
elif filename == "-":
# Read from stdin.
f = sys.stdin
else:
# Open regular file.
try:
f = open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
# Create simulation object.
sim = simclass(cachelimit, rearrange)
interval_sim = simclass(cachelimit, rearrange)
# Print output header.
sim.printheader()
# Read trace file, simulating cache behavior.
f_read = f.read
unpack = struct.unpack
FMT = ">iiH8s8s"
FMT_SIZE = struct.calcsize(FMT)
assert FMT_SIZE == 26
last_interval = None
while 1:
# Read a record and decode it.
r = f_read(FMT_SIZE)
if len(r) < FMT_SIZE:
break
ts, code, oidlen, start_tid, end_tid = unpack(FMT, r)
if ts == 0:
# Must be a misaligned record caused by a crash; skip 8 bytes
# and try again. Why 8? Lost in the mist of history.
f.seek(f.tell() - FMT_SIZE + 8)
continue
oid = f_read(oidlen)
if len(oid) < oidlen:
break
# Decode the code.
dlen, version, code = ((code & 0x7fffff00) >> 8,
code & 0x80,
code & 0x7e)
# And pass it to the simulation.
this_interval = int(ts)/interval_step
if this_interval != last_interval:
if last_interval is not None:
interval_sim.report()
interval_sim.restart()
if not interval_sim.warm:
sim.restart()
last_interval = this_interval
sim.event(ts, dlen, version, code, oid, start_tid, end_tid)
interval_sim.event(ts, dlen, version, code, oid, start_tid, end_tid)
f.close()
# Finish simulation.
interval_sim.report()
sim.finish()
class Simulation(object):
"""Base class for simulations.
The driver program calls: event(), printheader(), finish().
The standard event() method calls these additional methods:
write(), load(), inval(), report(), restart(); the standard
finish() method also calls report().
"""
def __init__(self, cachelimit, rearrange):
self.cachelimit = cachelimit
self.rearrange = rearrange
# Initialize global statistics.
self.epoch = None
self.warm = False
self.total_loads = 0
self.total_hits = 0 # subclass must increment
self.total_invals = 0 # subclass must increment
self.total_writes = 0
if not hasattr(self, "extras"):
self.extras = (self.extraname,)
self.format = self.format + " %7s" * len(self.extras)
# Reset per-run statistics and set up simulation data.
self.restart()
def restart(self):
# Reset per-run statistics.
self.loads = 0
self.hits = 0 # subclass must increment
self.invals = 0 # subclass must increment
self.writes = 0
self.ts0 = None
def event(self, ts, dlen, _version, code, oid,
start_tid, end_tid):
# Record first and last timestamp seen.
if self.ts0 is None:
self.ts0 = ts
if self.epoch is None:
self.epoch = ts
self.ts1 = ts
# Simulate cache behavior. Caution: the codes in the trace file
# record whether the actual cache missed or hit on each load, but
# that bears no necessary relationship to whether the simulated cache
# will hit or miss. Relatedly, if the actual cache needed to store
# an object, the simulated cache may not need to (it may already
# have the data).
action = code & 0x70
if action & 0x20:
# Load.
self.loads += 1
self.total_loads += 1
# Asserting that dlen is 0 iff it's a load miss.
# assert (dlen == 0) == (code in (0x20, 0x24))
self.load(oid, dlen, start_tid, code)
elif action & 0x40:
# Store.
assert dlen
self.write(oid, dlen, start_tid, end_tid)
elif action & 0x10:
# Invalidate.
self.inval(oid, start_tid)
elif action == 0x00:
# Restart.
self.restart()
else:
raise ValueError("unknown trace code 0x%x" % code)
def write(self, oid, size, start_tid, end_tid):
pass
def load(self, oid, size, start_tid, code):
# Must increment .hits and .total_hits as appropriate.
pass
def inval(self, oid, start_tid):
# Must increment .invals and .total_invals as appropriate.
pass
format = "%12s %6s %7s %7s %6s %6s %7s"
# Subclass should override extraname to name known instance variables;
# if extraname is 'foo', both self.foo and self.total_foo must exist:
extraname = "*** please override ***"
def printheader(self):
print "%s, cache size %s bytes" % (self.__class__.__name__,
addcommas(self.cachelimit))
self.extraheader()
extranames = tuple([s.upper() for s in self.extras])
args = ("START TIME", "DUR.", "LOADS", "HITS",
"INVALS", "WRITES", "HITRATE") + extranames
print self.format % args
def extraheader(self):
pass
nreports = 0
def report(self):
if not hasattr(self, 'ts1'):
return
self.nreports += 1
args = (ctime(self.ts0)[4:-8],
duration(self.ts1 - self.ts0),
self.loads, self.hits, self.invals, self.writes,
hitrate(self.loads, self.hits))
args += tuple([getattr(self, name) for name in self.extras])
print self.format % args
def finish(self):
# Make sure that the last line of output ends with "OVERALL". This
# makes it much easier for another program parsing the output to
# find summary statistics.
print '-'*74
if self.nreports < 2:
self.report()
else:
self.report()
args = (
ctime(self.epoch)[4:-8],
duration(self.ts1 - self.epoch),
self.total_loads,
self.total_hits,
self.total_invals,
self.total_writes,
hitrate(self.total_loads, self.total_hits))
args += tuple([getattr(self, "total_" + name)
for name in self.extras])
print self.format % args
# For use in CircularCacheSimulation.
class CircularCacheEntry(object):
__slots__ = (
# object key: an (oid, start_tid) pair, where start_tid is the
# tid of the transaction that created this revision of oid
'key',
# tid of transaction that created the next revision; z64 iff
# this is the current revision
'end_tid',
# Offset from start of file to the object's data record; this
# includes all overhead bytes (status byte, size bytes, etc).
'offset',
)
def __init__(self, key, end_tid, offset):
self.key = key
self.end_tid = end_tid
self.offset = offset
from ZEO.cache import ZEC_HEADER_SIZE
class CircularCacheSimulation(Simulation):
"""Simulate the ZEO 3.0 cache."""
# The cache is managed as a single file with a pointer that
# goes around the file, circularly, forever. New objects
# are written at the current pointer, evicting whatever was
# there previously.
extras = "evicts", "inuse"
evicts = 0
def __init__(self, cachelimit, rearrange):
from ZEO import cache
Simulation.__init__(self, cachelimit, rearrange)
self.total_evicts = 0 # number of cache evictions
# Current offset in file.
self.offset = ZEC_HEADER_SIZE
# Map offset in file to (size, CircularCacheEntry) pair, or to
# (size, None) if the offset starts a free block.
self.filemap = {ZEC_HEADER_SIZE: (self.cachelimit - ZEC_HEADER_SIZE,
None)}
# Map key to CircularCacheEntry. A key is an (oid, tid) pair.
self.key2entry = {}
# Map oid to tid of current revision.
self.current = {}
# Map oid to list of (start_tid, end_tid) pairs in sorted order.
# Used to find matching key for load of non-current data.
self.noncurrent = {}
# The number of overhead bytes needed to store an object pickle
# on disk (all bytes beyond those needed for the object pickle).
self.overhead = ZEO.cache.allocated_record_overhead
# save evictions so we can replay them, if necessary
self.evicted = {}
def restart(self):
Simulation.restart(self)
if self.evicts:
self.warm = True
self.evicts = 0
self.evicted_hit = self.evicted_miss = 0
evicted_hit = evicted_miss = 0
def load(self, oid, size, tid, code):
if (code == 0x20) or (code == 0x22):
# Trying to load current revision.
if oid in self.current: # else it's a cache miss
self.hits += 1
self.total_hits += 1
tid = self.current[oid]
entry = self.key2entry[(oid, tid)]
offset_offset = self.offset - entry.offset
if offset_offset < 0:
offset_offset += self.cachelimit
assert offset_offset >= 0
if offset_offset > self.rearrange * self.cachelimit:
# we haven't accessed it in a while. Move it forward
size = self.filemap[entry.offset][0]
self._remove(*entry.key)
self.add(oid, size, tid)
elif oid in self.evicted:
size, e = self.evicted[oid]
self.write(oid, size, e.key[1], z64, 1)
self.evicted_hit += 1
else:
self.evicted_miss += 1
return
# May or may not be trying to load current revision.
cur_tid = self.current.get(oid)
if cur_tid == tid:
self.hits += 1
self.total_hits += 1
return
# It's a load for non-current data. Do we know about this oid?
L = self.noncurrent.get(oid)
if L is None:
return # cache miss
i = bisect.bisect_left(L, (tid, None))
if i == 0:
# This tid is smaller than any we know about -- miss.
return
lo, hi = L[i-1]
assert lo < tid
if tid > hi:
# No data in the right tid range -- miss.
return
# Cache hit.
self.hits += 1
self.total_hits += 1
# (oid, tid) is in the cache. Remove it: take it out of key2entry,
# and in `filemap` mark the space it occupied as being free. The
# caller is responsible for removing it from `current` or `noncurrent`.
def _remove(self, oid, tid):
key = oid, tid
e = self.key2entry.pop(key)
pos = e.offset
size, _e = self.filemap[pos]
assert e is _e
self.filemap[pos] = size, None
def _remove_noncurrent_revisions(self, oid):
noncurrent_list = self.noncurrent.get(oid)
if noncurrent_list:
self.invals += len(noncurrent_list)
self.total_invals += len(noncurrent_list)
for start_tid, end_tid in noncurrent_list:
self._remove(oid, start_tid)
del self.noncurrent[oid]
def inval(self, oid, tid):
if tid == z64:
# This is part of startup cache verification: forget everything
# about this oid.
self._remove_noncurrent_revisions(oid)
if oid in self.evicted:
del self.evicted[oid]
cur_tid = self.current.get(oid)
if cur_tid is None:
# We don't have current data, so nothing more to do.
return
# We had current data for oid, but no longer.
self.invals += 1
self.total_invals += 1
del self.current[oid]
if tid == z64:
# Startup cache verification: forget this oid entirely.
self._remove(oid, cur_tid)
return
# Our current data becomes non-current data.
# Add the validity range to the list of non-current data for oid.
assert cur_tid < tid
L = self.noncurrent.setdefault(oid, [])
bisect.insort_left(L, (cur_tid, tid))
# Update the end of oid's validity range in its CircularCacheEntry.
e = self.key2entry[oid, cur_tid]
assert e.end_tid == z64
e.end_tid = tid
def write(self, oid, size, start_tid, end_tid, evhit=0):
if end_tid == z64:
# Storing current revision.
if oid in self.current: # we already have it in cache
if evhit:
import pdb; pdb.set_trace()
raise ValueError('WTF')
return
self.current[oid] = start_tid
self.writes += 1
self.total_writes += 1
self.add(oid, size, start_tid)
return
if evhit:
import pdb; pdb.set_trace()
raise ValueError('WTF')
# Storing non-current revision.
L = self.noncurrent.setdefault(oid, [])
p = start_tid, end_tid
if p in L:
return # we already have it in cache
bisect.insort_left(L, p)
self.writes += 1
self.total_writes += 1
self.add(oid, size, start_tid, end_tid)
# Add `oid` to the cache, evicting objects as needed to make room.
# This updates `filemap` and `key2entry`; it's the caller's
# responsibilty to update `current` or `noncurrent` appropriately.
def add(self, oid, size, start_tid, end_tid=z64):
key = oid, start_tid
assert key not in self.key2entry
size += self.overhead
avail = self.makeroom(size+1) # see cache.py
e = CircularCacheEntry(key, end_tid, self.offset)
self.filemap[self.offset] = size, e
self.key2entry[key] = e
self.offset += size
# All the space made available must be accounted for in filemap.
excess = avail - size
if excess:
self.filemap[self.offset] = excess, None
# Evict enough objects to make at least `need` contiguous bytes, starting
# at `self.offset`, available. Evicted objects are removed from
# `filemap`, `key2entry`, `current` and `noncurrent`. The caller is
# responsible for adding new entries to `filemap` to account for all
# the freed bytes, and for advancing `self.offset`. The number of bytes
# freed is the return value, and will be >= need.
def makeroom(self, need):
if self.offset + need > self.cachelimit:
self.offset = ZEC_HEADER_SIZE
pos = self.offset
while need > 0:
assert pos < self.cachelimit
size, e = self.filemap.pop(pos)
if e: # there is an object here (else it's already free space)
self.evicts += 1
self.total_evicts += 1
assert pos == e.offset
_e = self.key2entry.pop(e.key)
assert e is _e
oid, start_tid = e.key
if e.end_tid == z64:
del self.current[oid]
self.evicted[oid] = size-self.overhead, e
else:
L = self.noncurrent[oid]
L.remove((start_tid, e.end_tid))
need -= size
pos += size
return pos - self.offset # total number of bytes freed
def report(self):
self.check()
free = used = total = 0
for size, e in self.filemap.itervalues():
total += size
if e:
used += size
else:
free += size
self.inuse = round(100.0 * used / total, 1)
self.total_inuse = self.inuse
Simulation.report(self)
#print self.evicted_hit, self.evicted_miss
def check(self):
oidcount = 0
pos = ZEC_HEADER_SIZE
while pos < self.cachelimit:
size, e = self.filemap[pos]
if e:
oidcount += 1
assert self.key2entry[e.key].offset == pos
pos += size
assert oidcount == len(self.key2entry)
assert pos == self.cachelimit
def dump(self):
print len(self.filemap)
L = list(self.filemap)
L.sort()
for k in L:
v = self.filemap[k]
print k, v[0], repr(v[1])
def roundup(size):
k = MINSIZE
while k < size:
k += k
return k
def hitrate(loads, hits):
if loads < 1:
return 'n/a'
return "%5.1f%%" % (100.0 * hits / loads)
def duration(secs):
mm, ss = divmod(secs, 60)
hh, mm = divmod(mm, 60)
if hh:
return "%d:%02d:%02d" % (hh, mm, ss)
if mm:
return "%d:%02d" % (mm, ss)
return "%d" % ss
nre = re.compile('([=-]?)(\d+)([.]\d*)?').match
def addcommas(n):
sign, s, d = nre(str(n)).group(1, 2, 3)
if d == '.0':
d = ''
result = s[-3:]
s = s[:-3]
while s:
result = s[-3:]+','+result
s = s[:-3]
return (sign or '') + result + (d or '')
import random
def maybe(f, p=0.5):
if random.random() < p:
f()
if __name__ == "__main__":
sys.exit(main())
src/ZEO/scripts/cache_stats.py deleted 100755 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Trace file statistics analyzer.
Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
"""
"""File format:
Each record is 26 bytes, plus a variable number of bytes to store an oid,
with the following layout. Numbers are big-endian integers.
Offset Size Contents
0 4 timestamp (seconds since 1/1/1970)
4 3 data size, in 256-byte increments, rounded up
7 1 code (see below)
8 2 object id length
10 8 start tid
18 8 end tid
26 variable object id
The code at offset 7 packs three fields:
Mask bits Contents
0x80 1 set if there was a non-empty version string
0x7e 6 function and outcome code
0x01 1 current cache file (0 or 1)
The "current cache file" bit is no longer used; it refers to a 2-file
cache scheme used before ZODB 3.3.
The function and outcome codes are documented in detail at the end of
this file in the 'explain' dictionary. Note that the keys there (and
also the arguments to _trace() in ClientStorage.py) are 'code & 0x7e',
i.e. the low bit is always zero.
"""
import sys
import time
import getopt
import struct
# we assign ctime locally to facilitate test replacement!
from time import ctime
def usage(msg):
print >> sys.stderr, msg
print >> sys.stderr, __doc__
def main(args=None):
if args is None:
args = sys.argv[1:]
# Parse options
verbose = False
quiet = False
dostats = True
print_size_histogram = False
print_histogram = False
interval = 15*60 # Every 15 minutes
heuristic = False
try:
opts, args = getopt.getopt(args, "hi:qsSvX")
except getopt.error, msg:
usage(msg)
return 2
for o, a in opts:
if o == '-h':
print_histogram = True
elif o == "-i":
interval = int(60 * float(a))
if interval <= 0:
interval = 60
elif interval > 3600:
interval = 3600
elif o == "-q":
quiet = True
verbose = False
elif o == "-s":
print_size_histogram = True
elif o == "-S":
dostats = False
elif o == "-v":
verbose = True
elif o == '-X':
heuristic = True
else:
assert False, (o, opts)
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file
if filename.endswith(".gz"):
# Open gzipped file
try:
import gzip
except ImportError:
print >> sys.stderr, "can't read gzipped files (no module gzip)"
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
elif filename == '-':
# Read from stdin
f = sys.stdin
else:
# Open regular file
try:
f = open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
rt0 = time.time()
bycode = {} # map code to count of occurrences
byinterval = {} # map code to count in current interval
records = 0 # number of trace records read
versions = 0 # number of trace records with versions
datarecords = 0 # number of records with dlen set
datasize = 0L # sum of dlen across records with dlen set
oids = {} # map oid to number of times it was loaded
bysize = {} # map data size to number of loads
bysizew = {} # map data size to number of writes
total_loads = 0
t0 = None # first timestamp seen
te = None # most recent timestamp seen
h0 = None # timestamp at start of current interval
he = None # timestamp at end of current interval
thisinterval = None # generally te//interval
f_read = f.read
unpack = struct.unpack
FMT = ">iiH8s8s"
FMT_SIZE = struct.calcsize(FMT)
assert FMT_SIZE == 26
# Read file, gathering statistics, and printing each record if verbose.
print ' '*16, "%7s %7s %7s %7s" % ('loads', 'hits', 'inv(h)', 'writes'),
print 'hitrate'
try:
while 1:
r = f_read(FMT_SIZE)
if len(r) < FMT_SIZE:
break
ts, code, oidlen, start_tid, end_tid = unpack(FMT, r)
if ts == 0:
# Must be a misaligned record caused by a crash.
if not quiet:
print "Skipping 8 bytes at offset", f.tell() - FMT_SIZE
f.seek(f.tell() - FMT_SIZE + 8)
continue
oid = f_read(oidlen)
if len(oid) < oidlen:
break
records += 1
if t0 is None:
t0 = ts
thisinterval = t0 // interval
h0 = he = ts
te = ts
if ts // interval != thisinterval:
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts // interval
h0 = ts
he = ts
dlen, code = (code & 0x7fffff00) >> 8, code & 0xff
if dlen:
datarecords += 1
datasize += dlen
if code & 0x80:
version = 'V'
versions += 1
else:
version = '-'
code &= 0x7e
bycode[code] = bycode.get(code, 0) + 1
byinterval[code] = byinterval.get(code, 0) + 1
if dlen:
if code & 0x70 == 0x20: # All loads
bysize[dlen] = d = bysize.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
elif code & 0x70 == 0x50: # All stores
bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
if verbose:
print "%s %02x %s %016x %016x %c%s" % (
ctime(ts)[4:-5],
code,
oid_repr(oid),
U64(start_tid),
U64(end_tid),
version,
dlen and (' '+str(dlen)) or "")
if code & 0x70 == 0x20:
oids[oid] = oids.get(oid, 0) + 1
total_loads += 1
elif code == 0x00: # restart
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts // interval
h0 = he = ts
if not quiet:
print ctime(ts)[4:-5],
print '='*20, "Restart", '='*20
except KeyboardInterrupt:
print "\nInterrupted. Stats so far:\n"
end_pos = f.tell()
f.close()
rte = time.time()
if not quiet:
dumpbyinterval(byinterval, h0, he)
# Error if nothing was read
if not records:
print >> sys.stderr, "No records processed"
return 1
# Print statistics
if dostats:
print
print "Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0)
print "Versions: %s records used a version" % addcommas(versions)
print "First time: %s" % ctime(t0)
print "Last time: %s" % ctime(te)
print "Duration: %s seconds" % addcommas(te-t0)
print "Data recs: %s (%.1f%%), average size %d bytes" % (
addcommas(datarecords),
100.0 * datarecords / records,
datasize / datarecords)
print "Hit rate: %.1f%% (load hits / loads)" % hitrate(bycode)
print
codes = bycode.keys()
codes.sort()
print "%13s %4s %s" % ("Count", "Code", "Function (action)")
for code in codes:
print "%13s %02x %s" % (
addcommas(bycode.get(code, 0)),
code,
explain.get(code) or "*** unknown code ***")
# Print histogram.
if print_histogram:
print
print "Histogram of object load frequency"
total = len(oids)
print "Unique oids: %s" % addcommas(total)
print "Total loads: %s" % addcommas(total_loads)
s = addcommas(total)
width = max(len(s), len("objects"))
fmt = "%5d %" + str(width) + "s %5.1f%% %5.1f%% %5.1f%%"
hdr = "%5s %" + str(width) + "s %6s %6s %6s"
print hdr % ("loads", "objects", "%obj", "%load", "%cum")
cum = 0.0
for binsize, count in histogram(oids):
obj_percent = 100.0 * count / total
load_percent = 100.0 * count * binsize / total_loads
cum += load_percent
print fmt % (binsize, addcommas(count),
obj_percent, load_percent, cum)
# Print size histogram.
if print_size_histogram:
print
print "Histograms of object sizes"
print
dumpbysize(bysizew, "written", "writes")
dumpbysize(bysize, "loaded", "loads")
def dumpbysize(bysize, how, how2):
print
print "Unique sizes %s: %s" % (how, addcommas(len(bysize)))
print "%10s %6s %6s" % ("size", "objs", how2)
sizes = bysize.keys()
sizes.sort()
for size in sizes:
loads = 0
for n in bysize[size].itervalues():
loads += n
print "%10s %6d %6d" % (addcommas(size),
len(bysize.get(size, "")),
loads)
def dumpbyinterval(byinterval, h0, he):
loads = hits = invals = writes = 0
for code in byinterval:
if code & 0x20:
n = byinterval[code]
loads += n
if code in (0x22, 0x26):
hits += n
elif code & 0x40:
writes += byinterval[code]
elif code & 0x10:
if code != 0x10:
invals += byinterval[code]
if loads:
hr = "%5.1f%%" % (100.0 * hits / loads)
else:
hr = 'n/a'
print "%s-%s %7s %7s %7s %7s %7s" % (
ctime(h0)[4:-8], ctime(he)[14:-8],
loads, hits, invals, writes, hr)
def hitrate(bycode):
loads = hits = 0
for code in bycode:
if code & 0x70 == 0x20:
n = bycode[code]
loads += n
if code in (0x22, 0x26):
hits += n
if loads:
return 100.0 * hits / loads
else:
return 0.0
def histogram(d):
bins = {}
for v in d.itervalues():
bins[v] = bins.get(v, 0) + 1
L = bins.items()
L.sort()
return L
def U64(s):
return struct.unpack(">Q", s)[0]
def oid_repr(oid):
if isinstance(oid, str) and len(oid) == 8:
return '%16x' % U64(oid)
else:
return repr(oid)
def addcommas(n):
sign, s = '', str(n)
if s[0] == '-':
sign, s = '-', s[1:]
i = len(s) - 3
while i > 0:
s = s[:i] + ',' + s[i:]
i -= 3
return sign + s
explain = {
# The first hex digit shows the operation, the second the outcome.
# If the second digit is in "02468" then it is a 'miss'.
# If it is in "ACE" then it is a 'hit'.
0x00: "_setup_trace (initialization)",
0x10: "invalidate (miss)",
0x1A: "invalidate (hit, version)",
0x1C: "invalidate (hit, saving non-current)",
# 0x1E can occur during startup verification.
0x1E: "invalidate (hit, discarding current or non-current)",
0x20: "load (miss)",
0x22: "load (hit)",
0x24: "load (non-current, miss)",
0x26: "load (non-current, hit)",
0x50: "store (version)",
0x52: "store (current, non-version)",
0x54: "store (non-current)",
}
if __name__ == "__main__":
sys.exit(main())
src/ZEO/scripts/parsezeolog.py deleted 100644 → 0
View file @ 56c7dd30
#!/usr/bin/env python2.3
"""Parse the BLATHER logging generated by ZEO2.
An example of the log format is:
2002-04-15T13:05:29 BLATHER(-100) ZEO Server storea(3235680, [714], 235339406490168806) ('10.0.26.30', 45514)
"""
import re
import time
rx_time = re.compile('(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)')
def parse_time(line):
"""Return the time portion of a zLOG line in seconds or None."""
mo = rx_time.match(line)
if mo is None:
return None
date, time_ = mo.group(1, 2)
date_l = [int(elt) for elt in date.split('-')]
time_l = [int(elt) for elt in time_.split(':')]
return int(time.mktime(date_l + time_l + [0, 0, 0]))
rx_meth = re.compile("zrpc:\d+ calling (\w+)\((.*)")
def parse_method(line):
pass
def parse_line(line):
"""Parse a log entry and return time, method info, and client."""
t = parse_time(line)
if t is None:
return None, None
mo = rx_meth.search(line)
if mo is None:
return None, None
meth_name = mo.group(1)
meth_args = mo.group(2).strip()
if meth_args.endswith(')'):
meth_args = meth_args[:-1]
meth_args = [s.strip() for s in meth_args.split(",")]
m = meth_name, tuple(meth_args)
return t, m
class TStats:
counter = 1
def __init__(self):
self.id = TStats.counter
TStats.counter += 1
fields = ("time", "vote", "done", "user", "path")
fmt = "%-24s %5s %5s %-15s %s"
hdr = fmt % fields
def report(self):
"""Print a report about the transaction"""
t = time.ctime(self.begin)
if hasattr(self, "vote"):
d_vote = self.vote - self.begin
else:
d_vote = "*"
if hasattr(self, "finish"):
d_finish = self.finish - self.begin
else:
d_finish = "*"
print self.fmt % (time.ctime(self.begin), d_vote, d_finish,
self.user, self.url)
class TransactionParser:
def __init__(self):
self.txns = {}
self.skipped = 0
def parse(self, line):
t, m = parse_line(line)
if t is None:
return
name = m[0]
meth = getattr(self, name, None)
if meth is not None:
meth(t, m[1])
def tpc_begin(self, time, args):
t = TStats()
t.begin = time
t.user = args[1]
t.url = args[2]
t.objects = []
tid = eval(args[0])
self.txns[tid] = t
def get_txn(self, args):
tid = eval(args[0])
try:
return self.txns[tid]
except KeyError:
print "uknown tid", repr(tid)
return None
def tpc_finish(self, time, args):
t = self.get_txn(args)
if t is None:
return
t.finish = time
def vote(self, time, args):
t = self.get_txn(args)
if t is None:
return
t.vote = time
def get_txns(self):
L = [(t.id, t) for t in self.txns.values()]
L.sort()
return [t for (id, t) in L]
if __name__ == "__main__":
import fileinput
p = TransactionParser()
i = 0
for line in fileinput.input():
i += 1
try:
p.parse(line)
except:
print "line", i
raise
print "Transaction: %d" % len(p.txns)
print TStats.hdr
for txn in p.get_txns():
txn.report()
src/ZEO/scripts/tests.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import doctest, re, unittest
from zope.testing import renormalizing
def test_suite():
return unittest.TestSuite((
doctest.DocFileSuite(
'zeopack.test',
checker=renormalizing.RENormalizing([
(re.compile('usage: Usage: '), 'Usage: '), # Py 2.4
(re.compile('options:'), 'Options:'), # Py 2.4
])
),
))
src/ZEO/scripts/timeout.py deleted 100755 → 0
View file @ 56c7dd30
#!/usr/bin/env python2.3
"""Transaction timeout test script.
This script connects to a storage, begins a transaction, calls store()
and tpc_vote(), and then sleeps forever. This should trigger the
transaction timeout feature of the server.
usage: timeout.py address delay [storage-name]
"""
import sys
import time
from ZODB.Transaction import Transaction
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_pickle
from ZEO.ClientStorage import ClientStorage
ZERO = '\0'*8
def main():
if len(sys.argv) not in (3, 4):
sys.stderr.write("Usage: timeout.py address delay [storage-name]\n" %
sys.argv[0])
sys.exit(2)
hostport = sys.argv[1]
delay = float(sys.argv[2])
if sys.argv[3:]:
name = sys.argv[3]
else:
name = "1"
if "/" in hostport:
address = hostport
else:
if ":" in hostport:
i = hostport.index(":")
host, port = hostport[:i], hostport[i+1:]
else:
host, port = "", hostport
port = int(port)
address = (host, port)
print "Connecting to %s..." % repr(address)
storage = ClientStorage(address, name)
print "Connected. Now starting a transaction..."
oid = storage.new_oid()
revid = ZERO
data = MinPO("timeout.py")
pickled_data = zodb_pickle(data)
t = Transaction()
t.user = "timeout.py"
storage.tpc_begin(t)
storage.store(oid, revid, pickled_data, '', t)
print "Stored. Now voting..."
storage.tpc_vote(t)
print "Voted; now sleeping %s..." % delay
time.sleep(delay)
print "Done."
if __name__ == "__main__":
main()
src/ZEO/scripts/zeopack.py deleted 100755 → 0
View file @ 56c7dd30
#!/usr/bin/env python2.3
import logging
import optparse
import socket
import sys
import time
import traceback
import ZEO.ClientStorage
usage = """Usage: %prog [options] [servers]
Pack one or more storages hosted by ZEO servers.
The positional arguments specify 0 or more tcp servers to pack, where
each is of the form:
host:port[:name]
"""
WAIT = 10 # wait no more than 10 seconds for client to connect
def _main(args=None, prog=None):
if args is None:
args = sys.argv[1:]
parser = optparse.OptionParser(usage, prog=prog)
parser.add_option(
"-d", "--days", dest="days", type='int', default=0,
help=("Pack objects that are older than this number of days")
)
parser.add_option(
"-t", "--time", dest="time",
help=("Time of day to pack to of the form: HH[:MM[:SS]]. "
"Defaults to current time.")
)
parser.add_option(
"-u", "--unix", dest="unix_sockets", action="append",
help=("A unix-domain-socket server to connect to, of the form: "
"path[:name]")
)
parser.remove_option('-h')
parser.add_option(
"-h", dest="host",
help=("Deprecated: "
"Used with the -p and -S options, specified the host to "
"connect to.")
)
parser.add_option(
"-p", type="int", dest="port",
help=("Deprecated: "
"Used with the -h and -S options, specifies "
"the port to connect to.")
)
parser.add_option(
"-S", dest="name", default='1',
help=("Deprecated: Used with the -h and -p, options, or with the "
"-U option specified the storage name to use. Defaults to 1.")
)
parser.add_option(
"-U", dest="unix",
help=("Deprecated: Used with the -S option, "
"Unix-domain socket to connect to.")
)
if not args:
parser.print_help()
return
def error(message):
sys.stderr.write("Error:\n%s\n" % message)
sys.exit(1)
options, args = parser.parse_args(args)
packt = time.time()
if options.time:
time_ = map(int, options.time.split(':'))
if len(time_) == 1:
time_ += (0, 0)
elif len(time_) == 2:
time_ += (0,)
elif len(time_) > 3:
error("Invalid time value: %r" % options.time)
packt = time.localtime(packt)
packt = time.mktime(packt[:3]+tuple(time_)+packt[6:])
packt -= options.days * 86400
servers = []
if options.host:
if not options.port:
error("If host (-h) is specified then a port (-p) must be "
"specified as well.")
servers.append(((options.host, options.port), options.name))
elif options.port:
servers.append(((socket.gethostname(), options.port), options.name))
if options.unix:
servers.append((options.unix, options.name))
for server in args:
data = server.split(':')
if len(data) in (2, 3):
host = data[0]
try:
port = int(data[1])
except ValueError:
error("Invalid port in server specification: %r" % server)
addr = host, port
if len(data) == 2:
name = '1'
else:
name = data[2]
else:
error("Invalid server specification: %r" % server)
servers.append((addr, name))
for server in options.unix_sockets or ():
data = server.split(':')
if len(data) == 1:
addr = data[0]
name = '1'
elif len(data) == 2:
addr = data[0]
name = data[1]
else:
error("Invalid server specification: %r" % server)
servers.append((addr, name))
if not servers:
error("No servers specified.")
for addr, name in servers:
try:
cs = ZEO.ClientStorage.ClientStorage(
addr, storage=name, wait=False, read_only=1)
for i in range(60):
if cs.is_connected():
break
time.sleep(1)
else:
sys.stderr.write("Couldn't connect to: %r\n"
% ((addr, name), ))
cs.close()
continue
cs.pack(packt, wait=True)
cs.close()
except:
traceback.print_exception(*(sys.exc_info()+(99, sys.stderr)))
error("Error packing storage %s in %r" % (name, addr))
def main(*args):
root_logger = logging.getLogger()
old_level = root_logger.getEffectiveLevel()
logging.getLogger().setLevel(logging.WARNING)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(
"%(name)s %(levelname)s %(message)s"))
logging.getLogger().addHandler(handler)
try:
_main(*args)
finally:
logging.getLogger().setLevel(old_level)
logging.getLogger().removeHandler(handler)
if __name__ == "__main__":
main()
src/ZEO/scripts/zeopack.test deleted 100644 → 0
View file @ 56c7dd30
zeopack
=======
The zeopack script can be used to pack one or more storages. It uses
ClientStorage to do this. To test it's behavior, we'll replace the
normal ClientStorage with a fake one that echos information we'll want
for our test:
>>> class ClientStorage:
... connect_wait = 0
... def __init__(self, *args, **kw):
... if args[0] == 'bad':
... import logging
... logging.getLogger('test.ClientStorage').error(
... "I hate this address, %r", args[0])
... raise ValueError("Bad address")
... print "ClientStorage(%s %s)" % (
... repr(args)[1:-1],
... ', '.join("%s=%r" % i for i in sorted(kw.items())),
... )
... def pack(self, t=None, *args, **kw):
... now = time.localtime(time.time())
... local_midnight = time.mktime(now[:3]+(0, 0, 0)+now[6:])
... t -= local_midnight # adjust for tz
... t += 86400*7 # add a week to make sure we're positive
... print "pack(%r,%s %s)" % (
... t, repr(args)[1:-1],
... ', '.join("%s=%r" % i for i in sorted(kw.items())),
... )
... def is_connected(self):
... self.connect_wait -= 1
... print 'is_connected', self.connect_wait < 0
... return self.connect_wait < 0
... def close(self):
... print "close()"
>>> import ZEO
>>> ClientStorage_orig = ZEO.ClientStorage.ClientStorage
>>> ZEO.ClientStorage.ClientStorage = ClientStorage
Now, we're ready to try the script:
>>> from ZEO.scripts.zeopack import main
If we call it with no arguments, we get help:
>>> import os; os.environ['COLUMNS'] = '80' # for consistent optparse output
>>> main([], 'zeopack')
Usage: zeopack [options] [servers]
<BLANKLINE>
Pack one or more storages hosted by ZEO servers.
<BLANKLINE>
The positional arguments specify 0 or more tcp servers to pack, where
each is of the form:
<BLANKLINE>
host:port[:name]
<BLANKLINE>
<BLANKLINE>
<BLANKLINE>
Options:
-d DAYS, --days=DAYS Pack objects that are older than this number of days
-t TIME, --time=TIME Time of day to pack to of the form: HH[:MM[:SS]].
Defaults to current time.
-u UNIX_SOCKETS, --unix=UNIX_SOCKETS
A unix-domain-socket server to connect to, of the
form: path[:name]
-h HOST Deprecated: Used with the -p and -S options, specified
the host to connect to.
-p PORT Deprecated: Used with the -h and -S options, specifies
the port to connect to.
-S NAME Deprecated: Used with the -h and -p, options, or with
the -U option specified the storage name to use.
Defaults to 1.
-U UNIX Deprecated: Used with the -S option, Unix-domain
socket to connect to.
Since packing involves time, we'd better have our way with it. Replace
time.time() with a function that always returns the same value. The
value is timezone dependent.
>>> import time
>>> time_orig = time.time
>>> time.time = lambda : time.mktime((2009, 3, 24, 10, 55, 17, 1, 83, -1))
>>> sleep_orig = time.sleep
>>> def sleep(t):
... print 'sleep(%r)' % t
>>> time.sleep = sleep
Normally, we pass one or more TCP server specifications:
>>> main(["host1:8100", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
is_connected True
pack(644117.0, wait=True)
close()
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(644117.0, wait=True)
close()
We can also pass unix-domain-sockey servers using the -u option:
>>> main(["-ufoo", "-ubar:spam", "host1:8100", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
is_connected True
pack(644117.0, wait=True)
close()
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(644117.0, wait=True)
close()
ClientStorage('foo', read_only=1, storage='1', wait=False)
is_connected True
pack(644117.0, wait=True)
close()
ClientStorage('bar', read_only=1, storage='spam', wait=False)
is_connected True
pack(644117.0, wait=True)
close()
The -d option causes a pack time the given number of days earlier to
be used:
>>> main(["-ufoo", "-ubar:spam", "-d3", "host1:8100", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
is_connected True
pack(384917.0, wait=True)
close()
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(384917.0, wait=True)
close()
ClientStorage('foo', read_only=1, storage='1', wait=False)
is_connected True
pack(384917.0, wait=True)
close()
ClientStorage('bar', read_only=1, storage='spam', wait=False)
is_connected True
pack(384917.0, wait=True)
close()
The -t option allows us to control the time of day:
>>> main(["-ufoo", "-d3", "-t1:30", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(351000.0, wait=True)
close()
ClientStorage('foo', read_only=1, storage='1', wait=False)
is_connected True
pack(351000.0, wait=True)
close()
Connection timeout
------------------
The zeopack script tells ClientStorage not to wait for connections
before returning from the constructor, but will time out after 60
seconds of waiting for a connect.
>>> ClientStorage.connect_wait = 3
>>> main(["-d3", "-t1:30", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected False
sleep(1)
is_connected False
sleep(1)
is_connected False
sleep(1)
is_connected True
pack(351000.0, wait=True)
close()
>>> def call_main(args):
... import sys
... old_stderr = sys.stderr
... sys.stderr = sys.stdout
... try:
... try:
... main(args)
... except SystemExit, v:
... print "Exited", v
... finally:
... sys.stderr = old_stderr
>>> ClientStorage.connect_wait = 999
>>> call_main(["-d3", "-t1:30", "host1:8100", "host1:8100:2"])
... # doctest: +ELLIPSIS
ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
is_connected False
sleep(1)
...
is_connected False
sleep(1)
Couldn't connect to: (('host1', 8100), '1')
close()
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected False
sleep(1)
...
is_connected False
sleep(1)
Couldn't connect to: (('host1', 8100), '2')
close()
>>> ClientStorage.connect_wait = 0
Legacy support
--------------
>>> main(["-d3", "-h", "host1", "-p", "8100", "-S", "2"])
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(384917.0, wait=True)
close()
>>> import socket
>>> old_gethostname = socket.gethostname
>>> socket.gethostname = lambda : 'test.host.com'
>>> main(["-d3", "-p", "8100"])
ClientStorage(('test.host.com', 8100), read_only=1, storage='1', wait=False)
is_connected True
pack(384917.0, wait=True)
close()
>>> socket.gethostname = old_gethostname
>>> main(["-d3", "-U", "foo/bar", "-S", "2"])
ClientStorage('foo/bar', read_only=1, storage='2', wait=False)
is_connected True
pack(384917.0, wait=True)
close()
Error handling
--------------
>>> call_main(["-d3"])
Error:
No servers specified.
Exited 1
>>> call_main(["-d3", "a"])
Error:
Invalid server specification: 'a'
Exited 1
>>> call_main(["-d3", "a:b:c:d"])
Error:
Invalid server specification: 'a:b:c:d'
Exited 1
>>> call_main(["-d3", "a:b:2"])
Error:
Invalid port in server specification: 'a:b:2'
Exited 1
>>> call_main(["-d3", "-u", "a:b:2"])
Error:
Invalid server specification: 'a:b:2'
Exited 1
>>> call_main(["-d3", "-u", "bad"]) # doctest: +ELLIPSIS
test.ClientStorage ERROR I hate this address, 'bad'
Traceback (most recent call last):
...
ValueError: Bad address
Error:
Error packing storage 1 in 'bad'
Exited 1
Note that in the previous example, the first line was output through logging.
.. tear down
>>> ZEO.ClientStorage.ClientStorage = ClientStorage_orig
>>> time.time = time_orig
>>> time.sleep = sleep_orig
src/ZEO/scripts/zeoqueue.py deleted 100755 → 0
View file @ 56c7dd30
#!/usr/bin/env python2.3
"""Report on the number of currently waiting clients in the ZEO queue.
Usage: %(PROGRAM)s [options] logfile
Options:
-h / --help
Print this help text and exit.
-v / --verbose
Verbose output
-f file
--file file
Use the specified file to store the incremental state as a pickle. If
not given, %(STATEFILE)s is used.
-r / --reset
Reset the state of the tool. This blows away any existing state
pickle file and then exits -- it does not parse the file. Use this
when you rotate log files so that the next run will parse from the
beginning of the file.
"""
import os
import re
import sys
import time
import errno
import getopt
import cPickle as pickle
COMMASPACE = ', '
STATEFILE = 'zeoqueue.pck'
PROGRAM = sys.argv[0]
try:
True, False
except NameError:
True = 1
False = 0
tcre = re.compile(r"""
(?P<ymd>
\d{4}- # year
\d{2}- # month
\d{2}) # day
T # separator
(?P<hms>
\d{2}: # hour
\d{2}: # minute
\d{2}) # second
""", re.VERBOSE)
ccre = re.compile(r"""
zrpc-conn:(?P<addr>\d+.\d+.\d+.\d+:\d+)\s+
calling\s+
(?P<method>
\w+) # the method
\( # args open paren
\' # string quote start
(?P<tid>
\S+) # first argument -- usually the tid
\' # end of string
(?P<rest>
.*) # rest of line
""", re.VERBOSE)
wcre = re.compile(r'Clients waiting: (?P<num>\d+)')
def parse_time(line):
"""Return the time portion of a zLOG line in seconds or None."""
mo = tcre.match(line)
if mo is None:
return None
date, time_ = mo.group('ymd', 'hms')
date_l = [int(elt) for elt in date.split('-')]
time_l = [int(elt) for elt in time_.split(':')]
return int(time.mktime(date_l + time_l + [0, 0, 0]))
class Txn:
"""Track status of single transaction."""
def __init__(self, tid):
self.tid = tid
self.hint = None
self.begin = None
self.vote = None
self.abort = None
self.finish = None
self.voters = []
def isactive(self):
if self.begin and not (self.abort or self.finish):
return True
else:
return False
class Status:
"""Track status of ZEO server by replaying log records.
We want to keep track of several events:
- The last committed transaction.
- The last committed or aborted transaction.
- The last transaction that got the lock but didn't finish.
- The client address doing the first vote of a transaction.
- The number of currently active transactions.
- The number of reported queued transactions.
- Client restarts.
- Number of current connections (but this might not be useful).
We can observe these events by reading the following sorts of log
entries:
2002-12-16T06:16:05 BLATHER(-100) zrpc:12649 calling
tpc_begin('\x03I\x90((\xdbp\xd5', '', 'QueueCatal...
2002-12-16T06:16:06 BLATHER(-100) zrpc:12649 calling
vote('\x03I\x90((\xdbp\xd5')
2002-12-16T06:16:06 BLATHER(-100) zrpc:12649 calling
tpc_finish('\x03I\x90((\xdbp\xd5')
2002-12-16T10:46:10 INFO(0) ZSS:12649:1 Transaction blocked waiting
for storage. Clients waiting: 1.
2002-12-16T06:15:57 BLATHER(-100) zrpc:12649 connect from
('10.0.26.54', 48983): <ManagedServerConnection ('10.0.26.54', 48983)>
2002-12-16T10:30:09 INFO(0) ZSS:12649:1 disconnected
"""
def __init__(self):
self.lineno = 0
self.pos = 0
self.reset()
def reset(self):
self.commit = None
self.commit_or_abort = None
self.last_unfinished = None
self.n_active = 0
self.n_blocked = 0
self.n_conns = 0
self.t_restart = None
self.txns = {}
def iscomplete(self):
# The status report will always be complete if we encounter an
# explicit restart.
if self.t_restart is not None:
return True
# If we haven't seen a restart, assume that seeing a finished
# transaction is good enough.
return self.commit is not None
def process_file(self, fp):
if self.pos:
if VERBOSE:
print 'seeking to file position', self.pos
fp.seek(self.pos)
while True:
line = fp.readline()
if not line:
break
self.lineno += 1
self.process(line)
self.pos = fp.tell()
def process(self, line):
if line.find("calling") != -1:
self.process_call(line)
elif line.find("connect") != -1:
self.process_connect(line)
# test for "locked" because word may start with "B" or "b"
elif line.find("locked") != -1:
self.process_block(line)
elif line.find("Starting") != -1:
self.process_start(line)
def process_call(self, line):
mo = ccre.search(line)
if mo is None:
return
called_method = mo.group('method')
# Exit early if we've got zeoLoad, because it's the most
# frequently called method and we don't use it.
if called_method == "zeoLoad":
return
t = parse_time(line)
meth = getattr(self, "call_%s" % called_method, None)
if meth is None:
return
client = mo.group('addr')
tid = mo.group('tid')
rest = mo.group('rest')
meth(t, client, tid, rest)
def process_connect(self, line):
pass
def process_block(self, line):
mo = wcre.search(line)
if mo is None:
# assume that this was a restart message for the last blocked
# transaction.
self.n_blocked = 0
else:
self.n_blocked = int(mo.group('num'))
def process_start(self, line):
if line.find("Starting ZEO server") != -1:
self.reset()
self.t_restart = parse_time(line)
def call_tpc_begin(self, t, client, tid, rest):
txn = Txn(tid)
txn.begin = t
if rest[0] == ',':
i = 1
while rest[i].isspace():
i += 1
rest = rest[i:]
txn.hint = rest
self.txns[tid] = txn
self.n_active += 1
self.last_unfinished = txn
def call_vote(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
txn = self.txns[tid] = Txn(tid)
txn.vote = t
txn.voters.append(client)
def call_tpc_abort(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
txn = self.txns[tid] = Txn(tid)
txn.abort = t
txn.voters = []
self.n_active -= 1
if self.commit_or_abort:
# delete the old transaction
try:
del self.txns[self.commit_or_abort.tid]
except KeyError:
pass
self.commit_or_abort = txn
def call_tpc_finish(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
txn = self.txns[tid] = Txn(tid)
txn.finish = t
txn.voters = []
self.n_active -= 1
if self.commit:
# delete the old transaction
try:
del self.txns[self.commit.tid]
except KeyError:
pass
if self.commit_or_abort:
# delete the old transaction
try:
del self.txns[self.commit_or_abort.tid]
except KeyError:
pass
self.commit = self.commit_or_abort = txn
def report(self):
print "Blocked transactions:", self.n_blocked
if not VERBOSE:
return
if self.t_restart:
print "Server started:", time.ctime(self.t_restart)
if self.commit is not None:
t = self.commit_or_abort.finish
if t is None:
t = self.commit_or_abort.abort
print "Last finished transaction:", time.ctime(t)
# the blocked transaction should be the first one that calls vote
L = [(txn.begin, txn) for txn in self.txns.values()]
L.sort()
for x, txn in L:
if txn.isactive():
began = txn.begin
if txn.voters:
print "Blocked client (first vote):", txn.voters[0]
print "Blocked transaction began at:", time.ctime(began)
print "Hint:", txn.hint
print "Idle time: %d sec" % int(time.time() - began)
break
def usage(code, msg=''):
print >> sys.stderr, __doc__ % globals()
if msg:
print >> sys.stderr, msg
sys.exit(code)
def main():
global VERBOSE
VERBOSE = 0
file = STATEFILE
reset = False
# -0 is a secret option used for testing purposes only
seek = True
try:
opts, args = getopt.getopt(sys.argv[1:], 'vhf:r0',
['help', 'verbose', 'file=', 'reset'])
except getopt.error, msg:
usage(1, msg)
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-v', '--verbose'):
VERBOSE += 1
elif opt in ('-f', '--file'):
file = arg
elif opt in ('-r', '--reset'):
reset = True
elif opt == '-0':
seek = False
if reset:
# Blow away the existing state file and exit
try:
os.unlink(file)
if VERBOSE:
print 'removing pickle state file', file
except OSError, e:
if e.errno <> errno.ENOENT:
raise
return
if not args:
usage(1, 'logfile is required')
if len(args) > 1:
usage(1, 'too many arguments: %s' % COMMASPACE.join(args))
path = args[0]
# Get the previous status object from the pickle file, if it is available
# and if the --reset flag wasn't given.
status = None
try:
statefp = open(file, 'rb')
try:
status = pickle.load(statefp)
if VERBOSE:
print 'reading status from file', file
finally:
statefp.close()
except IOError, e:
if e.errno <> errno.ENOENT:
raise
if status is None:
status = Status()
if VERBOSE:
print 'using new status'
if not seek:
status.pos = 0
fp = open(path, 'rb')
try:
status.process_file(fp)
finally:
fp.close()
# Save state
statefp = open(file, 'wb')
pickle.dump(status, statefp, 1)
statefp.close()
# Print the report and return the number of blocked clients in the exit
# status code.
status.report()
sys.exit(status.n_blocked)
if __name__ == "__main__":
main()
src/ZEO/scripts/zeoreplay.py deleted 100644 → 0
View file @ 56c7dd30
#!/usr/bin/env python2.3
"""Parse the BLATHER logging generated by ZEO, and optionally replay it.
Usage: zeointervals.py [options]
Options:
--help / -h
Print this message and exit.
--replay=storage
-r storage
Replay the parsed transactions through the new storage
--maxtxn=count
-m count
Parse no more than count transactions.
--report / -p
Print a report as we're parsing.
Unlike parsezeolog.py, this script generates timestamps for each transaction,
and sub-command in the transaction. We can use this to compare timings with
synthesized data.
"""
import re
import sys
import time
import getopt
import operator
# ZEO logs measure wall-clock time so for consistency we need to do the same
#from time import clock as now
from time import time as now
from ZODB.FileStorage import FileStorage
#from BDBStorage.BDBFullStorage import BDBFullStorage
#from Standby.primary import PrimaryStorage
#from Standby.config import RS_PORT
from ZODB.Transaction import Transaction
from ZODB.utils import p64
datecre = re.compile('(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)')
methcre = re.compile("ZEO Server (\w+)\((.*)\) \('(.*)', (\d+)")
class StopParsing(Exception):
pass
def usage(code, msg=''):
print __doc__
if msg:
print msg
sys.exit(code)
def parse_time(line):
"""Return the time portion of a zLOG line in seconds or None."""
mo = datecre.match(line)
if mo is None:
return None
date, time_ = mo.group(1, 2)
date_l = [int(elt) for elt in date.split('-')]
time_l = [int(elt) for elt in time_.split(':')]
return int(time.mktime(date_l + time_l + [0, 0, 0]))
def parse_line(line):
"""Parse a log entry and return time, method info, and client."""
t = parse_time(line)
if t is None:
return None, None, None
mo = methcre.search(line)
if mo is None:
return None, None, None
meth_name = mo.group(1)
meth_args = mo.group(2)
meth_args = [s.strip() for s in meth_args.split(',')]
m = meth_name, tuple(meth_args)
c = mo.group(3), mo.group(4)
return t, m, c
class StoreStat:
def __init__(self, when, oid, size):
self.when = when
self.oid = oid
self.size = size
# Crufty
def __getitem__(self, i):
if i == 0: return self.oid
if i == 1: return self.size
raise IndexError
class TxnStat:
def __init__(self):
self._begintime = None
self._finishtime = None
self._aborttime = None
self._url = None
self._objects = []
def tpc_begin(self, when, args, client):
self._begintime = when
# args are txnid, user, description (looks like it's always a url)
self._url = args[2]
def storea(self, when, args, client):
oid = int(args[0])
# args[1] is "[numbytes]"
size = int(args[1][1:-1])
s = StoreStat(when, oid, size)
self._objects.append(s)
def tpc_abort(self, when):
self._aborttime = when
def tpc_finish(self, when):
self._finishtime = when
# Mapping oid -> revid
_revids = {}
class ReplayTxn(TxnStat):
def __init__(self, storage):
self._storage = storage
self._replaydelta = 0
TxnStat.__init__(self)
def replay(self):
ZERO = '\0'*8
t0 = now()
t = Transaction()
self._storage.tpc_begin(t)
for obj in self._objects:
oid = obj.oid
revid = _revids.get(oid, ZERO)
# BAW: simulate a pickle of the given size
data = 'x' * obj.size
# BAW: ignore versions for now
newrevid = self._storage.store(p64(oid), revid, data, '', t)
_revids[oid] = newrevid
if self._aborttime:
self._storage.tpc_abort(t)
origdelta = self._aborttime - self._begintime
else:
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
origdelta = self._finishtime - self._begintime
t1 = now()
# Shows how many seconds behind (positive) or ahead (negative) of the
# original reply our local update took
self._replaydelta = t1 - t0 - origdelta
class ZEOParser:
def __init__(self, maxtxns=-1, report=1, storage=None):
self.__txns = []
self.__curtxn = {}
self.__skipped = 0
self.__maxtxns = maxtxns
self.__finishedtxns = 0
self.__report = report
self.__storage = storage
def parse(self, line):
t, m, c = parse_line(line)
if t is None:
# Skip this line
return
name = m[0]
meth = getattr(self, name, None)
if meth is not None:
meth(t, m[1], c)
def tpc_begin(self, when, args, client):
txn = ReplayTxn(self.__storage)
self.__curtxn[client] = txn
meth = getattr(txn, 'tpc_begin', None)
if meth is not None:
meth(when, args, client)
def storea(self, when, args, client):
txn = self.__curtxn.get(client)
if txn is None:
self.__skipped += 1
return
meth = getattr(txn, 'storea', None)
if meth is not None:
meth(when, args, client)
def tpc_finish(self, when, args, client):
txn = self.__curtxn.get(client)
if txn is None:
self.__skipped += 1
return
meth = getattr(txn, 'tpc_finish', None)
if meth is not None:
meth(when)
if self.__report:
self.report(txn)
self.__txns.append(txn)
self.__curtxn[client] = None
self.__finishedtxns += 1
if self.__maxtxns > 0 and self.__finishedtxns >= self.__maxtxns:
raise StopParsing
def report(self, txn):
"""Print a report about the transaction"""
if txn._objects:
bytes = reduce(operator.add, [size for oid, size in txn._objects])
else:
bytes = 0
print '%s %s %4d %10d %s %s' % (
txn._begintime, txn._finishtime - txn._begintime,
len(txn._objects),
bytes,
time.ctime(txn._begintime),
txn._url)
def replay(self):
for txn in self.__txns:
txn.replay()
# How many fell behind?
slower = []
faster = []
for txn in self.__txns:
if txn._replaydelta > 0:
slower.append(txn)
else:
faster.append(txn)
print len(slower), 'laggards,', len(faster), 'on-time or faster'
# Find some averages
if slower:
sum = reduce(operator.add,
[txn._replaydelta for txn in slower], 0)
print 'average slower txn was:', float(sum) / len(slower)
if faster:
sum = reduce(operator.add,
[txn._replaydelta for txn in faster], 0)
print 'average faster txn was:', float(sum) / len(faster)
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:],
'hr:pm:',
['help', 'replay=', 'report', 'maxtxns='])
except getopt.error, e:
usage(1, e)
if args:
usage(1)
replay = 0
maxtxns = -1
report = 0
storagefile = None
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-r', '--replay'):
replay = 1
storagefile = arg
elif opt in ('-p', '--report'):
report = 1
elif opt in ('-m', '--maxtxns'):
try:
maxtxns = int(arg)
except ValueError:
usage(1, 'Bad -m argument: %s' % arg)
if replay:
storage = FileStorage(storagefile)
#storage = BDBFullStorage(storagefile)
#storage = PrimaryStorage('yyz', storage, RS_PORT)
t0 = now()
p = ZEOParser(maxtxns, report, storage)
i = 0
while 1:
line = sys.stdin.readline()
if not line:
break
i += 1
try:
p.parse(line)
except StopParsing:
break
except:
print 'input file line:', i
raise
t1 = now()
print 'total parse time:', t1-t0
t2 = now()
if replay:
p.replay()
t3 = now()
print 'total replay time:', t3-t2
print 'total time:', t3-t0
if __name__ == '__main__':
main()
src/ZEO/scripts/zeoserverlog.py deleted 100644 → 0
View file @ 56c7dd30
#!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Tools for analyzing ZEO Server logs.
This script contains a number of commands, implemented by command
functions. To run a command, give the command name and it's arguments
as arguments to this script.
Commands:
blocked_times file threshold
Output a summary of episodes where thransactions were blocked
when the episode lasted at least threshold seconds.
The file may be a file name or - to read from standard input.
The file may also be a command:
script blocked_times 'bunzip2 <foo.log.bz2' 60
If the file is a command, it must contain at least a single
space.
The columns of output are:
- The time the episode started
- The seconds from the start of the episode until the blocking
transaction finished.
- The client id (host and port) of the blocking transaction.
- The seconds from the start of the episode until the end of the
episode.
time_calls file threshold
Time how long calls took. Note that this is normally combined
with grep to time just a particulat kind of call:
script time_calls 'bunzip2 <foo.log.bz2 | grep tpc_finish' 10
time_trans threshold
The columns of output are:
- The time of the call invocation
- The seconds from the call to the return
- The client that made the call.
time_trans file threshold
Output a summary of transactions that held the global transaction
lock for at least threshold seconds. (This is the time from when
voting starts until the transaction is completed by the server.)
The columns of output are:
- time that the vote started.
- client id
- number of objects written / number of objects updated
- seconds from tpc_begin to vote start
- seconds spent voting
- vote status: n=normal, d=delayed, e=error
- seconds wating between vote return and finish call
- time spent finishing or 'abort' if the transaction aborted
minute file
Compute production statistics by minute
The columns of output are:
- date/time
- Number of active clients
- number of reads
- number of stores
- number of commits (finish)
- number of aborts
- number of transactions (commits + aborts)
Summary statistics are printed at the end
minutes file
Show just the summary statistics for production by minute.
hour file
Compute production statistics by hour
hours file
Show just the summary statistics for production by hour.
day file
Compute production statistics by day
days file
Show just the summary statistics for production by day.
verify file
Compute verification statistics
The columns of output are:
- client id
- verification start time
- number of object's verified
- wall time to verify
- average miliseconds to verify per object.
$Id$
"""
import datetime, sys, re, os
def time(line):
d = line[:10]
t = line[11:19]
y, mo, d = map(int, d.split('-'))
h, mi, s = map(int, t.split(':'))
return datetime.datetime(y, mo, d, h, mi, s)
def sub(t1, t2):
delta = t2 - t1
return delta.days*86400.0+delta.seconds+delta.microseconds/1000000.0
waitre = re.compile(r'Clients waiting: (\d+)')
idre = re.compile(r' ZSS:\d+/(\d+.\d+.\d+.\d+:\d+) ')
def blocked_times(args):
f, thresh = args
t1 = t2 = cid = blocking = waiting = 0
last_blocking = False
thresh = int(thresh)
for line in xopen(f):
line = line.strip()
if line.endswith('Blocked transaction restarted.'):
blocking = False
waiting = 0
else:
s = waitre.search(line)
if not s:
continue
waiting = int(s.group(1))
blocking = line.find(
'Transaction blocked waiting for storage') >= 0
if blocking and waiting == 1:
t1 = time(line)
t2 = t1
if not blocking and last_blocking:
last_wait = 0
t2 = time(line)
cid = idre.search(line).group(1)
if waiting == 0:
d = sub(t1, time(line))
if d >= thresh:
print t1, sub(t1, t2), cid, d
t1 = t2 = cid = blocking = waiting = last_wait = max_wait = 0
last_blocking = blocking
connidre = re.compile(r' zrpc-conn:(\d+.\d+.\d+.\d+:\d+) ')
def time_calls(f):
f, thresh = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
thresh = float(thresh)
t1 = None
maxd = 0
for line in f:
line = line.strip()
if ' calling ' in line:
t1 = time(line)
elif ' returns ' in line and t1 is not None:
d = sub(t1, time(line))
if d >= thresh:
print t1, d, connidre.search(line).group(1)
maxd = max(maxd, d)
t1 = None
print maxd
def xopen(f):
if f == '-':
return sys.stdin
if ' ' in f:
return os.popen(f, 'r')
return open(f)
def time_tpc(f):
f, thresh = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
thresh = float(thresh)
transactions = {}
for line in f:
line = line.strip()
if ' calling vote(' in line:
cid = connidre.search(line).group(1)
transactions[cid] = time(line),
elif ' vote returns None' in line:
cid = connidre.search(line).group(1)
transactions[cid] += time(line), 'n'
elif ' vote() raised' in line:
cid = connidre.search(line).group(1)
transactions[cid] += time(line), 'e'
elif ' vote returns ' in line:
# delayed, skip
cid = connidre.search(line).group(1)
transactions[cid] += time(line), 'd'
elif ' calling tpc_abort(' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
t1, t2, vs = transactions[cid]
t = time(line)
d = sub(t1, t)
if d >= thresh:
print 'a', t1, cid, sub(t1, t2), vs, sub(t2, t)
del transactions[cid]
elif ' calling tpc_finish(' in line:
if cid in transactions:
cid = connidre.search(line).group(1)
transactions[cid] += time(line),
elif ' tpc_finish returns ' in line:
if cid in transactions:
t1, t2, vs, t3 = transactions[cid]
t = time(line)
d = sub(t1, t)
if d >= thresh:
print 'c', t1, cid, sub(t1, t2), vs, sub(t2, t3), sub(t3, t)
del transactions[cid]
newobre = re.compile(r"storea\(.*, '\\x00\\x00\\x00\\x00\\x00")
def time_trans(f):
f, thresh = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
thresh = float(thresh)
transactions = {}
for line in f:
line = line.strip()
if ' calling tpc_begin(' in line:
cid = connidre.search(line).group(1)
transactions[cid] = time(line), [0, 0]
if ' calling storea(' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid][1][0] += 1
if not newobre.search(line):
transactions[cid][1][1] += 1
elif ' calling vote(' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid] += time(line),
elif ' vote returns None' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid] += time(line), 'n'
elif ' vote() raised' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid] += time(line), 'e'
elif ' vote returns ' in line:
# delayed, skip
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid] += time(line), 'd'
elif ' calling tpc_abort(' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
try:
t0, (stores, old), t1, t2, vs = transactions[cid]
except ValueError:
pass
else:
t = time(line)
d = sub(t1, t)
if d >= thresh:
print t1, cid, "%s/%s" % (stores, old), \
sub(t0, t1), sub(t1, t2), vs, \
sub(t2, t), 'abort'
del transactions[cid]
elif ' calling tpc_finish(' in line:
if cid in transactions:
cid = connidre.search(line).group(1)
transactions[cid] += time(line),
elif ' tpc_finish returns ' in line:
if cid in transactions:
t0, (stores, old), t1, t2, vs, t3 = transactions[cid]
t = time(line)
d = sub(t1, t)
if d >= thresh:
print t1, cid, "%s/%s" % (stores, old), \
sub(t0, t1), sub(t1, t2), vs, \
sub(t2, t3), sub(t3, t)
del transactions[cid]
def minute(f, slice=16, detail=1, summary=1):
f, = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
cols = ["time", "reads", "stores", "commits", "aborts", "txns"]
fmt = "%18s %6s %6s %7s %6s %6s"
print fmt % cols
print fmt % ["-"*len(col) for col in cols]
mlast = r = s = c = a = cl = None
rs = []
ss = []
cs = []
aborts = []
ts = []
cls = []
for line in f:
line = line.strip()
if (line.find('returns') > 0
or line.find('storea') > 0
or line.find('tpc_abort') > 0
):
client = connidre.search(line).group(1)
m = line[:slice]
if m != mlast:
if mlast:
if detail:
print fmt % (mlast, len(cl), r, s, c, a, a+c)
cls.append(len(cl))
rs.append(r)
ss.append(s)
cs.append(c)
aborts.append(a)
ts.append(c+a)
mlast = m
r = s = c = a = 0
cl = {}
if line.find('zeoLoad') > 0:
r += 1
cl[client] = 1
elif line.find('storea') > 0:
s += 1
cl[client] = 1
elif line.find('tpc_finish') > 0:
c += 1
cl[client] = 1
elif line.find('tpc_abort') > 0:
a += 1
cl[client] = 1
if mlast:
if detail:
print fmt % (mlast, len(cl), r, s, c, a, a+c)
cls.append(len(cl))
rs.append(r)
ss.append(s)
cs.append(c)
aborts.append(a)
ts.append(c+a)
if summary:
print
print 'Summary: \t', '\t'.join(('min', '10%', '25%', 'med',
'75%', '90%', 'max', 'mean'))
print "n=%6d\t" % len(cls), '-'*62
print 'Clients: \t', '\t'.join(map(str,stats(cls)))
print 'Reads: \t', '\t'.join(map(str,stats(rs)))
print 'Stores: \t', '\t'.join(map(str,stats(ss)))
print 'Commits: \t', '\t'.join(map(str,stats(cs)))
print 'Aborts: \t', '\t'.join(map(str,stats(aborts)))
print 'Trans: \t', '\t'.join(map(str,stats(ts)))
def stats(s):
s.sort()
min = s[0]
max = s[-1]
n = len(s)
out = [min]
ni = n + 1
for p in .1, .25, .5, .75, .90:
lp = ni*p
l = int(lp)
if lp < 1 or lp > n:
out.append('-')
elif abs(lp-l) < .00001:
out.append(s[l-1])
else:
out.append(int(s[l-1] + (lp - l) * (s[l] - s[l-1])))
mean = 0.0
for v in s:
mean += v
out.extend([max, int(mean/n)])
return out
def minutes(f):
minute(f, 16, detail=0)
def hour(f):
minute(f, 13)
def day(f):
minute(f, 10)
def hours(f):
minute(f, 13, detail=0)
def days(f):
minute(f, 10, detail=0)
new_connection_idre = re.compile(r"new connection \('(\d+.\d+.\d+.\d+)', (\d+)\):")
def verify(f):
f, = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
t1 = None
nv = {}
for line in f:
if line.find('new connection') > 0:
m = new_connection_idre.search(line)
cid = "%s:%s" % (m.group(1), m.group(2))
nv[cid] = [time(line), 0]
elif line.find('calling zeoVerify(') > 0:
cid = connidre.search(line).group(1)
nv[cid][1] += 1
elif line.find('calling endZeoVerify()') > 0:
cid = connidre.search(line).group(1)
t1, n = nv[cid]
if n:
d = sub(t1, time(line))
print cid, t1, n, d, n and (d*1000.0/n) or '-'
def recovery(f):
f, = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
last = ''
trans = []
n = 0
for line in f:
n += 1
if line.find('RecoveryServer') < 0:
continue
l = line.find('sending transaction ')
if l > 0 and last.find('sending transaction ') > 0:
trans.append(line[l+20:].strip())
else:
if trans:
if len(trans) > 1:
print " ... %s similar records skipped ..." % (
len(trans) - 1)
print n, last.strip()
trans=[]
print n, line.strip()
last = line
if len(trans) > 1:
print " ... %s similar records skipped ..." % (
len(trans) - 1)
print n, last.strip()
if __name__ == '__main__':
globals()[sys.argv[1]](sys.argv[2:])
src/ZEO/scripts/zeoup.py deleted 100755 → 0
View file @ 56c7dd30
#!/usr/bin/env python2.3
"""Make sure a ZEO server is running.
usage: zeoup.py [options]
The test will connect to a ZEO server, load the root object, and attempt to
update the zeoup counter in the root. It will report success if it updates
the counter or if it gets a ConflictError. A ConflictError is considered a
success, because the client was able to start a transaction.
Options:
-p port -- port to connect to
-h host -- host to connect to (default is current host)
-S storage -- storage name (default '1')
-U path -- Unix-domain socket to connect to
--nowrite -- Do not update the zeoup counter.
-1 -- Connect to a ZEO 1.0 server.
You must specify either -p and -h or -U.
"""
import getopt
import logging
import socket
import sys
import time
from persistent.mapping import PersistentMapping
import transaction
import ZODB
from ZODB.POSException import ConflictError
from ZODB.tests.MinPO import MinPO
from ZEO.ClientStorage import ClientStorage
ZEO_VERSION = 2
def setup_logging():
# Set up logging to stderr which will show messages originating
# at severity ERROR or higher.
root = logging.getLogger()
root.setLevel(logging.ERROR)
fmt = logging.Formatter(
"------\n%(asctime)s %(levelname)s %(name)s %(message)s",
"%Y-%m-%dT%H:%M:%S")
handler = logging.StreamHandler()
handler.setFormatter(fmt)
root.addHandler(handler)
def check_server(addr, storage, write):
t0 = time.time()
if ZEO_VERSION == 2:
# TODO: should do retries w/ exponential backoff.
cs = ClientStorage(addr, storage=storage, wait=0,
read_only=(not write))
else:
cs = ClientStorage(addr, storage=storage, debug=1,
wait_for_server_on_startup=1)
# _startup() is an artifact of the way ZEO 1.0 works. The
# ClientStorage doesn't get fully initialized until registerDB()
# is called. The only thing we care about, though, is that
# registerDB() calls _startup().
if write:
db = ZODB.DB(cs)
cn = db.open()
root = cn.root()
try:
# We store the data in a special `monitor' dict under the root,
# where other tools may also store such heartbeat and bookkeeping
# type data.
monitor = root.get('monitor')
if monitor is None:
monitor = root['monitor'] = PersistentMapping()
obj = monitor['zeoup'] = monitor.get('zeoup', MinPO(0))
obj.value += 1
transaction.commit()
except ConflictError:
pass
cn.close()
db.close()
else:
data, serial = cs.load("\0\0\0\0\0\0\0\0", "")
cs.close()
t1 = time.time()
print "Elapsed time: %.2f" % (t1 - t0)
def usage(exit=1):
print __doc__
print " ".join(sys.argv)
sys.exit(exit)
def main():
host = None
port = None
unix = None
write = 1
storage = '1'
try:
opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:1',
['nowrite'])
for o, a in opts:
if o == '-p':
port = int(a)
elif o == '-h':
host = a
elif o == '-U':
unix = a
elif o == '-S':
storage = a
elif o == '--nowrite':
write = 0
elif o == '-1':
ZEO_VERSION = 1
except Exception, err:
s = str(err)
if s:
s = ": " + s
print err.__class__.__name__ + s
usage()
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
if port is None:
usage()
addr = host, port
setup_logging()
check_server(addr, storage, write)
if __name__ == "__main__":
try:
main()
except SystemExit:
raise
except Exception, err:
s = str(err)
if s:
s = ": " + s
print err.__class__.__name__ + s
sys.exit(1)
src/ZEO/tests/Cache.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Tests of the ZEO cache"""
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_unpickle
from transaction import Transaction
class TransUndoStorageWithCache:
def checkUndoInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(23))
revid = self._dostore(oid, revid=revid, data=MinPO(24))
revid = self._dostore(oid, revid=revid, data=MinPO(25))
info = self._storage.undoInfo()
if not info:
# Preserved this comment, but don't understand it:
# "Perhaps we have an old storage implementation that
# does do the negative nonsense."
info = self._storage.undoInfo(0, 20)
tid = info[0]['id']
# Now start an undo transaction
t = Transaction()
t.note('undo1')
oids = self._begin_undos_vote(t, tid)
# Make sure this doesn't load invalid data into the cache
self._storage.load(oid, '')
self._storage.tpc_finish(t)
assert len(oids) == 1
assert oids[0] == oid
data, revid = self._storage.load(oid, '')
obj = zodb_unpickle(data)
assert obj == MinPO(24)
src/ZEO/tests/CommitLockTests.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Tests of the distributed commit lock."""
import threading
import time
from persistent.TimeStamp import TimeStamp
import transaction
from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
import ZEO.ClientStorage
from ZEO.Exceptions import ClientDisconnected
from ZEO.tests.TestThread import TestThread
ZERO = '\0'*8
class DummyDB:
def invalidate(self, *args, **kwargs):
pass
class WorkerThread(TestThread):
# run the entire test in a thread so that the blocking call for
# tpc_vote() doesn't hang the test suite.
def __init__(self, test, storage, trans):
self.storage = storage
self.trans = trans
self.ready = threading.Event()
TestThread.__init__(self, test)
def testrun(self):
try:
self.storage.tpc_begin(self.trans)
oid = self.storage.new_oid()
p = zodb_pickle(MinPO("c"))
self.storage.store(oid, ZERO, p, '', self.trans)
oid = self.storage.new_oid()
p = zodb_pickle(MinPO("c"))
self.storage.store(oid, ZERO, p, '', self.trans)
self.myvote()
self.storage.tpc_finish(self.trans)
except ClientDisconnected:
pass
def myvote(self):
# The vote() call is synchronous, which makes it difficult to
# coordinate the action of multiple threads that all call
# vote(). This method sends the vote call, then sets the
# event saying vote was called, then waits for the vote
# response. It digs deep into the implementation of the client.
# This method is a replacement for:
# self.ready.set()
# self.storage.tpc_vote(self.trans)
rpc = self.storage._server.rpc
msgid = rpc._deferred_call('vote', id(self.trans))
self.ready.set()
rpc._deferred_wait(msgid)
self.storage._check_serials()
class CommitLockTests:
NUM_CLIENTS = 5
# The commit lock tests verify that the storage successfully
# blocks and restarts transactions when there is contention for a
# single storage. There are a lot of cases to cover.
# The general flow of these tests is to start a transaction by
# getting far enough into 2PC to acquire the commit lock. Then
# begin one or more other connections that also want to commit.
# This causes the commit lock code to be exercised. Once the
# other connections are started, the first transaction completes.
def _cleanup(self):
for store, trans in self._storages:
store.tpc_abort(trans)
store.close()
self._storages = []
def _start_txn(self):
txn = transaction.Transaction()
self._storage.tpc_begin(txn)
oid = self._storage.new_oid()
self._storage.store(oid, ZERO, zodb_pickle(MinPO(1)), '', txn)
return oid, txn
def _begin_threads(self):
# Start a second transaction on a different connection without
# blocking the test thread. Returns only after each thread has
# set it's ready event.
self._storages = []
self._threads = []
for i in range(self.NUM_CLIENTS):
storage = self._duplicate_client()
txn = transaction.Transaction()
tid = self._get_timestamp()
t = WorkerThread(self, storage, txn)
self._threads.append(t)
t.start()
t.ready.wait()
# Close one of the connections abnormally to test server response
if i == 0:
storage.close()
else:
self._storages.append((storage, txn))
def _finish_threads(self):
for t in self._threads:
t.cleanup()
def _duplicate_client(self):
"Open another ClientStorage to the same server."
# It's hard to find the actual address.
# The rpc mgr addr attribute is a list. Each element in the
# list is a socket domain (AF_INET, AF_UNIX, etc.) and an
# address.
addr = self._storage._addr
new = ZEO.ClientStorage.ClientStorage(addr, wait=1)
new.registerDB(DummyDB())
return new
def _get_timestamp(self):
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
return `t`
class CommitLockVoteTests(CommitLockTests):
def checkCommitLockVoteFinish(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.tpc_finish(txn)
self._storage.load(oid, '')
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockVoteAbort(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.tpc_abort(txn)
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockVoteClose(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.close()
self._finish_threads()
self._cleanup()
src/ZEO/tests/ConnectionTests.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
import time
import socket
import asyncore
import threading
import logging
import ZEO.ServerStub
from ZEO.ClientStorage import ClientStorage
from ZEO.Exceptions import ClientDisconnected
from ZEO.zrpc.marshal import encode
from ZEO.tests import forker
from ZODB.DB import DB
from ZODB.POSException import ReadOnlyError, ConflictError
from ZODB.tests.StorageTestBase import StorageTestBase
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase \
import zodb_pickle, zodb_unpickle, handle_all_serials, handle_serials
import ZODB.tests.util
import transaction
from transaction import Transaction
logger = logging.getLogger('ZEO.tests.ConnectionTests')
ZERO = '\0'*8
class TestServerStub(ZEO.ServerStub.StorageServer):
__super_getInvalidations = ZEO.ServerStub.StorageServer.getInvalidations
def getInvalidations(self, tid):
# squirrel the results away for inspection by test case
self._last_invals = self.__super_getInvalidations(tid)
return self._last_invals
class TestClientStorage(ClientStorage):
test_connection = False
StorageServerStubClass = TestServerStub
connection_count_for_tests = 0
def notifyConnected(self, conn):
ClientStorage.notifyConnected(self, conn)
self.connection_count_for_tests += 1
def verify_cache(self, stub):
self.end_verify = threading.Event()
self.verify_result = ClientStorage.verify_cache(self, stub)
def endVerify(self):
ClientStorage.endVerify(self)
self.end_verify.set()
def testConnection(self, conn):
try:
return ClientStorage.testConnection(self, conn)
finally:
self.test_connection = True
class DummyDB:
def invalidate(self, *args, **kwargs):
pass
def invalidateCache(self):
pass
class CommonSetupTearDown(StorageTestBase):
"""Common boilerplate"""
__super_setUp = StorageTestBase.setUp
__super_tearDown = StorageTestBase.tearDown
keep = 0
invq = None
timeout = None
monitor = 0
db_class = DummyDB
def setUp(self, before=None):
"""Test setup for connection tests.
This starts only one server; a test may start more servers by
calling self._newAddr() and then self.startServer(index=i)
for i in 1, 2, ...
"""
self.__super_setUp()
logging.info("setUp() %s", self.id())
self.file = 'storage_conf'
self.addr = []
self._pids = []
self._servers = []
self.conf_paths = []
self.caches = []
self._newAddr()
self.startServer()
# self._old_log_level = logging.getLogger().getEffectiveLevel()
# logging.getLogger().setLevel(logging.WARNING)
# self._log_handler = logging.StreamHandler()
# logging.getLogger().addHandler(self._log_handler)
def tearDown(self):
"""Try to cause the tests to halt"""
# logging.getLogger().setLevel(self._old_log_level)
# logging.getLogger().removeHandler(self._log_handler)
# logging.info("tearDown() %s" % self.id())
for p in self.conf_paths:
os.remove(p)
if getattr(self, '_storage', None) is not None:
self._storage.close()
if hasattr(self._storage, 'cleanup'):
logging.debug("cleanup storage %s" %
self._storage.__name__)
self._storage.cleanup()
for adminaddr in self._servers:
if adminaddr is not None:
forker.shutdown_zeo_server(adminaddr)
for pid in self._pids:
try:
os.waitpid(pid, 0)
except OSError:
pass # The subprocess module may already have waited
for c in self.caches:
for i in 0, 1:
for ext in "", ".trace", ".lock":
path = "%s-%s.zec%s" % (c, "1", ext)
# On Windows before 2.3, we don't have a way to wait for
# the spawned server(s) to close, and they inherited
# file descriptors for our open files. So long as those
# processes are alive, we can't delete the files. Try
# a few times then give up.
need_to_delete = False
if os.path.exists(path):
need_to_delete = True
for dummy in range(5):
try:
os.unlink(path)
except:
time.sleep(0.5)
else:
need_to_delete = False
break
if need_to_delete:
os.unlink(path) # sometimes this is just gonna fail
self.__super_tearDown()
def _newAddr(self):
self.addr.append(self._getAddr())
def _getAddr(self):
return 'localhost', forker.get_port(self)
def getConfig(self, path, create, read_only):
raise NotImplementedError
cache_id = 1
def openClientStorage(self, cache=None, cache_size=200000, wait=1,
read_only=0, read_only_fallback=0,
username=None, password=None, realm=None):
if cache is None:
cache = str(self.__class__.cache_id)
self.__class__.cache_id += 1
self.caches.append(cache)
storage = TestClientStorage(self.addr,
client=cache,
var='.',
cache_size=cache_size,
wait=wait,
min_disconnect_poll=0.1,
read_only=read_only,
read_only_fallback=read_only_fallback,
username=username,
password=password,
realm=realm)
storage.registerDB(DummyDB())
return storage
def getServerConfig(self, addr, ro_svr):
zconf = forker.ZEOConfig(addr)
if ro_svr:
zconf.read_only = 1
if self.monitor:
zconf.monitor_address = ("", 42000)
if self.invq:
zconf.invalidation_queue_size = self.invq
if self.timeout:
zconf.transaction_timeout = self.timeout
return zconf
def startServer(self, create=1, index=0, read_only=0, ro_svr=0, keep=None,
path=None):
addr = self.addr[index]
logging.info("startServer(create=%d, index=%d, read_only=%d) @ %s" %
(create, index, read_only, addr))
if path is None:
path = "%s.%d" % (self.file, index)
sconf = self.getConfig(path, create, read_only)
zconf = self.getServerConfig(addr, ro_svr)
if keep is None:
keep = self.keep
zeoport, adminaddr, pid, path = forker.start_zeo_server(
sconf, zconf, addr[1], keep)
self.conf_paths.append(path)
self._pids.append(pid)
self._servers.append(adminaddr)
def shutdownServer(self, index=0):
logging.info("shutdownServer(index=%d) @ %s" %
(index, self._servers[index]))
adminaddr = self._servers[index]
if adminaddr is not None:
forker.shutdown_zeo_server(adminaddr)
self._servers[index] = None
def pollUp(self, timeout=30.0, storage=None):
if storage is None:
storage = self._storage
# Poll until we're connected.
now = time.time()
giveup = now + timeout
while not storage.is_connected():
asyncore.poll(0.1)
now = time.time()
if now > giveup:
self.fail("timed out waiting for storage to connect")
# When the socket map is empty, poll() returns immediately,
# and this is a pure busy-loop then. At least on some Linux
# flavors, that can starve the thread trying to connect,
# leading to grossly increased runtime (typical) or bogus
# "timed out" failures. A little sleep here cures both.
time.sleep(0.1)
def pollDown(self, timeout=30.0):
# Poll until we're disconnected.
now = time.time()
giveup = now + timeout
while self._storage.is_connected():
asyncore.poll(0.1)
now = time.time()
if now > giveup:
self.fail("timed out waiting for storage to disconnect")
# See pollUp() for why we sleep a little here.
time.sleep(0.1)
class ConnectionTests(CommonSetupTearDown):
"""Tests that explicitly manage the server process.
To test the cache or re-connection, these test cases explicit
start and stop a ZEO storage server.
"""
def checkMultipleAddresses(self):
for i in range(4):
self._newAddr()
self._storage = self.openClientStorage('test', 100000)
oid = self._storage.new_oid()
obj = MinPO(12)
self._dostore(oid, data=obj)
self._storage.close()
def checkReadOnlyClient(self):
# Open a read-only client to a read-write server; stores fail
# Start a read-only client for a read-write server
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
def checkReadOnlyServer(self):
# Open a read-only client to a read-only *server*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, ro_svr=1)
# Start a read-only client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
# Get rid of the 'test left new threads behind' warning
time.sleep(0.1)
def checkReadOnlyFallbackWritable(self):
# Open a fallback client to a read-write server; stores succeed
# Start a read-only-fallback client for a read-write server
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should succeed here
self._dostore()
self._storage.close()
def checkReadOnlyFallbackReadOnlyServer(self):
# Open a fallback client to a read-only *server*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, ro_svr=1)
# Start a read-only-fallback client
self._storage = self.openClientStorage(read_only_fallback=1)
self.assert_(self._storage.isReadOnly())
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
def checkDisconnectionError(self):
# Make sure we get a ClientDisconnected when we try to read an
# object when we're not connected to a storage server and the
# object is not in the cache.
self.shutdownServer()
self._storage = self.openClientStorage('test', 1000, wait=0)
self.assertRaises(ClientDisconnected,
self._storage.load, 'fredwash', '')
self._storage.close()
def checkBasicPersistence(self):
# Verify cached data persists across client storage instances.
# To verify that the cache is being used, the test closes the
# server and then starts a new client with the server down.
# When the server is down, a load() gets the data from its cache.
self._storage = self.openClientStorage('test', 100000)
oid = self._storage.new_oid()
obj = MinPO(12)
revid1 = self._dostore(oid, data=obj)
self._storage.close()
self.shutdownServer()
self._storage = self.openClientStorage('test', 100000, wait=0)
data, revid2 = self._storage.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO(12))
self.assertEqual(revid1, revid2)
self._storage.close()
def checkDisconnectedCacheWorks(self):
# Check that the cache works when the client is disconnected.
self._storage = self.openClientStorage('test')
oid1 = self._storage.new_oid()
obj1 = MinPO("1" * 500)
self._dostore(oid1, data=obj1)
oid2 = self._storage.new_oid()
obj2 = MinPO("2" * 500)
self._dostore(oid2, data=obj2)
expected1 = self._storage.load(oid1, '')
expected2 = self._storage.load(oid2, '')
# Shut it all down, and try loading from the persistent cache file
# without a server present.
self._storage.close()
self.shutdownServer()
self._storage = self.openClientStorage('test', wait=False)
self.assertEqual(expected1, self._storage.load(oid1, ''))
self.assertEqual(expected2, self._storage.load(oid2, ''))
self._storage.close()
def checkDisconnectedCacheFails(self):
# Like checkDisconnectedCacheWorks above, except the cache
# file is so small that only one object can be remembered.
self._storage = self.openClientStorage('test', cache_size=900)
oid1 = self._storage.new_oid()
obj1 = MinPO("1" * 500)
self._dostore(oid1, data=obj1)
oid2 = self._storage.new_oid()
obj2 = MinPO("2" * 500)
# The cache file is so small that adding oid2 will evict oid1.
self._dostore(oid2, data=obj2)
expected2 = self._storage.load(oid2, '')
# Shut it all down, and try loading from the persistent cache file
# without a server present.
self._storage.close()
self.shutdownServer()
self._storage = self.openClientStorage('test', cache_size=900,
wait=False)
# oid2 should still be in cache.
self.assertEqual(expected2, self._storage.load(oid2, ''))
# But oid1 should have been purged, so that trying to load it will
# try to fetch it from the (non-existent) ZEO server.
self.assertRaises(ClientDisconnected, self._storage.load, oid1, '')
self._storage.close()
def checkVerificationInvalidationPersists(self):
# This tests a subtle invalidation bug from ZODB 3.3:
# invalidations processed as part of ZEO cache verification acted
# kinda OK wrt the in-memory cache structures, but had no effect
# on the cache file. So opening the file cache again could
# incorrectly believe that a previously invalidated object was
# still current. This takes some effort to set up.
# First, using a persistent cache ('test'), create an object
# MinPO(13). We used to see this again at the end of this test,
# despite that we modify it, and despite that it gets invalidated
# in 'test', before the end.
self._storage = self.openClientStorage('test')
oid = self._storage.new_oid()
obj = MinPO(13)
self._dostore(oid, data=obj)
self._storage.close()
# Now modify obj via a temp connection. `test` won't learn about
# this until we open a connection using `test` again.
self._storage = self.openClientStorage()
pickle, rev = self._storage.load(oid, '')
newobj = zodb_unpickle(pickle)
self.assertEqual(newobj, obj)
newobj.value = 42 # .value *should* be 42 forever after now, not 13
self._dostore(oid, data=newobj, revid=rev)
self._storage.close()
# Open 'test' again. `oid` in this cache should be (and is)
# invalidated during cache verification. The bug was that it
# got invalidated (kinda) in memory, but not in the cache file.
self._storage = self.openClientStorage('test')
# The invalidation happened already. Now create and store a new
# object before closing this storage: this is so `test` believes
# it's seen transactions beyond the one that invalidated `oid`, so
# that the *next* time we open `test` it doesn't process another
# invalidation for `oid`. It's also important that we not try to
# load `oid` now: because it's been (kinda) invalidated in the
# cache's memory structures, loading it now would fetch the
# current revision from the server, thus hiding the bug.
obj2 = MinPO(666)
oid2 = self._storage.new_oid()
self._dostore(oid2, data=obj2)
self._storage.close()
# Finally, open `test` again and load `oid`. `test` believes
# it's beyond the transaction that modified `oid`, so its view
# of whether it has an up-to-date `oid` comes solely from the disk
# file, unaffected by cache verification.
self._storage = self.openClientStorage('test')
pickle, rev = self._storage.load(oid, '')
newobj_copy = zodb_unpickle(pickle)
# This used to fail, with
# AssertionError: MinPO(13) != MinPO(42)
# That is, `test` retained a stale revision of the object on disk.
self.assertEqual(newobj_copy, newobj)
self._storage.close()
def checkBadMessage1(self):
# not even close to a real message
self._bad_message("salty")
def checkBadMessage2(self):
# just like a real message, but with an unpicklable argument
global Hack
class Hack:
pass
msg = encode(1, 0, "foo", (Hack(),))
self._bad_message(msg)
del Hack
def _bad_message(self, msg):
# Establish a connection, then send the server an ill-formatted
# request. Verify that the connection is closed and that it is
# possible to establish a new connection.
self._storage = self.openClientStorage()
self._dostore()
# break into the internals to send a bogus message
zrpc_conn = self._storage._server.rpc
zrpc_conn.message_output(msg)
try:
self._dostore()
except ClientDisconnected:
pass
else:
self._storage.close()
self.fail("Server did not disconnect after bogus message")
self._storage.close()
self._storage = self.openClientStorage()
self._dostore()
self._storage.close()
# Test case for multiple storages participating in a single
# transaction. This is not really a connection test, but it needs
# about the same infrastructure (several storage servers).
# TODO: with the current ZEO code, this occasionally fails.
# That's the point of this test. :-)
def NOcheckMultiStorageTransaction(self):
# Configuration parameters (larger values mean more likely deadlocks)
N = 2
# These don't *have* to be all the same, but it's convenient this way
self.nservers = N
self.nthreads = N
self.ntrans = N
self.nobj = N
# Start extra servers
for i in range(1, self.nservers):
self._newAddr()
self.startServer(index=i)
# Spawn threads that each do some transactions on all storages
threads = []
try:
for i in range(self.nthreads):
t = MSTThread(self, "T%d" % i)
threads.append(t)
t.start()
# Wait for all threads to finish
for t in threads:
t.join(60)
self.failIf(t.isAlive(), "%s didn't die" % t.getName())
finally:
for t in threads:
t.closeclients()
def checkCrossDBInvalidations(self):
db1 = DB(self.openClientStorage())
c1 = db1.open()
r1 = c1.root()
r1["a"] = MinPO("a")
transaction.commit()
self.assertEqual(r1._p_state, 0) # up-to-date
db2 = DB(self.openClientStorage())
r2 = db2.open().root()
self.assertEqual(r2["a"].value, "a")
r2["b"] = MinPO("b")
transaction.commit()
# Make sure the invalidation is received in the other client.
# We've had problems with this timing out on "slow" and/or "very
# busy" machines, so we increase the sleep time on each trip, and
# are willing to wait quite a long time.
for i in range(20):
c1.sync()
if r1._p_state == -1:
break
time.sleep(i / 10.0)
self.assertEqual(r1._p_state, -1) # ghost
r1.keys() # unghostify
self.assertEqual(r1._p_serial, r2._p_serial)
self.assertEqual(r1["b"].value, "b")
db2.close()
db1.close()
def checkCheckForOutOfDateServer(self):
# We don't want to connect a client to a server if the client
# has seen newer transactions.
self._storage = self.openClientStorage()
self._dostore()
self.shutdownServer()
self.assertRaises(ClientDisconnected, self._storage.load, '\0'*8, '')
self.startServer()
# No matter how long we wait, the client won't reconnect:
time.sleep(2)
self.assertRaises(ClientDisconnected, self._storage.load, '\0'*8, '')
class InvqTests(CommonSetupTearDown):
invq = 3
def checkQuickVerificationWith2Clients(self):
perstorage = self.openClientStorage(cache="test", cache_size=4000)
self.assertEqual(perstorage.verify_result, "empty cache")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
oid2 = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "empty cache")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
revid = self._dostore(oid, revid)
# Create a second object and revision to guarantee it doesn't
# show up in the list of invalidations sent when perstore restarts.
revid2 = self._dostore(oid2)
revid2 = self._dostore(oid2, revid2)
perstorage.load(oid, '')
perstorage.close()
forker.wait_until(lambda : os.path.exists('test-1.zec'))
revid = self._dostore(oid, revid)
perstorage = self.openClientStorage(cache="test")
forker.wait_until(
(lambda : perstorage.verify_result == "quick verification"),
onfail=(lambda : None))
self.assertEqual(perstorage.verify_result, "quick verification")
self.assertEqual(perstorage._server._last_invals,
(revid, [oid]))
self.assertEqual(perstorage.load(oid, ''),
self._storage.load(oid, ''))
perstorage.close()
def checkVerificationWith2ClientsInvqOverflow(self):
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "empty cache")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "empty cache")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
revid = self._dostore(oid, revid)
forker.wait_until(
"Client has seen all of the transactions from the server",
lambda :
perstorage.lastTransaction() == self._storage.lastTransaction()
)
perstorage.load(oid, '')
perstorage.close()
# the test code sets invq bound to 2
for i in range(5):
revid = self._dostore(oid, revid)
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
t = time.time() + 30
while not perstorage.end_verify.isSet():
perstorage.sync()
if time.time() > t:
self.fail("timed out waiting for endVerify")
self.assertEqual(self._storage.load(oid, '')[1], revid)
self.assertEqual(perstorage.load(oid, ''),
self._storage.load(oid, ''))
perstorage.close()
class ReconnectionTests(CommonSetupTearDown):
# The setUp() starts a server automatically. In order for its
# state to persist, we set the class variable keep to 1. In
# order for its state to be cleaned up, the last startServer()
# call in the test must pass keep=0.
keep = 1
invq = 2
def checkReadOnlyStorage(self):
# Open a read-only client to a read-only *storage*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a read-only client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
def checkReadOnlyFallbackReadOnlyStorage(self):
# Open a fallback client to a read-only *storage*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a read-only-fallback client
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# TODO: Compare checkReconnectXXX() here to checkReconnection()
# further down. Is the code here hopelessly naive, or is
# checkReconnection() overwrought?
def checkReconnectWritable(self):
# A read-write client reconnects to a read-write server
# Start a client
self._storage = self.openClientStorage()
# Stores should succeed here
self._dostore()
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server
self.startServer(create=0)
# Poll until the client connects
self.pollUp()
# Stores should succeed here
self._dostore()
self._storage.close()
def checkReconnectReadOnly(self):
# A read-only client reconnects from a read-write to a
# read-only server
# Start a client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should still fail
self.assertRaises(ReadOnlyError, self._dostore)
# Restart the server
self.startServer(create=0, read_only=1, keep=0)
# Poll until the client connects
self.pollUp()
# Stores should still fail
self.assertRaises(ReadOnlyError, self._dostore)
def checkReconnectFallback(self):
# A fallback client reconnects from a read-write to a
# read-only server
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should succeed here
self._dostore()
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server
self.startServer(create=0, read_only=1, keep=0)
# Poll until the client connects
self.pollUp()
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
def checkReconnectUpgrade(self):
# A fallback client reconnects from a read-only to a
# read-write server
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, read_only=1)
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server, this time read-write
self.startServer(create=0, keep=0)
# Poll until the client sconnects
self.pollUp()
# Stores should now succeed
self._dostore()
def checkReconnectSwitch(self):
# A fallback client initially connects to a read-only server,
# then discovers a read-write server and switches to that
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Allocate a second address (for the second server)
self._newAddr()
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Start a read-write server
self.startServer(index=1, read_only=0, keep=0)
# After a while, stores should work
for i in range(300): # Try for 30 seconds
try:
self._dostore()
break
except (ClientDisconnected, ReadOnlyError):
# If the client isn't connected at all, sync() returns
# quickly and the test fails because it doesn't wait
# long enough for the client.
time.sleep(0.1)
else:
self.fail("Couldn't store after starting a read-write server")
def checkNoVerificationOnServerRestart(self):
self._storage = self.openClientStorage()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "empty cache")
self._dostore()
self.shutdownServer()
self.pollDown()
self._storage.verify_result = None
self.startServer(create=0, keep=0)
self.pollUp()
# There were no transactions committed, so no verification
# should be needed.
self.assertEqual(self._storage.verify_result, "no verification")
def checkNoVerificationOnServerRestartWith2Clients(self):
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "empty cache")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "empty cache")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
revid = self._dostore(oid, revid)
forker.wait_until(
"Client has seen all of the transactions from the server",
lambda :
perstorage.lastTransaction() == self._storage.lastTransaction()
)
perstorage.load(oid, '')
self.shutdownServer()
self.pollDown()
self._storage.verify_result = None
perstorage.verify_result = None
logging.info('2ALLBEEF')
self.startServer(create=0, keep=0)
self.pollUp()
self.pollUp(storage=perstorage)
# There were no transactions committed, so no verification
# should be needed.
self.assertEqual(self._storage.verify_result, "no verification")
self.assertEqual(perstorage.verify_result, "no verification")
perstorage.close()
self._storage.close()
def checkDisconnectedAbort(self):
self._storage = self.openClientStorage()
self._dostore()
oids = [self._storage.new_oid() for i in range(5)]
txn = Transaction()
self._storage.tpc_begin(txn)
for oid in oids:
data = zodb_pickle(MinPO(oid))
self._storage.store(oid, None, data, '', txn)
self.shutdownServer()
self.assertRaises(ClientDisconnected, self._storage.tpc_vote, txn)
self._storage.tpc_abort(txn)
self.startServer(create=0)
self._storage._wait()
self._dostore()
# This test is supposed to cover the following error, although
# I don't have much confidence that it does. The likely
# explanation for the error is that the _tbuf contained
# objects that weren't in the _seriald, because the client was
# interrupted waiting for tpc_vote() to return. When the next
# transaction committed, it tried to do something with the
# bogus _tbuf entries. The explanation is wrong/incomplete,
# because tpc_begin() should clear the _tbuf.
# 2003-01-15T15:44:19 ERROR(200) ZODB A storage error occurred
# in the last phase of a two-phase commit. This shouldn't happen.
# Traceback (innermost last):
# Module ZODB.Transaction, line 359, in _finish_one
# Module ZODB.Connection, line 691, in tpc_finish
# Module ZEO.ClientStorage, line 679, in tpc_finish
# Module ZEO.ClientStorage, line 709, in _update_cache
# KeyError: ...
def checkReconnection(self):
# Check that the client reconnects when a server restarts.
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
obj = MinPO(12)
self._dostore(oid, data=obj)
logging.info("checkReconnection(): About to shutdown server")
self.shutdownServer()
logging.info("checkReconnection(): About to restart server")
self.startServer(create=0)
forker.wait_until('reconnect', self._storage.is_connected)
oid = self._storage.new_oid()
obj = MinPO(12)
while 1:
try:
self._dostore(oid, data=obj)
break
except ClientDisconnected:
# Maybe the exception mess is better now
logging.info("checkReconnection(): Error after"
" server restart; retrying.", exc_info=True)
transaction.abort()
# Give the other thread a chance to run.
time.sleep(0.1)
logging.info("checkReconnection(): finished")
self._storage.close()
def checkMultipleServers(self):
# Crude test-- just start two servers and do a commit at each one.
self._newAddr()
self._storage = self.openClientStorage('test', 100000)
self._dostore()
self.shutdownServer(index=0)
# When we start the second server, we use file data file from
# the original server so tha the new server is a replica of
# the original. We need this becaise ClientStorage won't use
# a server if the server's last transaction is earlier than
# what the client has seen.
self.startServer(index=1, path=self.file+'.0', create=False)
# If we can still store after shutting down one of the
# servers, we must be reconnecting to the other server.
did_a_store = 0
for i in range(10):
try:
self._dostore()
did_a_store = 1
break
except ClientDisconnected:
time.sleep(0.5)
self.assert_(did_a_store)
self._storage.close()
class TimeoutTests(CommonSetupTearDown):
timeout = 1
def checkTimeout(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_vote(txn)
time.sleep(2)
self.assertRaises(ClientDisconnected, storage.tpc_finish, txn)
# Make sure it's logged as CRITICAL
for line in open("server-%s.log" % self.addr[0][1]):
if (('Transaction timeout after' in line) and
('CRITICAL ZEO.StorageServer' in line)
):
break
else:
self.assert_(False, 'bad logging')
storage.close()
def checkTimeoutOnAbort(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_vote(txn)
storage.tpc_abort(txn)
storage.close()
def checkTimeoutOnAbortNoLock(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_abort(txn)
storage.close()
def checkTimeoutAfterVote(self):
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty
self.assert_(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# Now do a store, sleeping before the finish so as to cause a timeout
t = Transaction()
old_connection_count = storage.connection_count_for_tests
storage.tpc_begin(t)
revid1 = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
storage.tpc_vote(t)
# Now sleep long enough for the storage to time out
time.sleep(3)
self.assert_(
(not storage.is_connected())
or
(storage.connection_count_for_tests > old_connection_count)
)
storage._wait()
self.assert_(storage.is_connected())
# We expect finish to fail
self.assertRaises(ClientDisconnected, storage.tpc_finish, t)
# The cache should still be empty
self.assert_(not list(storage._cache.contents()))
# Load should fail since the object should not be in either the cache
# or the server.
self.assertRaises(KeyError, storage.load, oid, '')
def checkTimeoutProvokingConflicts(self):
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty.
self.assert_(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# We need to successfully commit an object now so we have something to
# conflict about.
t = Transaction()
storage.tpc_begin(t)
revid1a = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
revid1b = storage.tpc_vote(t)
revid1 = handle_serials(oid, revid1a, revid1b)
storage.tpc_finish(t)
# Now do a store, sleeping before the finish so as to cause a timeout.
obj.value = 8
t = Transaction()
old_connection_count = storage.connection_count_for_tests
storage.tpc_begin(t)
revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
revid2b = storage.tpc_vote(t)
revid2 = handle_serials(oid, revid2a, revid2b)
# Now sleep long enough for the storage to time out.
# This used to sleep for 3 seconds, and sometimes (but very rarely)
# failed then. Now we try for a minute. It typically succeeds
# on the second time thru the loop, and, since self.timeout is 1,
# it's typically faster now (2/1.8 ~= 1.11 seconds sleeping instead
# of 3).
deadline = time.time() + 60 # wait up to a minute
while time.time() < deadline:
if (storage.is_connected() and
(storage.connection_count_for_tests == old_connection_count)
):
time.sleep(self.timeout / 1.8)
else:
break
self.assert_(
(not storage.is_connected())
or
(storage.connection_count_for_tests > old_connection_count)
)
storage._wait()
self.assert_(storage.is_connected())
# We expect finish to fail.
self.assertRaises(ClientDisconnected, storage.tpc_finish, t)
storage.tpc_abort(t)
# Now we think we've committed the second transaction, but we really
# haven't. A third one should produce a POSKeyError on the server,
# which manifests as a ConflictError on the client.
obj.value = 9
t = Transaction()
storage.tpc_begin(t)
storage.store(oid, revid2, zodb_pickle(obj), '', t)
self.assertRaises(ConflictError, storage.tpc_vote, t)
# Even aborting won't help.
storage.tpc_abort(t)
self.assertRaises(ZODB.POSException.StorageTransactionError,
storage.tpc_finish, t)
# Try again.
obj.value = 10
t = Transaction()
storage.tpc_begin(t)
storage.store(oid, revid2, zodb_pickle(obj), '', t)
# Even aborting won't help.
self.assertRaises(ConflictError, storage.tpc_vote, t)
# Abort this one and try a transaction that should succeed.
storage.tpc_abort(t)
# Now do a store.
obj.value = 11
t = Transaction()
storage.tpc_begin(t)
revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
revid2b = storage.tpc_vote(t)
revid2 = handle_serials(oid, revid2a, revid2b)
storage.tpc_finish(t)
# Now load the object and verify that it has a value of 11.
data, revid = storage.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO(11))
self.assertEqual(revid, revid2)
class MSTThread(threading.Thread):
__super_init = threading.Thread.__init__
def __init__(self, testcase, name):
self.__super_init(name=name)
self.testcase = testcase
self.clients = []
def run(self):
tname = self.getName()
testcase = self.testcase
# Create client connections to each server
clients = self.clients
for i in range(len(testcase.addr)):
c = testcase.openClientStorage(addr=testcase.addr[i])
c.__name = "C%d" % i
clients.append(c)
for i in range(testcase.ntrans):
# Because we want a transaction spanning all storages,
# we can't use _dostore(). This is several _dostore() calls
# expanded in-line (mostly).
# Create oid->serial mappings
for c in clients:
c.__oids = []
c.__serials = {}
# Begin a transaction
t = Transaction()
for c in clients:
#print "%s.%s.%s begin\n" % (tname, c.__name, i),
c.tpc_begin(t)
for j in range(testcase.nobj):
for c in clients:
# Create and store a new object on each server
oid = c.new_oid()
c.__oids.append(oid)
data = MinPO("%s.%s.t%d.o%d" % (tname, c.__name, i, j))
#print data.value
data = zodb_pickle(data)
s = c.store(oid, ZERO, data, '', t)
c.__serials.update(handle_all_serials(oid, s))
# Vote on all servers and handle serials
for c in clients:
#print "%s.%s.%s vote\n" % (tname, c.__name, i),
s = c.tpc_vote(t)
c.__serials.update(handle_all_serials(None, s))
# Finish on all servers
for c in clients:
#print "%s.%s.%s finish\n" % (tname, c.__name, i),
c.tpc_finish(t)
for c in clients:
# Check that we got serials for all oids
for oid in c.__oids:
testcase.failUnless(c.__serials.has_key(oid))
# Check that we got serials for no other oids
for oid in c.__serials.keys():
testcase.failUnless(oid in c.__oids)
def closeclients(self):
# Close clients opened by run()
for c in self.clients:
try:
c.close()
except:
pass
# Run IPv6 tests if V6 sockets are supported
try:
socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
except (socket.error, AttributeError):
pass
else:
class V6Setup:
def _getAddr(self):
return '::1', forker.get_port(self)
_g = globals()
for name, value in _g.items():
if isinstance(value, type) and issubclass(value, CommonSetupTearDown):
_g[name+"V6"] = type(name+"V6", (V6Setup, value), {})
src/ZEO/tests/InvalidationTests.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import threading
import time
from random import Random
import transaction
from BTrees.check import check, display
from BTrees.OOBTree import OOBTree
from ZEO.tests.TestThread import TestThread
from ZODB.DB import DB
from ZODB.POSException import ReadConflictError, ConflictError
# The tests here let several threads have a go at one or more database
# instances simultaneously. Each thread appends a disjoint (from the
# other threads) sequence of increasing integers to an OOBTree, one at
# at time (per thread). This provokes lots of conflicts, and BTrees
# work hard at conflict resolution too. An OOBTree is used because
# that flavor has the smallest maximum bucket size, and so splits buckets
# more often than other BTree flavors.
#
# When these tests were first written, they provoked an amazing number
# of obscure timing-related bugs in cache consistency logic, revealed
# by failure of the BTree to pass internal consistency checks at the end,
# and/or by failure of the BTree to contain all the keys the threads
# thought they added (i.e., the keys for which transaction.commit()
# did not raise any exception).
class FailableThread(TestThread):
# mixin class
# subclass must provide
# - self.stop attribute (an event)
# - self._testrun() method
# TestThread.run() invokes testrun().
def testrun(self):
try:
self._testrun()
except:
# Report the failure here to all the other threads, so
# that they stop quickly.
self.stop.set()
raise
class StressTask:
# Append integers startnum, startnum + step, startnum + 2*step, ...
# to 'tree'. If sleep is given, sleep
# that long after each append. At the end, instance var .added_keys
# is a list of the ints the thread believes it added successfully.
def __init__(self, db, threadnum, startnum, step=2, sleep=None):
self.db = db
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.tm = transaction.TransactionManager()
self.cn = self.db.open(transaction_manager=self.tm)
self.cn.sync()
def doStep(self):
tree = self.cn.root()["tree"]
key = self.startnum
tree[key] = self.threadnum
def commit(self):
cn = self.cn
key = self.startnum
self.tm.get().note("add key %s" % key)
try:
self.tm.get().commit()
except ConflictError, msg:
self.tm.abort()
else:
if self.sleep:
time.sleep(self.sleep)
self.added_keys.append(key)
self.startnum += self.step
def cleanup(self):
self.tm.get().abort()
self.cn.close()
def _runTasks(rounds, *tasks):
'''run *task* interleaved for *rounds* rounds.'''
def commit(run, actions):
actions.append(':')
for t in run:
t.commit()
del run[:]
r = Random()
r.seed(1064589285) # make it deterministic
run = []
actions = []
try:
for i in range(rounds):
t = r.choice(tasks)
if t in run:
commit(run, actions)
run.append(t)
t.doStep()
actions.append(`t.startnum`)
commit(run,actions)
# stderr.write(' '.join(actions)+'\n')
finally:
for t in tasks:
t.cleanup()
class StressThread(FailableThread):
# Append integers startnum, startnum + step, startnum + 2*step, ...
# to 'tree' until Event stop is set. If sleep is given, sleep
# that long after each append. At the end, instance var .added_keys
# is a list of the ints the thread believes it added successfully.
def __init__(self, testcase, db, stop, threadnum, commitdict,
startnum, step=2, sleep=None):
TestThread.__init__(self, testcase)
self.db = db
self.stop = stop
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.commitdict = commitdict
def _testrun(self):
tm = transaction.TransactionManager()
cn = self.db.open(transaction_manager=tm)
while not self.stop.isSet():
try:
tree = cn.root()["tree"]
break
except (ConflictError, KeyError):
tm.abort()
key = self.startnum
while not self.stop.isSet():
try:
tree[key] = self.threadnum
tm.get().note("add key %s" % key)
tm.commit()
self.commitdict[self] = 1
if self.sleep:
time.sleep(self.sleep)
except (ReadConflictError, ConflictError), msg:
tm.abort()
else:
self.added_keys.append(key)
key += self.step
cn.close()
class LargeUpdatesThread(FailableThread):
# A thread that performs a lot of updates. It attempts to modify
# more than 25 objects so that it can test code that runs vote
# in a separate thread when it modifies more than 25 objects.
def __init__(self, test, db, stop, threadnum, commitdict, startnum,
step=2, sleep=None):
TestThread.__init__(self, test)
self.db = db
self.stop = stop
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.commitdict = commitdict
def _testrun(self):
cn = self.db.open()
while not self.stop.isSet():
try:
tree = cn.root()["tree"]
break
except (ConflictError, KeyError):
# print "%d getting tree abort" % self.threadnum
transaction.abort()
keys_added = {} # set of keys we commit
tkeys = []
while not self.stop.isSet():
# The test picks 50 keys spread across many buckets.
# self.startnum and self.step ensure that all threads use
# disjoint key sets, to minimize conflict errors.
nkeys = len(tkeys)
if nkeys < 50:
tkeys = range(self.startnum, 3000, self.step)
nkeys = len(tkeys)
step = max(int(nkeys / 50), 1)
keys = [tkeys[i] for i in range(0, nkeys, step)]
for key in keys:
try:
tree[key] = self.threadnum
except (ReadConflictError, ConflictError), msg:
# print "%d setting key %s" % (self.threadnum, msg)
transaction.abort()
break
else:
# print "%d set #%d" % (self.threadnum, len(keys))
transaction.get().note("keys %s" % ", ".join(map(str, keys)))
try:
transaction.commit()
self.commitdict[self] = 1
if self.sleep:
time.sleep(self.sleep)
except ConflictError, msg:
# print "%d commit %s" % (self.threadnum, msg)
transaction.abort()
continue
for k in keys:
tkeys.remove(k)
keys_added[k] = 1
self.added_keys = keys_added.keys()
cn.close()
class InvalidationTests:
# Minimum # of seconds the main thread lets the workers run. The
# test stops as soon as this much time has elapsed, and all threads
# have managed to commit a change.
MINTIME = 10
# Maximum # of seconds the main thread lets the workers run. We
# stop after this long has elapsed regardless of whether all threads
# have managed to commit a change.
MAXTIME = 300
StressThread = StressThread
def _check_tree(self, cn, tree):
# Make sure the BTree is sane at the C level.
retries = 3
while retries:
retries -= 1
try:
check(tree)
tree._check()
except ReadConflictError:
if retries:
transaction.abort()
else:
raise
except:
display(tree)
raise
def _check_threads(self, tree, *threads):
# Make sure the thread's view of the world is consistent with
# the actual database state.
expected_keys = []
errormsgs = []
err = errormsgs.append
for t in threads:
if not t.added_keys:
err("thread %d didn't add any keys" % t.threadnum)
expected_keys.extend(t.added_keys)
expected_keys.sort()
for i in range(100):
tree._p_jar.sync()
actual_keys = list(tree.keys())
if expected_keys == actual_keys:
break
time.sleep(.1)
else:
err("expected keys != actual keys")
for k in expected_keys:
if k not in actual_keys:
err("key %s expected but not in tree" % k)
for k in actual_keys:
if k not in expected_keys:
err("key %s in tree but not expected" % k)
self.fail('\n'.join(errormsgs))
def go(self, stop, commitdict, *threads):
# Run the threads
for t in threads:
t.start()
delay = self.MINTIME
start = time.time()
while time.time() - start <= self.MAXTIME:
stop.wait(delay)
if stop.isSet():
# Some thread failed. Stop right now.
break
delay = 2.0
if len(commitdict) >= len(threads):
break
# Some thread still hasn't managed to commit anything.
stop.set()
# Give all the threads some time to stop before trying to clean up.
# cleanup() will cause the test to fail if some thread ended with
# an uncaught exception, and unittest will call the base class
# tearDown then immediately, but if other threads are still
# running that can lead to a cascade of spurious exceptions.
for t in threads:
t.join(30)
for t in threads:
t.cleanup(10)
def checkConcurrentUpdates2Storages_emulated(self):
self._storage = storage1 = self.openClientStorage()
storage2 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(storage2)
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
# DM: allow time for invalidations to come in and process them
time.sleep(0.1)
# Run two threads that update the BTree
t1 = StressTask(db1, 1, 1,)
t2 = StressTask(db2, 2, 2,)
_runTasks(100, t1, t2)
cn.sync()
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def checkConcurrentUpdates2Storages(self):
self._storage = storage1 = self.openClientStorage()
storage2 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(storage2)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1)
t2 = self.StressThread(self, db2, stop, 2, cd, 2)
self.go(stop, cd, t1, t2)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def checkConcurrentUpdates19Storages(self):
n = 19
dbs = [DB(self.openClientStorage()) for i in range(n)]
self._storage = dbs[0].storage
stop = threading.Event()
cn = dbs[0].open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run threads that update the BTree
cd = {}
threads = [self.StressThread(self, dbs[i], stop, i, cd, i, n)
for i in range(n)]
self.go(stop, cd, *threads)
while len(set(db.lastTransaction() for db in dbs)) > 1:
_ = [db._storage.sync() for db in dbs]
cn = dbs[0].open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, *threads)
cn.close()
_ = [db.close() for db in dbs]
def checkConcurrentUpdates1Storage(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1, sleep=0.01)
t2 = self.StressThread(self, db1, stop, 2, cd, 2, sleep=0.01)
self.go(stop, cd, t1, t2)
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
def checkConcurrentUpdates2StoragesMT(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1, 3)
t2 = self.StressThread(self, db2, stop, 2, cd, 2, 3, 0.01)
t3 = self.StressThread(self, db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
time.sleep(.1)
time.sleep(.1)
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
def checkConcurrentLargeUpdates(self):
# Use 3 threads like the 2StorageMT test above.
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
for i in range(0, 3000, 2):
tree[i] = 0
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = LargeUpdatesThread(self, db1, stop, 1, cd, 1, 3, 0.02)
t2 = LargeUpdatesThread(self, db2, stop, 2, cd, 2, 3, 0.01)
t3 = LargeUpdatesThread(self, db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
# Purge the tree of the dummy entries mapping to 0.
losers = [k for k, v in tree.items() if v == 0]
for k in losers:
del tree[k]
transaction.commit()
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
src/ZEO/tests/IterationTests.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2008 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZEO iterator protocol tests."""
import transaction
class IterationTests:
def checkIteratorGCProtocol(self):
# Test garbage collection on protocol level.
server = self._storage._server
iid = server.iterator_start(None, None)
# None signals the end of iteration.
self.assertEquals(None, server.iterator_next(iid))
# The server has disposed the iterator already.
self.assertRaises(KeyError, server.iterator_next, iid)
iid = server.iterator_start(None, None)
# This time, we tell the server to throw the iterator away.
server.iterator_gc([iid])
self.assertRaises(KeyError, server.iterator_next, iid)
def checkIteratorExhaustionStorage(self):
# Test the storage's garbage collection mechanism.
self._dostore()
iterator = self._storage.iterator()
# At this point, a wrapping iterator might not have called the CS
# iterator yet. We'll consume one item to make sure this happens.
iterator.next()
self.assertEquals(1, len(self._storage._iterator_ids))
iid = list(self._storage._iterator_ids)[0]
self.assertEquals([], list(iterator))
self.assertEquals(0, len(self._storage._iterator_ids))
# The iterator has run through, so the server has already disposed it.
self.assertRaises(KeyError, self._storage._server.iterator_next, iid)
def checkIteratorGCSpanTransactions(self):
# Keep a hard reference to the iterator so it won't be automatically
# garbage collected at the transaction boundary.
self._dostore()
iterator = self._storage.iterator()
self._dostore()
# As the iterator was not garbage collected, we can still use it. (We
# don't see the transaction we just wrote being picked up, because
# iterators only see the state from the point in time when they were
# created.)
self.assert_(list(iterator))
def checkIteratorGCStorageCommitting(self):
# We want the iterator to be garbage-collected, so we don't keep any
# hard references to it. The storage tracks its ID, though.
# The odd little jig we do below arises from the fact that the
# CS iterator may not be constructed right away if the CS is wrapped.
# We need to actually do some iteration to get the iterator created.
# We do a store to make sure the iterator isn't exhausted right away.
self._dostore()
self._storage.iterator().next()
self.assertEquals(1, len(self._storage._iterator_ids))
iid = list(self._storage._iterator_ids)[0]
# GC happens at the transaction boundary. After that, both the storage
# and the server have forgotten the iterator.
self._dostore()
self.assertEquals(0, len(self._storage._iterator_ids))
self.assertRaises(KeyError, self._storage._server.iterator_next, iid)
def checkIteratorGCStorageTPCAborting(self):
# The odd little jig we do below arises from the fact that the
# CS iterator may not be constructed right away if the CS is wrapped.
# We need to actually do some iteration to get the iterator created.
# We do a store to make sure the iterator isn't exhausted right away.
self._dostore()
self._storage.iterator().next()
iid = list(self._storage._iterator_ids)[0]
t = transaction.Transaction()
self._storage.tpc_begin(t)
self._storage.tpc_abort(t)
self.assertEquals(0, len(self._storage._iterator_ids))
self.assertRaises(KeyError, self._storage._server.iterator_next, iid)
def checkIteratorGCStorageDisconnect(self):
# The odd little jig we do below arises from the fact that the
# CS iterator may not be constructed right away if the CS is wrapped.
# We need to actually do some iteration to get the iterator created.
# We do a store to make sure the iterator isn't exhausted right away.
self._dostore()
self._storage.iterator().next()
iid = list(self._storage._iterator_ids)[0]
t = transaction.Transaction()
self._storage.tpc_begin(t)
# Show that after disconnecting, the client side GCs the iterators
# as well. I'm calling this directly to avoid accidentally
# calling tpc_abort implicitly.
self._storage.notifyDisconnected()
self.assertEquals(0, len(self._storage._iterator_ids))
def checkIteratorParallel(self):
self._dostore()
self._dostore()
iter1 = self._storage.iterator()
iter2 = self._storage.iterator()
txn_info1 = iter1.next()
txn_info2 = iter2.next()
self.assertEquals(txn_info1.tid, txn_info2.tid)
txn_info1 = iter1.next()
txn_info2 = iter2.next()
self.assertEquals(txn_info1.tid, txn_info2.tid)
self.assertRaises(StopIteration, iter1.next)
self.assertRaises(StopIteration, iter2.next)
def iterator_sane_after_reconnect():
r"""Make sure that iterators are invalidated on disconnect.
Start a server:
>>> addr, adminaddr = start_server(
... '<filestorage>\npath fs\n</filestorage>', keep=1)
Open a client storage to it and commit a some transactions:
>>> import ZEO, transaction
>>> db = ZEO.DB(addr)
>>> conn = db.open()
>>> for i in range(10):
... conn.root().i = i
... transaction.commit()
Create an iterator:
>>> it = conn._storage.iterator()
>>> tid1 = it.next().tid
Restart the storage:
>>> stop_server(adminaddr)
>>> wait_disconnected(conn._storage)
>>> _ = start_server('<filestorage>\npath fs\n</filestorage>', addr=addr)
>>> wait_connected(conn._storage)
Now, we'll create a second iterator:
>>> it2 = conn._storage.iterator()
If we try to advance the first iterator, we should get an error:
>>> it.next().tid > tid1
Traceback (most recent call last):
...
ClientDisconnected: Disconnected iterator
The second iterator should be peachy:
>>> it2.next().tid == tid1
True
Cleanup:
>>> db.close()
"""
src/ZEO/tests/TestThread.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A Thread base class for use with unittest."""
import threading
import sys
class TestThread(threading.Thread):
"""Base class for defining threads that run from unittest.
The subclass should define a testrun() method instead of a run()
method.
Call cleanup() when the test is done with the thread, instead of join().
If the thread exits with an uncaught exception, it's captured and
re-raised when cleanup() is called. cleanup() should be called by
the main thread! Trying to tell unittest that a test failed from
another thread creates a nightmare of timing-depending cascading
failures and missed errors (tracebacks that show up on the screen,
but don't cause unittest to believe the test failed).
cleanup() also joins the thread. If the thread ended without raising
an uncaught exception, and the join doesn't succeed in the timeout
period, then the test is made to fail with a "Thread still alive"
message.
"""
def __init__(self, testcase):
threading.Thread.__init__(self)
# In case this thread hangs, don't stop Python from exiting.
self.setDaemon(1)
self._exc_info = None
self._testcase = testcase
def run(self):
try:
self.testrun()
except:
self._exc_info = sys.exc_info()
def cleanup(self, timeout=15):
self.join(timeout)
if self._exc_info:
raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
if self.isAlive():
self._testcase.fail("Thread did not finish: %s" % self)
src/ZEO/tests/ThreadTests.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Compromising positions involving threads."""
import threading
import transaction
from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
import ZEO.ClientStorage
ZERO = '\0'*8
class BasicThread(threading.Thread):
def __init__(self, storage, doNextEvent, threadStartedEvent):
self.storage = storage
self.trans = transaction.Transaction()
self.doNextEvent = doNextEvent
self.threadStartedEvent = threadStartedEvent
self.gotValueError = 0
self.gotDisconnected = 0
threading.Thread.__init__(self)
self.setDaemon(1)
def join(self):
threading.Thread.join(self, 10)
assert not self.isAlive()
class GetsThroughVoteThread(BasicThread):
# This thread gets partially through a transaction before it turns
# execution over to another thread. We're trying to establish that a
# tpc_finish() after a storage has been closed by another thread will get
# a ClientStorageError error.
#
# This class gets does a tpc_begin(), store(), tpc_vote() and is waiting
# to do the tpc_finish() when the other thread closes the storage.
def run(self):
self.storage.tpc_begin(self.trans)
oid = self.storage.new_oid()
self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans)
self.storage.tpc_vote(self.trans)
self.threadStartedEvent.set()
self.doNextEvent.wait(10)
try:
self.storage.tpc_finish(self.trans)
except ZEO.ClientStorage.ClientStorageError:
self.gotValueError = 1
self.storage.tpc_abort(self.trans)
class GetsThroughBeginThread(BasicThread):
# This class is like the above except that it is intended to be run when
# another thread is already in a tpc_begin(). Thus, this thread will
# block in the tpc_begin until another thread closes the storage. When
# that happens, this one will get disconnected too.
def run(self):
try:
self.storage.tpc_begin(self.trans)
except ZEO.ClientStorage.ClientStorageError:
self.gotValueError = 1
class ThreadTests:
# Thread 1 should start a transaction, but not get all the way through it.
# Main thread should close the connection. Thread 1 should then get
# disconnected.
def checkDisconnectedOnThread2Close(self):
doNextEvent = threading.Event()
threadStartedEvent = threading.Event()
thread1 = GetsThroughVoteThread(self._storage,
doNextEvent, threadStartedEvent)
thread1.start()
threadStartedEvent.wait(10)
self._storage.close()
doNextEvent.set()
thread1.join()
self.assertEqual(thread1.gotValueError, 1)
# Thread 1 should start a transaction, but not get all the way through
# it. While thread 1 is in the middle of the transaction, a second thread
# should start a transaction, and it will block in the tcp_begin() --
# because thread 1 has acquired the lock in its tpc_begin(). Now the main
# thread closes the storage and both sub-threads should get disconnected.
def checkSecondBeginFails(self):
doNextEvent = threading.Event()
threadStartedEvent = threading.Event()
thread1 = GetsThroughVoteThread(self._storage,
doNextEvent, threadStartedEvent)
thread2 = GetsThroughBeginThread(self._storage,
doNextEvent, threadStartedEvent)
thread1.start()
threadStartedEvent.wait(1)
thread2.start()
self._storage.close()
doNextEvent.set()
thread1.join()
thread2.join()
self.assertEqual(thread1.gotValueError, 1)
self.assertEqual(thread2.gotValueError, 1)
# Run a bunch of threads doing small and large stores in parallel
def checkMTStores(self):
threads = []
for i in range(5):
t = threading.Thread(target=self.mtstorehelper)
threads.append(t)
t.start()
for t in threads:
t.join(30)
for i in threads:
self.failUnless(not t.isAlive())
# Helper for checkMTStores
def mtstorehelper(self):
name = threading.currentThread().getName()
objs = []
for i in range(10):
objs.append(MinPO("X" * 200000))
objs.append(MinPO("X"))
for obj in objs:
self._dostore(data=obj)
src/ZEO/tests/__init__.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
src/ZEO/tests/auth_plaintext.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Implements plaintext password authentication. The password is stored in
an SHA hash in the Database. The client sends over the plaintext
password, and the SHA hashing is done on the server side.
This mechanism offers *no network security at all*; the only security
is provided by not storing plaintext passwords on disk.
"""
from ZEO.hash import sha1
from ZEO.StorageServer import ZEOStorage
from ZEO.auth import register_module
from ZEO.auth.base import Client, Database
def session_key(username, realm, password):
return sha1("%s:%s:%s" % (username, realm, password)).hexdigest()
class StorageClass(ZEOStorage):
def auth(self, username, password):
try:
dbpw = self.database.get_password(username)
except LookupError:
return 0
password_dig = sha1(password).hexdigest()
if dbpw == password_dig:
self.connection.setSessionKey(session_key(username,
self.database.realm,
password))
return self._finish_auth(dbpw == password_dig)
class PlaintextClient(Client):
extensions = ["auth"]
def start(self, username, realm, password):
if self.stub.auth(username, password):
return session_key(username, realm, password)
else:
return None
register_module("plaintext", StorageClass, PlaintextClient, Database)
src/ZEO/tests/client-config.test deleted 100644 → 0
View file @ 56c7dd30
ZEO Client Configuration
========================
Here we'll describe (and test) the various ZEO Client configuration
options. To facilitate this, we'l start a server that our client can
connect to:
>>> addr, _ = start_server(blob_dir='server-blobs')
The simplest client configuration specified a server address:
>>> import ZODB.config
>>> storage = ZODB.config.storageFromString("""
... <zeoclient>
... server %s:%s
... </zeoclient>
... """ % addr)
>>> storage.getName(), storage.__class__.__name__
... # doctest: +ELLIPSIS
("[('localhost', ...)] (connected)", 'ClientStorage')
>>> storage.blob_dir
>>> storage._storage
'1'
>>> storage._cache.maxsize
20971520
>>> storage._cache.path
>>> storage._rpc_mgr.tmin
5
>>> storage._rpc_mgr.tmax
300
>>> storage._is_read_only
False
>>> storage._read_only_fallback
False
>>> storage._drop_cache_rather_verify
False
>>> storage._blob_cache_size
>>> storage.close()
>>> storage = ZODB.config.storageFromString("""
... <zeoclient>
... server %s:%s
... blob-dir blobs
... storage 2
... cache-size 100
... name bob
... client cache
... min-disconnect-poll 1
... max-disconnect-poll 5
... read-only true
... drop-cache-rather-verify true
... blob-cache-size 1000MB
... blob-cache-size-check 10
... wait false
... </zeoclient>
... """ % addr)
>>> storage.getName(), storage.__class__.__name__
('bob (disconnected)', 'ClientStorage')
>>> storage.blob_dir
'blobs'
>>> storage._storage
'2'
>>> storage._cache.maxsize
100
>>> import os
>>> storage._cache.path == os.path.abspath('cache-2.zec')
True
>>> storage._rpc_mgr.tmin
1
>>> storage._rpc_mgr.tmax
5
>>> storage._is_read_only
True
>>> storage._read_only_fallback
False
>>> storage._drop_cache_rather_verify
True
>>> storage._blob_cache_size
1048576000
>>> print storage._blob_cache_size_check
104857600
>>> storage.close()
src/ZEO/tests/drop_cache_rather_than_verify.txt deleted 100644 → 0
View file @ 56c7dd30
Avoiding cache verifification
=============================
For large databases it is common to also use very large ZEO cache
files. If a client has beed disconnected for too long, cache verification
might be necessary, but cache verification can be very hard on the
storage server.
When verification is needed, a ZEO.interfaces.StaleCache event is
published. Applications may handle this event to perform actions such
as exiting the process to avoid a cold restart.
ClientStorage provides an option to drop it's cache rather than doing
verification. When this option is used, and verification would be
necessary, after publishing the event, ClientStorage:
- Invalidates all object caches
- Drops or clears it's client cache. (The end result is that the cache
is working but empty.)
- Logs a CRITICAL message.
Here's an example that shows that this is actually what happens.
Start a server, create a cient to it and commit some data
>>> addr, admin = start_server(keep=1)
>>> import ZEO, transaction
>>> db = ZEO.DB(addr, drop_cache_rather_verify=True, client='cache',
... name='test')
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root()[1] = conn.root().__class__()
>>> conn.root()[1].x = 1
>>> transaction.commit()
>>> len(db.storage._cache)
3
Now, we'll stop the server and restart with a different address:
>>> stop_server(admin)
>>> addr2, admin = start_server(keep=1)
And create another client and write some data to it:
>>> db2 = ZEO.DB(addr2)
>>> wait_connected(db2.storage)
>>> conn2 = db2.open()
>>> for i in range(5):
... conn2.root()[1].x += 1
... transaction.commit()
>>> db2.close()
>>> stop_server(admin)
Now, we'll restart the server. Before we do that, we'll capture
logging and event data:
>>> import logging, zope.testing.loggingsupport, ZODB.event
>>> handler = zope.testing.loggingsupport.InstalledHandler(
... 'ZEO.ClientStorage', level=logging.ERROR)
>>> events = []
>>> def event_handler(e):
... events.append((
... len(e.storage._cache), str(handler), e.__class__.__name__))
>>> old_notify = ZODB.event.notify
>>> ZODB.event.notify = event_handler
Note that the event handler is saving away the length of the cache and
the state of the log handler. We'll use this to show that the event
is generated before the cache is dropped or the message is logged.
Now, we'll restart the server on the original address:
>>> _, admin = start_server(zeo_conf=dict(invalidation_queue_size=1),
... addr=addr, keep=1)
>>> wait_connected(db.storage)
Now, let's verify our assertions above:
- Publishes a stale-cache event.
>>> for e in events:
... print e
(3, '', 'StaleCache')
Note that the length of the cache when the event handler was
called waa non-zero. This is because the cache wasn't cleared
yet. Similarly, the dropping-cache message hasn't been logged
yet.
>>> del events[:]
- Drops or clears it's client cache. (The end result is that the cache
is working but empty.)
>>> len(db.storage._cache)
0
- Invalidates all object caches
>>> transaction.abort()
>>> conn.root()._p_changed
- Logs a CRITICAL message.
>>> print handler
ZEO.ClientStorage CRITICAL
test dropping stale cache
>>> handler.clear()
If we access the root object, it'll be loaded from the server:
>>> conn.root()[1].x
6
>>> len(db.storage._cache)
2
Similarly, if we simply disconnect the client, and write data from
another client:
>>> db.close()
>>> db2 = ZEO.DB(addr)
>>> wait_connected(db2.storage)
>>> conn2 = db2.open()
>>> for i in range(5):
... conn2.root()[1].x += 1
... transaction.commit()
>>> db2.close()
>>> db = ZEO.DB(addr, drop_cache_rather_verify=True, client='cache',
... name='test')
>>> wait_connected(db.storage)
- Drops or clears it's client cache. (The end result is that the cache
is working but empty.)
>>> len(db.storage._cache)
1
(When a database is created, it checks to make sure the root object is
in the database, which is why we get 1, rather than 0 objects in the cache.)
- Publishes a stake-cache event.
>>> for e in events:
... print e
(2, '', 'StaleCache')
>>> del events[:]
- Logs a CRITICAL message.
>>> print handler
ZEO.ClientStorage CRITICAL
test dropping stale cache
>>> handler.clear()
If we access the root object, it'll be loaded from the server:
>>> conn = db.open()
>>> conn.root()[1].x
11
Finally, let's look at what happens without the
drop_cache_rather_verify option:
>>> db.close()
>>> db = ZEO.DB(addr, client='cache')
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root()[1].x
11
>>> conn.root()[2] = conn.root().__class__()
>>> transaction.commit()
>>> len(db.storage._cache)
4
>>> stop_server(admin)
>>> addr2, admin = start_server(keep=1)
>>> db2 = ZEO.DB(addr2)
>>> wait_connected(db2.storage)
>>> conn2 = db2.open()
>>> for i in range(5):
... conn2.root()[1].x += 1
... transaction.commit()
>>> db2.close()
>>> stop_server(admin)
>>> _, admin = start_server(zeo_conf=dict(invalidation_queue_size=1),
... addr=addr)
>>> wait_connected(db.storage)
>>> for e in events:
... print e
(4, '', 'StaleCache')
>>> print handler
<BLANKLINE>
>>> len(db.storage._cache)
3
Here we see the cache wasn't dropped, although one of the records was
invalidated during verification.
.. Cleanup
>>> db.close()
>>> handler.uninstall()
>>> ZODB.event.notify = old_notify
src/ZEO/tests/dynamic_server_ports.test deleted 100644 → 0
View file @ 56c7dd30
The storage server can be told to bind to port 0, allowing the OS to
pick a port dynamically. For this to be useful, there needs to be a
way to tell someone. For this reason, the server posts events to
ZODB.notify.
>>> import ZODB.event
>>> old_notify = ZODB.event.notify
>>> last_event = None
>>> def notify(event):
... global last_event
... last_event = event
>>> ZODB.event.notify = notify
Now, let's start a server and verify that we get a serving event:
>>> import ZEO.StorageServer, ZODB.MappingStorage
>>> server = ZEO.StorageServer.StorageServer(
... ('127.0.0.1', 0), {'1': ZODB.MappingStorage.MappingStorage()})
>>> isinstance(last_event, ZEO.StorageServer.Serving)
True
>>> last_event.server is server
True
>>> last_event.address[0], last_event.address[1] > 0
('127.0.0.1', True)
If the host part pf the address passed to the constructor is not an
empty string. then the server addr attribute is the same as the
address attribute of the event:
>>> server.addr == last_event.address
True
Let's run the server in a thread, to make sure we can connect.
>>> server.start_thread()
>>> client = ZEO.client(last_event.address)
>>> client.is_connected()
True
If we close the server, we'll get a closed event:
>>> server.close()
>>> isinstance(last_event, ZEO.StorageServer.Closed)
True
>>> last_event.server is server
True
>>> wait_until(lambda : not client.is_connected(test=True))
>>> client.close()
If we pass an empty string as the host part of the server address, we
can't really assign a single address, so the server addr attribute is
left alone:
>>> server = ZEO.StorageServer.StorageServer(
... ('', 0), {'1': ZODB.MappingStorage.MappingStorage()})
>>> isinstance(last_event, ZEO.StorageServer.Serving)
True
>>> last_event.server is server
True
>>> last_event.address[1] > 0
True
If the host part pf the address passed to the constructor is not an
empty string. then the server addr attribute is the same as the
address attribute of the event:
>>> server.addr
('', 0)
>>> server.close()
The runzeo module provides some process support, including getting the
server configuration via a ZConfig configuration file. To spell a
dynamic port using ZConfig, you'd use a hostname by itself. In this
case, ZConfig passes None as the port.
>>> import ZEO.runzeo
>>> open('conf', 'w').write("""
... <zeo>
... address 127.0.0.1
... </zeo>
... <mappingstorage>
... </mappingstorage>
... """)
>>> options = ZEO.runzeo.ZEOOptions()
>>> options.realize('-C conf'.split())
>>> options.address
('127.0.0.1', None)
>>> rs = ZEO.runzeo.ZEOServer(options)
>>> rs.check_socket()
>>> options.address
('127.0.0.1', 0)
.. cleanup
>>> ZODB.event.notify = old_notify
src/ZEO/tests/forker.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Library for forking storage server and connecting client storage"""
import os
import random
import sys
import time
import errno
import socket
import subprocess
import logging
import StringIO
import tempfile
import logging
import ZODB.tests.util
import zope.testing.setupstack
logger = logging.getLogger('ZEO.tests.forker')
class ZEOConfig:
"""Class to generate ZEO configuration file. """
def __init__(self, addr):
if isinstance(addr, str):
self.logpath = addr+'.log'
else:
self.logpath = 'server-%s.log' % addr[1]
addr = '%s:%s' % addr
self.address = addr
self.read_only = None
self.invalidation_queue_size = None
self.invalidation_age = None
self.monitor_address = None
self.transaction_timeout = None
self.authentication_protocol = None
self.authentication_database = None
self.authentication_realm = None
self.loglevel = 'INFO'
def dump(self, f):
print >> f, "<zeo>"
print >> f, "address " + self.address
if self.read_only is not None:
print >> f, "read-only", self.read_only and "true" or "false"
if self.invalidation_queue_size is not None:
print >> f, "invalidation-queue-size", self.invalidation_queue_size
if self.invalidation_age is not None:
print >> f, "invalidation-age", self.invalidation_age
if self.monitor_address is not None:
print >> f, "monitor-address %s:%s" % self.monitor_address
if self.transaction_timeout is not None:
print >> f, "transaction-timeout", self.transaction_timeout
if self.authentication_protocol is not None:
print >> f, "authentication-protocol", self.authentication_protocol
if self.authentication_database is not None:
print >> f, "authentication-database", self.authentication_database
if self.authentication_realm is not None:
print >> f, "authentication-realm", self.authentication_realm
print >> f, "</zeo>"
print >> f, """
<eventlog>
level %s
<logfile>
path %s
</logfile>
</eventlog>
""" % (self.loglevel, self.logpath)
def __str__(self):
f = StringIO.StringIO()
self.dump(f)
return f.getvalue()
def encode_format(fmt):
# The list of replacements mirrors
# ZConfig.components.logger.handlers._control_char_rewrites
for xform in (("\n", r"\n"), ("\t", r"\t"), ("\b", r"\b"),
("\f", r"\f"), ("\r", r"\r")):
fmt = fmt.replace(*xform)
return fmt
def start_zeo_server(storage_conf=None, zeo_conf=None, port=None, keep=False,
path='Data.fs', protocol=None, blob_dir=None,
suicide=True, debug=False):
"""Start a ZEO server in a separate process.
Takes two positional arguments a string containing the storage conf
and a ZEOConfig object.
Returns the ZEO address, the test server address, the pid, and the path
to the config file.
"""
if not storage_conf:
storage_conf = '<filestorage>\npath %s\n</filestorage>' % path
if blob_dir:
storage_conf = '<blobstorage>\nblob-dir %s\n%s\n</blobstorage>' % (
blob_dir, storage_conf)
if port is None:
raise AssertionError("The port wasn't specified")
if isinstance(port, int):
addr = 'localhost', port
adminaddr = 'localhost', port+1
else:
addr = port
adminaddr = port+'-test'
if zeo_conf is None or isinstance(zeo_conf, dict):
z = ZEOConfig(addr)
if zeo_conf:
z.__dict__.update(zeo_conf)
zeo_conf = z
# Store the config info in a temp file.
tmpfile = tempfile.mktemp(".conf", dir=os.getcwd())
fp = open(tmpfile, 'w')
zeo_conf.dump(fp)
fp.write(storage_conf)
fp.close()
# Find the zeoserver script
import ZEO.tests.zeoserver
script = ZEO.tests.zeoserver.__file__
if script.endswith('.pyc'):
script = script[:-1]
# Create a list of arguments, which we'll tuplify below
qa = _quote_arg
args = [qa(sys.executable), qa(script), '-C', qa(tmpfile)]
if keep:
args.append("-k")
if debug:
args.append("-d")
if not suicide:
args.append("-S")
if protocol:
args.extend(["-v", protocol])
d = os.environ.copy()
d['PYTHONPATH'] = os.pathsep.join(sys.path)
if sys.platform.startswith('win'):
pid = os.spawnve(os.P_NOWAIT, sys.executable, tuple(args), d)
else:
pid = subprocess.Popen(args, env=d, close_fds=True).pid
# We need to wait until the server starts, but not forever.
# 30 seconds is a somewhat arbitrary upper bound. A BDBStorage
# takes a long time to open -- more than 10 seconds on occasion.
for i in range(300):
time.sleep(0.1)
try:
if isinstance(adminaddr, str) and not os.path.exists(adminaddr):
continue
logger.debug('connect %s', i)
if isinstance(adminaddr, str):
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(adminaddr)
ack = s.recv(1024)
s.close()
logging.debug('acked: %s' % ack)
break
except socket.error, e:
if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
raise
s.close()
else:
logging.debug('boo hoo')
raise
return addr, adminaddr, pid, tmpfile
if sys.platform[:3].lower() == "win":
def _quote_arg(s):
return '"%s"' % s
else:
def _quote_arg(s):
return s
def shutdown_zeo_server(adminaddr):
# Do this in a loop to guard against the possibility that the
# client failed to connect to the adminaddr earlier. That really
# only requires two iterations, but do a third for pure
# superstition.
for i in range(3):
if isinstance(adminaddr, str):
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(.3)
try:
s.connect(adminaddr)
except socket.timeout:
# On FreeBSD 5.3 the connection just timed out
if i > 0:
break
raise
except socket.error, e:
if (e[0] == errno.ECONNREFUSED
or
# MAC OS X uses EINVAL when connecting to a port
# that isn't being listened on.
(sys.platform == 'darwin' and e[0] == errno.EINVAL)
) and i > 0:
break
raise
try:
ack = s.recv(1024)
except socket.error, e:
ack = 'no ack received'
logger.debug('shutdown_zeo_server(): acked: %s' % ack)
s.close()
def get_port(test=None):
"""Return a port that is not in use.
Checks if a port is in use by trying to connect to it. Assumes it
is not in use if connect raises an exception. We actually look for
2 consective free ports because most of the clients of this
function will use the returned port and the next one.
Raises RuntimeError after 10 tries.
"""
if test is not None:
return get_port2(test)
for i in range(10):
port = random.randrange(20000, 30000)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
try:
s.connect(('localhost', port))
except socket.error:
pass # Perhaps we should check value of error too.
else:
continue
try:
s1.connect(('localhost', port+1))
except socket.error:
pass # Perhaps we should check value of error too.
else:
continue
return port
finally:
s.close()
s1.close()
raise RuntimeError("Can't find port")
def get_port2(test):
for i in range(10):
while 1:
port = random.randrange(20000, 30000)
if port%3 == 0:
break
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind(('localhost', port+2))
except socket.error, e:
if e[0] != errno.EADDRINUSE:
raise
continue
if not (can_connect(port) or can_connect(port+1)):
zope.testing.setupstack.register(test, s.close)
return port
s.close()
raise RuntimeError("Can't find port")
def can_connect(port):
c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
c.connect(('localhost', port))
except socket.error:
return False # Perhaps we should check value of error too.
else:
c.close()
return True
def setUp(test):
ZODB.tests.util.setUp(test)
servers = {}
def start_server(storage_conf=None, zeo_conf=None, port=None, keep=False,
addr=None, path='Data.fs', protocol=None, blob_dir=None,
suicide=True, debug=False):
"""Start a ZEO server.
Return the server and admin addresses.
"""
if port is None:
if addr is None:
port = get_port2(test)
else:
port = addr[1]
elif addr is not None:
raise TypeError("Can't specify port and addr")
addr, adminaddr, pid, config_path = start_zeo_server(
storage_conf, zeo_conf, port, keep, path, protocol, blob_dir,
suicide, debug)
os.remove(config_path)
servers[adminaddr] = pid
return addr, adminaddr
test.globs['start_server'] = start_server
def get_port():
return get_port2(test)
test.globs['get_port'] = get_port
def stop_server(adminaddr):
pid = servers.pop(adminaddr)
shutdown_zeo_server(adminaddr)
os.waitpid(pid, 0)
test.globs['stop_server'] = stop_server
def cleanup_servers():
for adminaddr in list(servers):
stop_server(adminaddr)
zope.testing.setupstack.register(test, cleanup_servers)
test.globs['wait_until'] = wait_until
test.globs['wait_connected'] = wait_connected
test.globs['wait_disconnected'] = wait_disconnected
def wait_until(label=None, func=None, timeout=30, onfail=None):
if label is None:
if func is not None:
label = func.__name__
elif not isinstance(label, basestring) and func is None:
func = label
label = func.__name__
if func is None:
def wait_decorator(f):
wait_until(label, f, timeout, onfail)
return wait_decorator
giveup = time.time() + timeout
while not func():
if time.time() > giveup:
if onfail is None:
raise AssertionError("Timed out waiting for: ", label)
else:
return onfail()
time.sleep(0.01)
def wait_connected(storage):
wait_until("storage is connected", storage.is_connected)
def wait_disconnected(storage):
wait_until("storage is disconnected",
lambda : not storage.is_connected())
src/ZEO/tests/invalidation-age.txt deleted 100644 → 0
View file @ 56c7dd30
Invalidation age
================
When a ZEO client with a non-empty cache connects to the server, it
needs to verify whether the data in its cache is current. It does
this in one of 2 ways:
quick verification
It gets a list of invalidations from the server since the last
transaction the client has seen and applies those to it's disk and
in-memory caches. This is only possible if there haven't been too
many transactions since the client was last connected.
full verification
If quick verification isn't possible, the client iterates through
it's disk cache asking the server to verify whether each current
entry is valid.
Unfortunately, for large caches, full verification is soooooo not
quick that it is impractical. Quick verificatioin is highly
desireable.
To support quick verification, the server keeps a list of recent
invalidations. The size of this list is controlled by the
invalidation_queue_size parameter. If there is a lot of database
activity, the size might need to be quite large to support having
clients be disconnected for more than a few minutes. A very large
invalidation queue size can use a lot of memory.
To suppliment the invalidation queue, you can also specify an
invalidation_age parameter. When a client connects and presents the
last transaction id it has seen, we first check to see if the
invalidation queue has that transaction id. It it does, then we send
all transactions since that id. Otherwise, we check to see if the
difference between storage's last transaction id and the given id is
less than or equal to the invalidation age. If it is, then we iterate
over the storage, starting with the given id, to get the invalidations
since the given id.
NOTE: This assumes that iterating from a point near the "end" of a
database is inexpensive. Don't use this option for a storage for which
that is not the case.
Here's an example. We set up a server, using an
invalidation-queue-size of 5:
>>> addr, admin = start_server(zeo_conf=dict(invalidation_queue_size=5),
... keep=True)
Now, we'll open a client with a persistent cache, set up some data,
and then close client:
>>> import ZEO, transaction
>>> db = ZEO.DB(addr, client='test')
>>> conn = db.open()
>>> for i in range(9):
... conn.root()[i] = conn.root().__class__()
... conn.root()[i].x = 0
>>> transaction.commit()
>>> db.close()
We'll open another client, and commit some transactions:
>>> db = ZEO.DB(addr)
>>> conn = db.open()
>>> import transaction
>>> for i in range(2):
... conn.root()[i].x = 1
... transaction.commit()
>>> db.close()
If we reopen the first client, we'll do quick verification. We'll
turn on logging so we can see this:
>>> import logging, sys
>>> old_logging_level = logging.getLogger().getEffectiveLevel()
>>> logging.getLogger().setLevel(logging.INFO)
>>> handler = logging.StreamHandler(sys.stdout)
>>> logging.getLogger().addHandler(handler)
>>> db = ZEO.DB(addr, client='test') # doctest: +ELLIPSIS
('localhost', ...
('localhost', ...) Recovering 2 invalidations
>>> logging.getLogger().removeHandler(handler)
>>> [v.x for v in db.open().root().values()]
[1, 1, 0, 0, 0, 0, 0, 0, 0]
Now, if we disconnect and commit more than 5 transactions, we'll see
that verification is necessary:
>>> db.close()
>>> db = ZEO.DB(addr)
>>> conn = db.open()
>>> import transaction
>>> for i in range(9):
... conn.root()[i].x = 2
... transaction.commit()
>>> db.close()
>>> logging.getLogger().addHandler(handler)
>>> db = ZEO.DB(addr, client='test') # doctest: +ELLIPSIS
('localhost', ...
('localhost', ...) Verifying cache
('localhost', ...) endVerify finishing
('localhost', ...) endVerify finished
>>> logging.getLogger().removeHandler(handler)
>>> [v.x for v in db.open().root().values()]
[2, 2, 2, 2, 2, 2, 2, 2, 2]
>>> db.close()
But if we restart the server with invalidation-age set, we can
do quick verification:
>>> stop_server(admin)
>>> addr, admin = start_server(zeo_conf=dict(invalidation_queue_size=5,
... invalidation_age=100))
>>> db = ZEO.DB(addr)
>>> conn = db.open()
>>> import transaction
>>> for i in range(9):
... conn.root()[i].x = 3
... transaction.commit()
>>> db.close()
>>> logging.getLogger().addHandler(handler)
>>> db = ZEO.DB(addr, client='test') # doctest: +ELLIPSIS
('localhost', ...
('localhost', ...) Recovering 9 invalidations
>>> logging.getLogger().removeHandler(handler)
>>> [v.x for v in db.open().root().values()]
[3, 3, 3, 3, 3, 3, 3, 3, 3]
>>> db.close()
>>> logging.getLogger().setLevel(old_logging_level)
src/ZEO/tests/new_addr.test deleted 100644 → 0
View file @ 56c7dd30
You can change the address(es) of a client storaage.
We'll start by setting up a server and connecting to it:
>>> import ZEO, ZEO.StorageServer, ZODB.FileStorage, transaction
>>> server = ZEO.StorageServer.StorageServer(
... ('127.0.0.1', 0), {'1': ZODB.FileStorage.FileStorage('t.fs')})
>>> server.start_thread()
>>> conn = ZEO.connection(server.addr)
>>> client = conn.db().storage
>>> client.is_connected()
True
>>> conn.root()
{}
>>> conn.root.x = 1
>>> transaction.commit()
Now we'll close the server:
>>> server.close()
And wait for the connectin to notice it's disconnected:
>>> wait_until(lambda : not client.is_connected())
Now, we'll restart the server and update the connection:
>>> server = ZEO.StorageServer.StorageServer(
... ('127.0.0.1', 0), {'1': ZODB.FileStorage.FileStorage('t.fs')})
>>> server.start_thread()
>>> client.new_addr(server.addr)
Update with another client:
>>> conn2 = ZEO.connection(server.addr)
>>> conn2.root.x += 1
>>> transaction.commit()
Wait for connect:
>>> wait_until(lambda : client.is_connected())
>>> _ = transaction.begin()
>>> conn.root()
{'x': 2}
.. cleanup
>>> conn.close()
>>> conn2.close()
>>> server.close()
src/ZEO/tests/protocols.test deleted 100644 → 0
View file @ 56c7dd30
Test that multiple protocols are supported
==========================================
A full test of all protocols isn't practical. But we'll do a limited
test that at least the current and previous protocols are supported in
both directions.
Let's start a Z308 server
>>> storage_conf = '''
... <blobstorage>
... blob-dir server-blobs
... <filestorage>
... path Data.fs
... </filestorage>
... </blobstorage>
... '''
>>> addr, admin = start_server(
... storage_conf, dict(invalidation_queue_size=5), protocol='Z308')
A current client should be able to connect to a old server:
>>> import ZEO, ZODB.blob, transaction
>>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
>>> wait_connected(db.storage)
>>> db.storage._connection.peer_protocol_version
'Z308'
>>> conn = db.open()
>>> conn.root().x = 0
>>> transaction.commit()
>>> len(db.history(conn.root()._p_oid, 99))
2
>>> conn.root()['blob1'] = ZODB.blob.Blob()
>>> conn.root()['blob1'].open('w').write('blob data 1')
>>> transaction.commit()
>>> db2 = ZEO.DB(addr, blob_dir='server-blobs', shared_blob_dir=True)
>>> wait_connected(db2.storage)
>>> conn2 = db2.open()
>>> for i in range(5):
... conn2.root().x += 1
... transaction.commit()
>>> conn2.root()['blob2'] = ZODB.blob.Blob()
>>> conn2.root()['blob2'].open('w').write('blob data 2')
>>> transaction.commit()
>>> @wait_until("Get the new data")
... def f():
... conn.sync()
... return conn.root().x == 5
>>> db.close()
>>> for i in range(2):
... conn2.root().x += 1
... transaction.commit()
>>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root().x
7
>>> db.close()
>>> for i in range(10):
... conn2.root().x += 1
... transaction.commit()
>>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root().x
17
>>> conn.root()['blob1'].open().read()
'blob data 1'
>>> conn.root()['blob2'].open().read()
'blob data 2'
Note that when taking to a 3.8 server, iteration won't work:
>>> db.storage.iterator()
Traceback (most recent call last):
...
NotImplementedError
>>> db2.close()
>>> db.close()
>>> stop_server(admin)
>>> import os, zope.testing.setupstack
>>> os.remove('client-1.zec')
>>> zope.testing.setupstack.rmtree('blobs')
>>> zope.testing.setupstack.rmtree('server-blobs')
And the other way around:
>>> addr, _ = start_server(storage_conf, dict(invalidation_queue_size=5))
Note that we'll have to pull some hijinks:
>>> import ZEO.zrpc.connection
>>> old_current_protocol = ZEO.zrpc.connection.Connection.current_protocol
>>> ZEO.zrpc.connection.Connection.current_protocol = 'Z308'
>>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
>>> db.storage._connection.peer_protocol_version
'Z308'
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root().x = 0
>>> transaction.commit()
>>> len(db.history(conn.root()._p_oid, 99))
2
>>> conn.root()['blob1'] = ZODB.blob.Blob()
>>> conn.root()['blob1'].open('w').write('blob data 1')
>>> transaction.commit()
>>> db2 = ZEO.DB(addr, blob_dir='server-blobs', shared_blob_dir=True)
>>> wait_connected(db2.storage)
>>> conn2 = db2.open()
>>> for i in range(5):
... conn2.root().x += 1
... transaction.commit()
>>> conn2.root()['blob2'] = ZODB.blob.Blob()
>>> conn2.root()['blob2'].open('w').write('blob data 2')
>>> transaction.commit()
>>> @wait_until()
... def x_to_be_5():
... conn.sync()
... return conn.root().x == 5
>>> db.close()
>>> for i in range(2):
... conn2.root().x += 1
... transaction.commit()
>>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root().x
7
>>> db.close()
>>> for i in range(10):
... conn2.root().x += 1
... transaction.commit()
>>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root().x
17
>>> conn.root()['blob1'].open().read()
'blob data 1'
>>> conn.root()['blob2'].open().read()
'blob data 2'
Make some old protocol calls:
>>> db.storage._server.rpc.call('getSerial', conn.root()._p_oid
... ) == conn.root()._p_serial
True
>>> p, s, v, x, y = db.storage._server.rpc.call('zeoLoad',
... conn.root()._p_oid)
>>> (v, x, y) == ('', None, None)
True
>>> db.storage.load(conn.root()._p_oid) == (p, s)
True
>>> db2.close()
>>> db.close()
Undo the hijinks:
>>> ZEO.zrpc.connection.Connection.current_protocol = old_current_protocol
src/ZEO/tests/registerDB.test deleted 100644 → 0
View file @ 56c7dd30
Storage Servers should call registerDB on storages to propigate invalidations
=============================================================================
Storages servers propagate invalidations from their storages. Among
other things, this allows client storages to be used in storage
servers, allowing storage-server fan out, spreading read load over
multiple storage servers.
We'll create a Faux storage that has a registerDB method.
>>> class FauxStorage:
... invalidations = [('trans0', ['ob0']),
... ('trans1', ['ob0', 'ob1']),
... ]
... def registerDB(self, db):
... self.db = db
... def isReadOnly(self):
... return False
... def getName(self):
... return 'faux'
... def lastTransaction(self):
... return self.invq[0][0]
... def lastInvalidations(self, size):
... return list(self.invalidations)
We dont' want the storage server to try to bind to a socket. We'll
subclass it and give it a do-nothing dispatcher "class":
>>> import ZEO.StorageServer
>>> class StorageServer(ZEO.StorageServer.StorageServer):
... class DispatcherClass:
... __init__ = lambda *a, **kw: None
... class socket:
... getsockname = staticmethod(lambda : 'socket')
We'll create a storage instance and a storage server using it:
>>> storage = FauxStorage()
>>> server = StorageServer('addr', dict(t=storage))
Our storage now has a db attribute that provides IStorageDB. It's
references method is just the referencesf function from ZODB.Serialize
>>> import ZODB.serialize
>>> storage.db.references is ZODB.serialize.referencesf
True
To see the effects of the invalidation messages, we'll create a client
stub that implements the client invalidation calls:
>>> class Client:
... def __init__(self, name):
... self.name = name
... def invalidateTransaction(self, tid, invalidated):
... print 'invalidateTransaction', tid, self.name
... print invalidated
>>> class Connection:
... def __init__(self, mgr, obj):
... self.mgr = mgr
... self.obj = obj
... def should_close(self):
... print 'closed', self.obj.name
... self.mgr.close_conn(self)
... def poll(self):
... pass
...
... @property
... def trigger(self):
... return self
...
... def pull_trigger(self):
... pass
>>> class ZEOStorage:
... def __init__(self, server, name):
... self.name = name
... self.connection = Connection(server, self)
... self.client = Client(name)
Now, we'll register the client with the storage server:
>>> _ = server.register_connection('t', ZEOStorage(server, 1))
>>> _ = server.register_connection('t', ZEOStorage(server, 2))
Now, if we call invalidate, we'll see it propigate to the client:
>>> storage.db.invalidate('trans2', ['ob1', 'ob2'])
invalidateTransaction trans2 1
['ob1', 'ob2']
invalidateTransaction trans2 2
['ob1', 'ob2']
>>> storage.db.invalidate('trans3', ['ob1', 'ob2'])
invalidateTransaction trans3 1
['ob1', 'ob2']
invalidateTransaction trans3 2
['ob1', 'ob2']
The storage servers queue will reflect the invalidations:
>>> for tid, invalidated in server.invq['t']:
... print repr(tid), invalidated
'trans3' ['ob1', 'ob2']
'trans2' ['ob1', 'ob2']
'trans1' ['ob0', 'ob1']
'trans0' ['ob0']
If we call invalidateCache, the storage server will close each of it's
connections:
>>> storage.db.invalidateCache()
closed 1
closed 2
The connections will then reopen and revalidate their caches.
The servers's invalidation queue will get reset
>>> for tid, invalidated in server.invq['t']:
... print repr(tid), invalidated
'trans1' ['ob0', 'ob1']
'trans0' ['ob0']
src/ZEO/tests/servertesting.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# Testing the current ZEO implementation is rather hard due to the
# architecture, which mixes concerns, especially between application
# and networking. Still, it's not as bad as it could be.
# The 2 most important classes in the architecture are ZEOStorage and
# StorageServer. A ZEOStorage is created for each client connection.
# The StorageServer maintains data shared or needed for coordination
# among clients.
# The other important part of the architecture is connections.
# Connections are used by ZEOStorages to send messages or return data
# to clients.
# Here, we'll try to provide some testing infrastructure to isolate
# servers from the network.
import ZEO.StorageServer
import ZEO.zrpc.connection
import ZEO.zrpc.error
import ZODB.MappingStorage
class StorageServer(ZEO.StorageServer.StorageServer):
def __init__(self, addr='test_addr', storages=None, **kw):
if storages is None:
storages = {'1': ZODB.MappingStorage.MappingStorage()}
ZEO.StorageServer.StorageServer.__init__(self, addr, storages, **kw)
class DispatcherClass:
__init__ = lambda *a, **kw: None
class socket:
getsockname = staticmethod(lambda : 'socket')
class Connection:
peer_protocol_version = ZEO.zrpc.connection.Connection.current_protocol
connected = True
def __init__(self, name='connection', addr=''):
name = str(name)
self.name = name
self.addr = addr or 'test-addr-'+name
def close(self):
print self.name, 'closed'
self.connected = False
def poll(self):
if not self.connected:
raise ZEO.zrpc.error.DisconnectedError()
def callAsync(self, meth, *args):
print self.name, 'callAsync', meth, repr(args)
callAsyncNoPoll = callAsync
def call_from_thread(self, *args):
if args:
args[0](*args[1:])
def send_reply(self, *args):
pass
def client(server, name='client', addr=''):
zs = ZEO.StorageServer.ZEOStorage(server)
zs.notifyConnected(Connection(name, addr))
zs.register('1', 0)
return zs
src/ZEO/tests/speed.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
usage="""Test speed of a ZODB storage
Options:
-d file The data file to use as input.
The default is this script.
-n n The number of repititions
-s module A module that defines a 'Storage'
attribute, which is an open storage.
If not specified, a FileStorage will ne
used.
-z Test compressing data
-D Run in debug mode
-L Test loads as well as stores by minimizing
the cache after eachrun
-M Output means only
-C Run with a persistent client cache
-U Run ZEO using a Unix domain socket
-t n Number of concurrent threads to run.
"""
import asyncore
import sys, os, getopt, time
##sys.path.insert(0, os.getcwd())
import persistent
import transaction
import ZODB
from ZODB.POSException import ConflictError
from ZEO.tests import forker
class P(persistent.Persistent):
pass
fs_name = "zeo-speed.fs"
class ZEOExit(asyncore.file_dispatcher):
"""Used to exit ZEO.StorageServer when run is done"""
def writable(self):
return 0
def readable(self):
return 1
def handle_read(self):
buf = self.recv(4)
assert buf == "done"
self.delete_fs()
os._exit(0)
def handle_close(self):
print "Parent process exited unexpectedly"
self.delete_fs()
os._exit(0)
def delete_fs(self):
os.unlink(fs_name)
os.unlink(fs_name + ".lock")
os.unlink(fs_name + ".tmp")
def work(db, results, nrep, compress, data, detailed, minimize, threadno=None):
for j in range(nrep):
for r in 1, 10, 100, 1000:
t = time.time()
conflicts = 0
jar = db.open()
while 1:
try:
transaction.begin()
rt = jar.root()
key = 's%s' % r
if rt.has_key(key):
p = rt[key]
else:
rt[key] = p =P()
for i in range(r):
v = getattr(p, str(i), P())
if compress is not None:
v.d = compress(data)
else:
v.d = data
setattr(p, str(i), v)
transaction.commit()
except ConflictError:
conflicts = conflicts + 1
else:
break
jar.close()
t = time.time() - t
if detailed:
if threadno is None:
print "%s\t%s\t%.4f\t%d" % (j, r, t, conflicts)
else:
print "%s\t%s\t%.4f\t%d\t%d" % (j, r, t, conflicts,
threadno)
results[r].append((t, conflicts))
rt=d=p=v=None # release all references
if minimize:
time.sleep(3)
jar.cacheMinimize()
def main(args):
opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U')
s = None
compress = None
data=sys.argv[0]
nrep=5
minimize=0
detailed=1
cache = None
domain = 'AF_INET'
threads = 1
for o, v in opts:
if o=='-n': nrep = int(v)
elif o=='-d': data = v
elif o=='-s': s = v
elif o=='-z':
import zlib
compress = zlib.compress
elif o=='-L':
minimize=1
elif o=='-M':
detailed=0
elif o=='-D':
global debug
os.environ['STUPID_LOG_FILE']=''
os.environ['STUPID_LOG_SEVERITY']='-999'
debug = 1
elif o == '-C':
cache = 'speed'
elif o == '-U':
domain = 'AF_UNIX'
elif o == '-t':
threads = int(v)
zeo_pipe = None
if s:
s = __import__(s, globals(), globals(), ('__doc__',))
s = s.Storage
server = None
else:
s, server, pid = forker.start_zeo("FileStorage",
(fs_name, 1), domain=domain)
data=open(data).read()
db=ZODB.DB(s,
# disable cache deactivation
cache_size=4000,
cache_deactivate_after=6000,)
print "Beginning work..."
results={1:[], 10:[], 100:[], 1000:[]}
if threads > 1:
import threading
l = []
for i in range(threads):
t = threading.Thread(target=work,
args=(db, results, nrep, compress, data,
detailed, minimize, i))
l.append(t)
for t in l:
t.start()
for t in l:
t.join()
else:
work(db, results, nrep, compress, data, detailed, minimize)
if server is not None:
server.close()
os.waitpid(pid, 0)
if detailed:
print '-'*24
print "num\tmean\tmin\tmax"
for r in 1, 10, 100, 1000:
times = []
for time, conf in results[r]:
times.append(time)
t = mean(times)
print "%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times))
def mean(l):
tot = 0
for v in l:
tot = tot + v
return tot / len(l)
##def compress(s):
## c = zlib.compressobj()
## o = c.compress(s)
## return o + c.flush()
if __name__=='__main__':
main(sys.argv[1:])
src/ZEO/tests/stress.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A ZEO client-server stress test to look for leaks.
The stress test should run in an infinite loop and should involve
multiple connections.
"""
# TODO: This code is currently broken.
import transaction
import ZODB
from ZODB.MappingStorage import MappingStorage
from ZODB.tests import MinPO
from ZEO.ClientStorage import ClientStorage
from ZEO.tests import forker
import os
import random
NUM_TRANSACTIONS_PER_CONN = 10
NUM_CONNECTIONS = 10
NUM_ROOTS = 20
MAX_DEPTH = 20
MIN_OBJSIZE = 128
MAX_OBJSIZE = 2048
def an_object():
"""Return an object suitable for a PersistentMapping key"""
size = random.randrange(MIN_OBJSIZE, MAX_OBJSIZE)
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
buf = f.read(size)
f.close()
return buf
else:
f = open(MinPO.__file__)
l = list(f.read(size))
f.close()
random.shuffle(l)
return "".join(l)
def setup(cn):
"""Initialize the database with some objects"""
root = cn.root()
for i in range(NUM_ROOTS):
prev = an_object()
for j in range(random.randrange(1, MAX_DEPTH)):
o = MinPO.MinPO(prev)
prev = o
root[an_object()] = o
transaction.commit()
cn.close()
def work(cn):
"""Do some work with a transaction"""
cn.sync()
root = cn.root()
obj = random.choice(root.values())
# walk down to the bottom
while not isinstance(obj.value, str):
obj = obj.value
obj.value = an_object()
transaction.commit()
def main():
# Yuck! Need to cleanup forker so that the API is consistent
# across Unix and Windows, at least if that's possible.
if os.name == "nt":
zaddr, tport, pid = forker.start_zeo_server('MappingStorage', ())
def exitserver():
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(tport)
s.close()
else:
zaddr = '', random.randrange(20000, 30000)
pid, exitobj = forker.start_zeo_server(MappingStorage(), zaddr)
def exitserver():
exitobj.close()
while 1:
pid = start_child(zaddr)
print "started", pid
os.waitpid(pid, 0)
exitserver()
def start_child(zaddr):
pid = os.fork()
if pid != 0:
return pid
try:
_start_child(zaddr)
finally:
os._exit(0)
def _start_child(zaddr):
storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1)
db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS)
setup(db.open())
conns = []
conn_count = 0
for i in range(NUM_CONNECTIONS):
c = db.open()
c.__count = 0
conns.append(c)
conn_count += 1
while conn_count < 25:
c = random.choice(conns)
if c.__count > NUM_TRANSACTIONS_PER_CONN:
conns.remove(c)
c.close()
conn_count += 1
c = db.open()
c.__count = 0
conns.append(c)
else:
c.__count += 1
work(c)
if __name__ == "__main__":
main()
src/ZEO/tests/testAuth.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for AuthZEO."""
import os
import tempfile
import time
import unittest
from ZEO import zeopasswd
from ZEO.Exceptions import ClientDisconnected
from ZEO.tests.ConnectionTests import CommonSetupTearDown
class AuthTest(CommonSetupTearDown):
__super_getServerConfig = CommonSetupTearDown.getServerConfig
__super_setUp = CommonSetupTearDown.setUp
__super_tearDown = CommonSetupTearDown.tearDown
realm = None
def setUp(self):
fd, self.pwfile = tempfile.mkstemp('pwfile')
os.close(fd)
if self.realm:
self.pwdb = self.dbclass(self.pwfile, self.realm)
else:
self.pwdb = self.dbclass(self.pwfile)
self.pwdb.add_user("foo", "bar")
self.pwdb.save()
self._checkZEOpasswd()
self.__super_setUp()
def _checkZEOpasswd(self):
args = ["-f", self.pwfile, "-p", self.protocol]
if self.protocol == "plaintext":
from ZEO.auth.base import Database
zeopasswd.main(args + ["-d", "foo"], Database)
zeopasswd.main(args + ["foo", "bar"], Database)
else:
zeopasswd.main(args + ["-d", "foo"])
zeopasswd.main(args + ["foo", "bar"])
def tearDown(self):
os.remove(self.pwfile)
self.__super_tearDown()
def getConfig(self, path, create, read_only):
return "<mappingstorage 1/>"
def getServerConfig(self, addr, ro_svr):
zconf = self.__super_getServerConfig(addr, ro_svr)
zconf.authentication_protocol = self.protocol
zconf.authentication_database = self.pwfile
zconf.authentication_realm = self.realm
return zconf
def wait(self):
for i in range(25):
time.sleep(0.1)
if self._storage.test_connection:
return
self.fail("Timed out waiting for client to authenticate")
def testOK(self):
# Sleep for 0.2 seconds to give the server some time to start up
# seems to be needed before and after creating the storage
self._storage = self.openClientStorage(wait=0, username="foo",
password="bar", realm=self.realm)
self.wait()
self.assert_(self._storage._connection)
self._storage._connection.poll()
self.assert_(self._storage.is_connected())
# Make a call to make sure the mechanism is working
self._storage.undoInfo()
def testNOK(self):
self._storage = self.openClientStorage(wait=0, username="foo",
password="noogie",
realm=self.realm)
self.wait()
# If the test established a connection, then it failed.
self.failIf(self._storage._connection)
def testUnauthenticatedMessage(self):
# Test that an unauthenticated message is rejected by the server
# if it was sent after the connection was authenticated.
self._storage = self.openClientStorage(wait=0, username="foo",
password="bar", realm=self.realm)
# Sleep for 0.2 seconds to give the server some time to start up
# seems to be needed before and after creating the storage
self.wait()
self._storage.undoInfo()
# Manually clear the state of the hmac connection
self._storage._connection._SizedMessageAsyncConnection__hmac_send = None
# Once the client stops using the hmac, it should be disconnected.
self.assertRaises(ClientDisconnected, self._storage.undoInfo)
class PlainTextAuth(AuthTest):
import ZEO.tests.auth_plaintext
protocol = "plaintext"
database = "authdb.sha"
dbclass = ZEO.tests.auth_plaintext.Database
realm = "Plaintext Realm"
class DigestAuth(AuthTest):
import ZEO.auth.auth_digest
protocol = "digest"
database = "authdb.digest"
dbclass = ZEO.auth.auth_digest.DigestDatabase
realm = "Digest Realm"
test_classes = [PlainTextAuth, DigestAuth]
def test_suite():
suite = unittest.TestSuite()
for klass in test_classes:
sub = unittest.makeSuite(klass)
suite.addTest(sub)
return suite
if __name__ == "__main__":
unittest.main(defaultTest='test_suite')
src/ZEO/tests/testConnection.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test setup for ZEO connection logic.
The actual tests are in ConnectionTests.py; this file provides the
platform-dependent scaffolding.
"""
from __future__ import with_statement
from ZEO.tests import ConnectionTests, InvalidationTests
from zope.testing import setupstack
import os
if os.environ.get('USE_ZOPE_TESTING_DOCTEST'):
from zope.testing import doctest
else:
import doctest
import unittest
import ZEO.tests.forker
import ZEO.tests.testMonitor
import ZEO.zrpc.connection
import ZODB.tests.util
class FileStorageConfig:
def getConfig(self, path, create, read_only):
return """\
<filestorage 1>
path %s
create %s
read-only %s
</filestorage>""" % (path,
create and 'yes' or 'no',
read_only and 'yes' or 'no')
class MappingStorageConfig:
def getConfig(self, path, create, read_only):
return """<mappingstorage 1/>"""
class FileStorageConnectionTests(
FileStorageConfig,
ConnectionTests.ConnectionTests,
InvalidationTests.InvalidationTests
):
"""FileStorage-specific connection tests."""
class FileStorageReconnectionTests(
FileStorageConfig,
ConnectionTests.ReconnectionTests,
):
"""FileStorage-specific re-connection tests."""
# Run this at level 1 because MappingStorage can't do reconnection tests
class FileStorageInvqTests(
FileStorageConfig,
ConnectionTests.InvqTests
):
"""FileStorage-specific invalidation queue tests."""
class FileStorageTimeoutTests(
FileStorageConfig,
ConnectionTests.TimeoutTests
):
pass
class MappingStorageConnectionTests(
MappingStorageConfig,
ConnectionTests.ConnectionTests
):
"""Mapping storage connection tests."""
# The ReconnectionTests can't work with MappingStorage because it's only an
# in-memory storage and has no persistent state.
class MappingStorageTimeoutTests(
MappingStorageConfig,
ConnectionTests.TimeoutTests
):
pass
class MonitorTests(ZEO.tests.testMonitor.MonitorTests):
def check_connection_management(self):
# Open and close a few connections, making sure that
# the resulting number of clients is 0.
s1 = self.openClientStorage()
s2 = self.openClientStorage()
s3 = self.openClientStorage()
stats = self.parse(self.get_monitor_output())[1]
self.assertEqual(stats.clients, 3)
s1.close()
s3.close()
s2.close()
ZEO.tests.forker.wait_until(
"Number of clients shown in monitor drops to 0",
lambda :
self.parse(self.get_monitor_output())[1].clients == 0
)
def check_connection_management_with_old_client(self):
# Check that connection management works even when using an
# older protcool that requires a connection adapter.
test_protocol = "Z303"
current_protocol = ZEO.zrpc.connection.Connection.current_protocol
ZEO.zrpc.connection.Connection.current_protocol = test_protocol
ZEO.zrpc.connection.Connection.servers_we_can_talk_to.append(
test_protocol)
try:
self.check_connection_management()
finally:
ZEO.zrpc.connection.Connection.current_protocol = current_protocol
ZEO.zrpc.connection.Connection.servers_we_can_talk_to.pop()
test_classes = [FileStorageConnectionTests,
FileStorageReconnectionTests,
FileStorageInvqTests,
FileStorageTimeoutTests,
MappingStorageConnectionTests,
MappingStorageTimeoutTests,
MonitorTests,
]
def invalidations_while_connecting():
r"""
As soon as a client registers with a server, it will recieve
invalidations from the server. The client must be careful to queue
these invalidations until it is ready to deal with them. At the time
of the writing of this test, clients weren't careful enough about
queing invalidations. This led to cache corruption in the form of
both low-level file corruption as well as out-of-date records marked
as current.
This tests tries to provoke this bug by:
- starting a server
>>> addr, _ = start_server()
- opening a client to the server that writes some objects, filling
it's cache at the same time,
>>> import ZODB.tests.MinPO, transaction
>>> db = ZEO.DB(addr, client='x')
>>> conn = db.open()
>>> nobs = 1000
>>> for i in range(nobs):
... conn.root()[i] = ZODB.tests.MinPO.MinPO(0)
>>> transaction.commit()
>>> import zope.testing.loggingsupport, logging
>>> handler = zope.testing.loggingsupport.InstalledHandler(
... 'ZEO', level=logging.INFO)
# >>> logging.getLogger('ZEO').debug(
# ... 'Initial tid %r' % conn.root()._p_serial)
- disconnecting the first client (closing it with a persistent cache),
>>> db.close()
- starting a second client that writes objects more or less
constantly,
>>> import random, threading, time
>>> stop = False
>>> db2 = ZEO.DB(addr)
>>> tm = transaction.TransactionManager()
>>> conn2 = db2.open(transaction_manager=tm)
>>> random = random.Random(0)
>>> lock = threading.Lock()
>>> def run():
... while 1:
... i = random.randint(0, nobs-1)
... if stop:
... return
... with lock:
... conn2.root()[i].value += 1
... tm.commit()
... #logging.getLogger('ZEO').debug(
... # 'COMMIT %s %s %r' % (
... # i, conn2.root()[i].value, conn2.root()[i]._p_serial))
... time.sleep(0)
>>> thread = threading.Thread(target=run)
>>> thread.setDaemon(True)
>>> thread.start()
- restarting the first client, and
- testing for cache validity.
>>> bad = False
>>> try:
... for c in range(10):
... time.sleep(.1)
... db = ZODB.DB(ZEO.ClientStorage.ClientStorage(addr, client='x'))
... with lock:
... #logging.getLogger('ZEO').debug('Locked %s' % c)
... @wait_until("connected and we have caught up", timeout=199)
... def _():
... if (db.storage.is_connected()
... and db.storage.lastTransaction()
... == db.storage._server.lastTransaction()
... ):
... #logging.getLogger('ZEO').debug(
... # 'Connected %r' % db.storage.lastTransaction())
... return True
...
... conn = db.open()
... for i in range(1000):
... if conn.root()[i].value != conn2.root()[i].value:
... print 'bad', c, i, conn.root()[i].value,
... print conn2.root()[i].value
... bad = True
... print 'client debug log with lock held'
... while handler.records:
... record = handler.records.pop(0)
... print record.name, record.levelname,
... print handler.format(record)
... if bad:
... print open('server-%s.log' % addr[1]).read()
... #else:
... # logging.getLogger('ZEO').debug('GOOD %s' % c)
... db.close()
... finally:
... stop = True
... thread.join(10)
>>> thread.isAlive()
False
>>> for record in handler.records:
... if record.levelno < logging.ERROR:
... continue
... print record.name, record.levelname
... print handler.format(record)
>>> handler.uninstall()
>>> db.close()
>>> db2.close()
"""
def test_suite():
suite = unittest.TestSuite()
for klass in test_classes:
sub = unittest.makeSuite(klass, 'check')
suite.addTest(sub)
suite.addTest(doctest.DocTestSuite(
setUp=ZEO.tests.forker.setUp, tearDown=setupstack.tearDown,
))
suite.layer = ZODB.tests.util.MininalTestLayer('ZEO Connection Tests')
return suite
src/ZEO/tests/testConversionSupport.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2006 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import doctest
import unittest
class FakeStorageBase:
def __getattr__(self, name):
if name in ('getTid', 'history', 'load', 'loadSerial',
'lastTransaction', 'getSize', 'getName', 'supportsUndo',
'tpc_transaction'):
return lambda *a, **k: None
raise AttributeError(name)
def isReadOnly(self):
return False
def __len__(self):
return 4
class FakeStorage(FakeStorageBase):
def record_iternext(self, next=None):
if next == None:
next = '0'
next = str(int(next) + 1)
oid = next
if next == '4':
next = None
return oid, oid*8, 'data ' + oid, next
class FakeServer:
storages = {
'1': FakeStorage(),
'2': FakeStorageBase(),
}
def register_connection(*args):
return None, None
def test_server_record_iternext():
"""
On the server, record_iternext calls are simply delegated to the
underlying storage.
>>> import ZEO.StorageServer
>>> zeo = ZEO.StorageServer.ZEOStorage(FakeServer(), False)
>>> zeo.register('1', False)
>>> next = None
>>> while 1:
... oid, serial, data, next = zeo.record_iternext(next)
... print oid
... if next is None:
... break
1
2
3
4
The storage info also reflects the fact that record_iternext is supported.
>>> zeo.get_info()['supports_record_iternext']
True
>>> zeo = ZEO.StorageServer.ZEOStorage(FakeServer(), False)
>>> zeo.register('2', False)
>>> zeo.get_info()['supports_record_iternext']
False
"""
def test_client_record_iternext():
"""\
The client simply delegates record_iternext calls to it's server stub.
There's really no decent way to test ZEO without running too much crazy
stuff. I'd rather do a lame test than a really lame test, so here goes.
First, fake out the connection manager so we can make a connection:
>>> import ZEO.ClientStorage
>>> from ZEO.ClientStorage import ClientStorage
>>> oldConnectionManagerClass = ClientStorage.ConnectionManagerClass
>>> class FauxConnectionManagerClass:
... def __init__(*a, **k):
... pass
... def attempt_connect(self):
... return True
>>> ClientStorage.ConnectionManagerClass = FauxConnectionManagerClass
>>> client = ClientStorage('', wait=False)
>>> ClientStorage.ConnectionManagerClass = oldConnectionManagerClass
Now we'll have our way with it's private _server attr:
>>> client._server = FakeStorage()
>>> next = None
>>> while 1:
... oid, serial, data, next = client.record_iternext(next)
... print oid
... if next is None:
... break
1
2
3
4
"""
def test_server_stub_record_iternext():
"""\
The server stub simply delegates record_iternext calls to it's rpc.
There's really no decent way to test ZEO without running to much crazy
stuff. I'd rather do a lame test than a really lame test, so here goes.
>>> class FauxRPC:
... storage = FakeStorage()
... def call(self, meth, *args):
... return getattr(self.storage, meth)(*args)
... peer_protocol_version = 1
>>> import ZEO.ServerStub
>>> stub = ZEO.ServerStub.StorageServer(FauxRPC())
>>> next = None
>>> while 1:
... oid, serial, data, next = stub.record_iternext(next)
... print oid
... if next is None:
... break
1
2
3
4
"""
def history_to_version_compatible_storage():
"""
Some storages work under ZODB <= 3.8 and ZODB >= 3.9.
This means they have a history method that accepts a version parameter:
>>> class VersionCompatibleStorage(FakeStorageBase):
... def history(self,oid,version='',size=1):
... return oid,version,size
A ZEOStorage such as the following should support this type of storage:
>>> class OurFakeServer(FakeServer):
... storages = {'1':VersionCompatibleStorage()}
>>> import ZEO.StorageServer
>>> zeo = ZEO.StorageServer.ZEOStorage(OurFakeServer(), False)
>>> zeo.register('1', False)
The ZEOStorage should sort out the following call such that the storage gets
the correct parameters and so should return the parameters it was called with:
>>> zeo.history('oid',99)
('oid', '', 99)
The same problem occurs when a Z308 client connects to a Z309 server,
but different code is executed:
>>> from ZEO.StorageServer import ZEOStorage308Adapter
>>> zeo = ZEOStorage308Adapter(VersionCompatibleStorage())
The history method should still return the parameters it was called with:
>>> zeo.history('oid','',99)
('oid', '', 99)
"""
def test_suite():
return doctest.DocTestSuite()
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
src/ZEO/tests/testMonitor.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test that the monitor produce sensible results.
$Id$
"""
import socket
import unittest
from ZEO.tests.ConnectionTests import CommonSetupTearDown
from ZEO.monitor import StorageStats
class MonitorTests(CommonSetupTearDown):
monitor = 1
def get_monitor_output(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 42000))
L = []
while 1:
buf = s.recv(8192)
if buf:
L.append(buf)
else:
break
s.close()
return "".join(L)
def parse(self, s):
# Return a list of StorageStats, one for each storage.
lines = s.split("\n")
self.assert_(lines[0].startswith("ZEO monitor server"))
# lines[1] is a date
# Break up rest of lines into sections starting with Storage:
# and ending with a blank line.
sections = []
cur = None
for line in lines[2:]:
if line.startswith("Storage:"):
cur = [line]
elif line:
cur.append(line)
else:
if cur is not None:
sections.append(cur)
cur = None
assert cur is None # bug in the test code if this fails
d = {}
for sect in sections:
hdr = sect[0]
key, value = hdr.split(":")
storage = int(value)
s = d[storage] = StorageStats()
s.parse("\n".join(sect[1:]))
return d
def getConfig(self, path, create, read_only):
return """<mappingstorage 1/>"""
def testMonitor(self):
# Just open a client to know that the server is up and running
# TODO: should put this in setUp.
self.storage = self.openClientStorage()
s = self.get_monitor_output()
self.storage.close()
self.assert_(s.find("monitor") != -1)
d = self.parse(s)
stats = d[1]
self.assertEqual(stats.clients, 1)
self.assertEqual(stats.commits, 0)
def test_suite():
return unittest.makeSuite(MonitorTests)
src/ZEO/tests/testTransactionBuffer.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import random
import unittest
from ZEO.TransactionBuffer import TransactionBuffer
def random_string(size):
"""Return a random string of size size."""
l = [chr(random.randrange(256)) for i in range(size)]
return "".join(l)
def new_store_data():
"""Return arbitrary data to use as argument to store() method."""
return random_string(8), random_string(random.randrange(1000))
def new_invalidate_data():
"""Return arbitrary data to use as argument to invalidate() method."""
return random_string(8)
class TransBufTests(unittest.TestCase):
def checkTypicalUsage(self):
tbuf = TransactionBuffer()
tbuf.store(*new_store_data())
tbuf.invalidate(new_invalidate_data())
for o in tbuf:
pass
def doUpdates(self, tbuf):
data = []
for i in range(10):
d = new_store_data()
tbuf.store(*d)
data.append(d)
d = new_invalidate_data()
tbuf.invalidate(d)
data.append(d)
for i, x in enumerate(tbuf):
if x[1] is None:
# the tbuf add a dummy None to invalidates
x = x[0]
self.assertEqual(x, data[i])
def checkOrderPreserved(self):
tbuf = TransactionBuffer()
self.doUpdates(tbuf)
def checkReusable(self):
tbuf = TransactionBuffer()
self.doUpdates(tbuf)
tbuf.clear()
self.doUpdates(tbuf)
tbuf.clear()
self.doUpdates(tbuf)
def test_suite():
return unittest.makeSuite(TransBufTests, 'check')
src/ZEO/tests/testZEO.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for ZEO based on ZODB.tests."""
from ZEO.ClientStorage import ClientStorage
from ZEO.tests.forker import get_port
from ZEO.tests import forker, Cache, CommitLockTests, ThreadTests
from ZEO.tests import IterationTests
from ZEO.zrpc.error import DisconnectedError
from ZODB.tests import StorageTestBase, BasicStorage, \
TransactionalUndoStorage, \
PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \
MTStorage, ReadOnlyStorage, IteratorStorage, RecoveryStorage
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_unpickle
from ZODB.utils import p64, u64
from zope.testing import renormalizing
import doctest
import logging
import os
import persistent
import re
import shutil
import signal
import stat
import sys
import tempfile
import threading
import time
import transaction
import unittest
import ZEO.ServerStub
import ZEO.StorageServer
import ZEO.tests.ConnectionTests
import ZEO.zrpc.connection
import ZODB
import ZODB.blob
import ZODB.tests.hexstorage
import ZODB.tests.testblob
import ZODB.tests.util
import ZODB.utils
import zope.testing.setupstack
logger = logging.getLogger('ZEO.tests.testZEO')
class DummyDB:
def invalidate(self, *args):
pass
def invalidateCache(*unused):
pass
transform_record_data = untransform_record_data = lambda self, v: v
class CreativeGetState(persistent.Persistent):
def __getstate__(self):
self.name = 'me'
return super(CreativeGetState, self).__getstate__()
class MiscZEOTests:
"""ZEO tests that don't fit in elsewhere."""
def checkCreativeGetState(self):
# This test covers persistent objects that provide their own
# __getstate__ which modifies the state of the object.
# For details see bug #98275
db = ZODB.DB(self._storage)
cn = db.open()
rt = cn.root()
m = CreativeGetState()
m.attr = 'hi'
rt['a'] = m
# This commit used to fail because of the `Mine` object being put back
# into `changed` state although it was already stored causing the ZEO
# cache to bail out.
transaction.commit()
cn.close()
def checkLargeUpdate(self):
obj = MinPO("X" * (10 * 128 * 1024))
self._dostore(data=obj)
def checkZEOInvalidation(self):
addr = self._storage._addr
storage2 = self._wrap_client(
ClientStorage(addr, wait=1, min_disconnect_poll=0.1))
try:
oid = self._storage.new_oid()
ob = MinPO('first')
revid1 = self._dostore(oid, data=ob)
data, serial = storage2.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO('first'))
self.assertEqual(serial, revid1)
revid2 = self._dostore(oid, data=MinPO('second'), revid=revid1)
# Now, storage 2 should eventually get the new data. It
# will take some time, although hopefully not much.
# We'll poll till we get it and whine if we time out:
for n in range(30):
time.sleep(.1)
data, serial = storage2.load(oid, '')
if (serial == revid2 and
zodb_unpickle(data) == MinPO('second')
):
break
else:
raise AssertionError('Invalidation message was not sent!')
finally:
storage2.close()
def checkVolatileCacheWithImmediateLastTransaction(self):
# Earlier, a ClientStorage would not have the last transaction id
# available right after successful connection, this is required now.
addr = self._storage._addr
storage2 = ClientStorage(addr)
self.assert_(storage2.is_connected())
self.assertEquals(ZODB.utils.z64, storage2.lastTransaction())
storage2.close()
self._dostore()
storage3 = ClientStorage(addr)
self.assert_(storage3.is_connected())
self.assertEquals(8, len(storage3.lastTransaction()))
self.assertNotEquals(ZODB.utils.z64, storage3.lastTransaction())
storage3.close()
class ConfigurationTests(unittest.TestCase):
def checkDropCacheRatherVerifyConfiguration(self):
from ZODB.config import storageFromString
# the default is to do verification and not drop the cache
cs = storageFromString('''
<zeoclient>
server localhost:9090
wait false
</zeoclient>
''')
self.assertEqual(cs._drop_cache_rather_verify, False)
cs.close()
# now for dropping
cs = storageFromString('''
<zeoclient>
server localhost:9090
wait false
drop-cache-rather-verify true
</zeoclient>
''')
self.assertEqual(cs._drop_cache_rather_verify, True)
cs.close()
class GenericTests(
# Base class for all ZODB tests
StorageTestBase.StorageTestBase,
# ZODB test mixin classes (in the same order as imported)
BasicStorage.BasicStorage,
PackableStorage.PackableStorage,
Synchronization.SynchronizedStorage,
MTStorage.MTStorage,
ReadOnlyStorage.ReadOnlyStorage,
# ZEO test mixin classes (in the same order as imported)
CommitLockTests.CommitLockVoteTests,
ThreadTests.ThreadTests,
# Locally defined (see above)
MiscZEOTests,
):
"""Combine tests from various origins in one class."""
shared_blob_dir = False
blob_cache_dir = None
def setUp(self):
StorageTestBase.StorageTestBase.setUp(self)
logger.info("setUp() %s", self.id())
port = get_port(self)
zconf = forker.ZEOConfig(('', port))
zport, adminaddr, pid, path = forker.start_zeo_server(self.getConfig(),
zconf, port)
self._pids = [pid]
self._servers = [adminaddr]
self._conf_path = path
if not self.blob_cache_dir:
# This is the blob cache for ClientStorage
self.blob_cache_dir = tempfile.mkdtemp(
'blob_cache',
dir=os.path.abspath(os.getcwd()))
self._storage = self._wrap_client(ClientStorage(
zport, '1', cache_size=20000000,
min_disconnect_poll=0.5, wait=1,
wait_timeout=60, blob_dir=self.blob_cache_dir,
shared_blob_dir=self.shared_blob_dir))
self._storage.registerDB(DummyDB())
def _wrap_client(self, client):
return client
def tearDown(self):
self._storage.close()
for server in self._servers:
forker.shutdown_zeo_server(server)
if hasattr(os, 'waitpid'):
# Not in Windows Python until 2.3
for pid in self._pids:
os.waitpid(pid, 0)
StorageTestBase.StorageTestBase.tearDown(self)
def runTest(self):
try:
super(GenericTests, self).runTest()
except:
self._failed = True
raise
else:
self._failed = False
def open(self, read_only=0):
# Needed to support ReadOnlyStorage tests. Ought to be a
# cleaner way.
addr = self._storage._addr
self._storage.close()
self._storage = ClientStorage(addr, read_only=read_only, wait=1)
def checkWriteMethods(self):
# ReadOnlyStorage defines checkWriteMethods. The decision
# about where to raise the read-only error was changed after
# Zope 2.5 was released. So this test needs to detect Zope
# of the 2.5 vintage and skip the test.
# The __version__ attribute was not present in Zope 2.5.
if hasattr(ZODB, "__version__"):
ReadOnlyStorage.ReadOnlyStorage.checkWriteMethods(self)
def checkSortKey(self):
key = '%s:%s' % (self._storage._storage, self._storage._server_addr)
self.assertEqual(self._storage.sortKey(), key)
def _do_store_in_separate_thread(self, oid, revid, voted):
def do_store():
store = ZEO.ClientStorage.ClientStorage(self._storage._addr)
try:
t = transaction.get()
store.tpc_begin(t)
store.store(oid, revid, 'x', '', t)
store.tpc_vote(t)
store.tpc_finish(t)
except Exception, v:
import traceback
print 'E'*70
print v
traceback.print_exception(*sys.exc_info())
finally:
store.close()
thread = threading.Thread(name='T2', target=do_store)
thread.setDaemon(True)
thread.start()
thread.join(voted and .1 or 9)
return thread
class FullGenericTests(
GenericTests,
Cache.TransUndoStorageWithCache,
ConflictResolution.ConflictResolvingStorage,
ConflictResolution.ConflictResolvingTransUndoStorage,
PackableStorage.PackableUndoStorage,
RevisionStorage.RevisionStorage,
TransactionalUndoStorage.TransactionalUndoStorage,
IteratorStorage.IteratorStorage,
IterationTests.IterationTests,
):
"""Extend GenericTests with tests that MappingStorage can't pass."""
class FileStorageRecoveryTests(StorageTestBase.StorageTestBase,
RecoveryStorage.RecoveryStorage):
def getConfig(self):
return """\
<filestorage 1>
path %s
</filestorage>
""" % tempfile.mktemp(dir='.')
def _new_storage(self):
port = get_port(self)
zconf = forker.ZEOConfig(('', port))
zport, adminaddr, pid, path = forker.start_zeo_server(self.getConfig(),
zconf, port)
self._pids.append(pid)
self._servers.append(adminaddr)
blob_cache_dir = tempfile.mkdtemp(dir='.')
storage = ClientStorage(
zport, '1', cache_size=20000000,
min_disconnect_poll=0.5, wait=1,
wait_timeout=60, blob_dir=blob_cache_dir)
storage.registerDB(DummyDB())
return storage
def setUp(self):
StorageTestBase.StorageTestBase.setUp(self)
self._pids = []
self._servers = []
self._storage = self._new_storage()
self._dst = self._new_storage()
def tearDown(self):
self._storage.close()
self._dst.close()
for server in self._servers:
forker.shutdown_zeo_server(server)
if hasattr(os, 'waitpid'):
# Not in Windows Python until 2.3
for pid in self._pids:
os.waitpid(pid, 0)
StorageTestBase.StorageTestBase.tearDown(self)
def new_dest(self):
return self._new_storage()
class FileStorageTests(FullGenericTests):
"""Test ZEO backed by a FileStorage."""
def getConfig(self):
return """\
<filestorage 1>
path Data.fs
</filestorage>
"""
_expected_interfaces = (
('ZODB.interfaces', 'IStorageRestoreable'),
('ZODB.interfaces', 'IStorageIteration'),
('ZODB.interfaces', 'IStorageUndoable'),
('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
('ZODB.interfaces', 'IExternalGC'),
('ZODB.interfaces', 'IStorage'),
('zope.interface', 'Interface'),
)
def checkInterfaceFromRemoteStorage(self):
# ClientStorage itself doesn't implement IStorageIteration, but the
# FileStorage on the other end does, and thus the ClientStorage
# instance that is connected to it reflects this.
self.failIf(ZODB.interfaces.IStorageIteration.implementedBy(
ZEO.ClientStorage.ClientStorage))
self.failUnless(ZODB.interfaces.IStorageIteration.providedBy(
self._storage))
# This is communicated using ClientStorage's _info object:
self.assertEquals(self._expected_interfaces,
self._storage._info['interfaces']
)
class FileStorageHexTests(FileStorageTests):
_expected_interfaces = (
('ZODB.interfaces', 'IStorageRestoreable'),
('ZODB.interfaces', 'IStorageIteration'),
('ZODB.interfaces', 'IStorageUndoable'),
('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
('ZODB.interfaces', 'IExternalGC'),
('ZODB.interfaces', 'IStorage'),
('ZODB.interfaces', 'IStorageWrapper'),
('zope.interface', 'Interface'),
)
def getConfig(self):
return """\
%import ZODB.tests
<hexstorage>
<filestorage 1>
path Data.fs
</filestorage>
</hexstorage>
"""
class FileStorageClientHexTests(FileStorageHexTests):
def getConfig(self):
return """\
%import ZODB.tests
<serverhexstorage>
<filestorage 1>
path Data.fs
</filestorage>
</serverhexstorage>
"""
def _wrap_client(self, client):
return ZODB.tests.hexstorage.HexStorage(client)
class MappingStorageTests(GenericTests):
"""ZEO backed by a Mapping storage."""
def getConfig(self):
return """<mappingstorage 1/>"""
def checkSimpleIteration(self):
# The test base class IteratorStorage assumes that we keep undo data
# to construct our iterator, which we don't, so we disable this test.
pass
def checkUndoZombie(self):
# The test base class IteratorStorage assumes that we keep undo data
# to construct our iterator, which we don't, so we disable this test.
pass
class DemoStorageTests(
GenericTests,
):
def getConfig(self):
return """
<demostorage 1>
<filestorage 1>
path Data.fs
</filestorage>
</demostorage>
"""
def checkUndoZombie(self):
# The test base class IteratorStorage assumes that we keep undo data
# to construct our iterator, which we don't, so we disable this test.
pass
def checkPackWithMultiDatabaseReferences(self):
pass # DemoStorage pack doesn't do gc
checkPackAllRevisions = checkPackWithMultiDatabaseReferences
class HeartbeatTests(ZEO.tests.ConnectionTests.CommonSetupTearDown):
"""Make sure a heartbeat is being sent and that it does no harm
This is really hard to test properly because we can't see the data
flow between the client and server and we can't really tell what's
going on in the server very well. :(
"""
def setUp(self):
# Crank down the select frequency
self.__old_client_timeout = ZEO.zrpc.client.client_timeout
ZEO.zrpc.client.client_timeout = self.__client_timeout
ZEO.tests.ConnectionTests.CommonSetupTearDown.setUp(self)
__client_timeouts = 0
def __client_timeout(self):
self.__client_timeouts += 1
return .1
def tearDown(self):
ZEO.zrpc.client.client_timeout = self.__old_client_timeout
ZEO.tests.ConnectionTests.CommonSetupTearDown.tearDown(self)
def getConfig(self, path, create, read_only):
return """<mappingstorage 1/>"""
def checkHeartbeatWithServerClose(self):
# This is a minimal test that mainly tests that the heartbeat
# function does no harm.
self._storage = self.openClientStorage()
client_timeouts = self.__client_timeouts
forker.wait_until('got a timeout',
lambda : self.__client_timeouts > client_timeouts
)
self._dostore()
if hasattr(os, 'kill') and hasattr(signal, 'SIGKILL'):
# Kill server violently, in hopes of provoking problem
os.kill(self._pids[0], signal.SIGKILL)
self._servers[0] = None
else:
self.shutdownServer()
forker.wait_until('disconnected',
lambda : not self._storage.is_connected()
)
self._storage.close()
class ZRPCConnectionTests(ZEO.tests.ConnectionTests.CommonSetupTearDown):
def getConfig(self, path, create, read_only):
return """<mappingstorage 1/>"""
def checkCatastrophicClientLoopFailure(self):
# Test what happens when the client loop falls over
self._storage = self.openClientStorage()
class Evil:
def writable(self):
raise SystemError("I'm evil")
import zope.testing.loggingsupport
handler = zope.testing.loggingsupport.InstalledHandler(
'ZEO.zrpc.client')
self._storage._rpc_mgr.map[None] = Evil()
try:
self._storage._rpc_mgr.trigger.pull_trigger()
except DisconnectedError:
pass
forker.wait_until(
'disconnected',
lambda : not self._storage.is_connected()
)
log = str(handler)
handler.uninstall()
self.assert_("ZEO client loop failed" in log)
self.assert_("Couldn't close a dispatcher." in log)
def checkExceptionLogsAtError(self):
# Test the exceptions are logged at error
self._storage = self.openClientStorage()
conn = self._storage._connection
# capture logging
log = []
conn.logger.log = (
lambda l, m, *a, **kw: log.append((l,m % a, kw))
)
# This is a deliberately bogus call to get an exception
# logged
self._storage._connection.handle_request(
'foo', 0, 'history', (1, 2, 3, 4))
# test logging
for level, message, kw in log:
if message.endswith(
') history() raised exception: history() takes at'
' most 3 arguments (5 given)'
):
self.assertEqual(level,logging.ERROR)
self.assertEqual(kw,{'exc_info':True})
break
else:
self.fail("error not in log")
# cleanup
del conn.logger.log
def checkConnectionInvalidationOnReconnect(self):
storage = ClientStorage(self.addr, wait=1, min_disconnect_poll=0.1)
self._storage = storage
# and we'll wait for the storage to be reconnected:
for i in range(100):
if storage.is_connected():
break
time.sleep(0.1)
else:
raise AssertionError("Couldn't connect to server")
class DummyDB:
_invalidatedCache = 0
def invalidateCache(self):
self._invalidatedCache += 1
def invalidate(*a, **k):
pass
db = DummyDB()
storage.registerDB(db)
base = db._invalidatedCache
# Now we'll force a disconnection and reconnection
storage._connection.close()
# and we'll wait for the storage to be reconnected:
for i in range(100):
if storage.is_connected():
break
time.sleep(0.1)
else:
raise AssertionError("Couldn't connect to server")
# Now, the root object in the connection should have been invalidated:
self.assertEqual(db._invalidatedCache, base+1)
class CommonBlobTests:
def getConfig(self):
return """
<blobstorage 1>
blob-dir blobs
<filestorage 2>
path Data.fs
</filestorage>
</blobstorage>
"""
blobdir = 'blobs'
blob_cache_dir = 'blob_cache'
def checkStoreBlob(self):
from ZODB.utils import oid_repr, tid_repr
from ZODB.blob import Blob, BLOB_SUFFIX
from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
handle_serials
import transaction
somedata = 'a' * 10
blob = Blob()
bd_fh = blob.open('w')
bd_fh.write(somedata)
bd_fh.close()
tfname = bd_fh.name
oid = self._storage.new_oid()
data = zodb_pickle(blob)
self.assert_(os.path.exists(tfname))
t = transaction.Transaction()
try:
self._storage.tpc_begin(t)
r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t)
r2 = self._storage.tpc_vote(t)
revid = handle_serials(oid, r1, r2)
self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
self.assert_(not os.path.exists(tfname))
filename = self._storage.fshelper.getBlobFilename(oid, revid)
self.assert_(os.path.exists(filename))
self.assertEqual(somedata, open(filename).read())
def checkStoreBlob_wrong_partition(self):
os_rename = os.rename
try:
def fail(*a):
raise OSError
os.rename = fail
self.checkStoreBlob()
finally:
os.rename = os_rename
def checkLoadBlob(self):
from ZODB.blob import Blob
from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
handle_serials
import transaction
somedata = 'a' * 10
blob = Blob()
bd_fh = blob.open('w')
bd_fh.write(somedata)
bd_fh.close()
tfname = bd_fh.name
oid = self._storage.new_oid()
data = zodb_pickle(blob)
t = transaction.Transaction()
try:
self._storage.tpc_begin(t)
r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t)
r2 = self._storage.tpc_vote(t)
serial = handle_serials(oid, r1, r2)
self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
filename = self._storage.loadBlob(oid, serial)
self.assertEquals(somedata, open(filename, 'rb').read())
self.assert_(not(os.stat(filename).st_mode & stat.S_IWRITE))
self.assert_((os.stat(filename).st_mode & stat.S_IREAD))
def checkTemporaryDirectory(self):
self.assertEquals(os.path.join(self.blob_cache_dir, 'tmp'),
self._storage.temporaryDirectory())
def checkTransactionBufferCleanup(self):
oid = self._storage.new_oid()
open('blob_file', 'w').write('I am a happy blob.')
t = transaction.Transaction()
self._storage.tpc_begin(t)
self._storage.storeBlob(
oid, ZODB.utils.z64, 'foo', 'blob_file', '', t)
self._storage.close()
class BlobAdaptedFileStorageTests(FullGenericTests, CommonBlobTests):
"""ZEO backed by a BlobStorage-adapted FileStorage."""
def checkStoreAndLoadBlob(self):
from ZODB.utils import oid_repr, tid_repr
from ZODB.blob import Blob, BLOB_SUFFIX
from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
handle_serials
import transaction
somedata_path = os.path.join(self.blob_cache_dir, 'somedata')
somedata = open(somedata_path, 'w+b')
for i in range(1000000):
somedata.write("%s\n" % i)
somedata.seek(0)
blob = Blob()
bd_fh = blob.open('w')
ZODB.utils.cp(somedata, bd_fh)
bd_fh.close()
tfname = bd_fh.name
oid = self._storage.new_oid()
data = zodb_pickle(blob)
self.assert_(os.path.exists(tfname))
t = transaction.Transaction()
try:
self._storage.tpc_begin(t)
r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t)
r2 = self._storage.tpc_vote(t)
revid = handle_serials(oid, r1, r2)
self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
# The uncommitted data file should have been removed
self.assert_(not os.path.exists(tfname))
def check_data(path):
self.assert_(os.path.exists(path))
f = open(path, 'rb')
somedata.seek(0)
d1 = d2 = 1
while d1 or d2:
d1 = f.read(8096)
d2 = somedata.read(8096)
self.assertEqual(d1, d2)
# The file should be in the cache ...
filename = self._storage.fshelper.getBlobFilename(oid, revid)
check_data(filename)
# ... and on the server
server_filename = os.path.join(
self.blobdir,
ZODB.blob.BushyLayout().getBlobFilePath(oid, revid),
)
self.assert_(server_filename.startswith(self.blobdir))
check_data(server_filename)
# If we remove it from the cache and call loadBlob, it should
# come back. We can do this in many threads. We'll instrument
# the method that is used to request data from teh server to
# verify that it is only called once.
sendBlob_org = ZEO.ServerStub.StorageServer.sendBlob
calls = []
def sendBlob(self, oid, serial):
calls.append((oid, serial))
sendBlob_org(self, oid, serial)
ZODB.blob.remove_committed(filename)
returns = []
threads = [
threading.Thread(
target=lambda :
returns.append(self._storage.loadBlob(oid, revid))
)
for i in range(10)
]
[thread.start() for thread in threads]
[thread.join() for thread in threads]
[self.assertEqual(r, filename) for r in returns]
check_data(filename)
class BlobWritableCacheTests(FullGenericTests, CommonBlobTests):
blob_cache_dir = 'blobs'
shared_blob_dir = True
class FauxConn:
addr = 'x'
peer_protocol_version = ZEO.zrpc.connection.Connection.current_protocol
class StorageServerClientWrapper:
def __init__(self):
self.serials = []
def serialnos(self, serials):
self.serials.extend(serials)
def info(self, info):
pass
class StorageServerWrapper:
def __init__(self, server, storage_id):
self.storage_id = storage_id
self.server = ZEO.StorageServer.ZEOStorage(server, server.read_only)
self.server.notifyConnected(FauxConn())
self.server.register(storage_id, False)
self.server.client = StorageServerClientWrapper()
def sortKey(self):
return self.storage_id
def __getattr__(self, name):
return getattr(self.server, name)
def registerDB(self, *args):
pass
def supportsUndo(self):
return False
def new_oid(self):
return self.server.new_oids(1)[0]
def tpc_begin(self, transaction):
self.server.tpc_begin(id(transaction), '', '', {}, None, ' ')
def tpc_vote(self, transaction):
vote_result = self.server.vote(id(transaction))
assert vote_result is None
result = self.server.client.serials[:]
del self.server.client.serials[:]
return result
def store(self, oid, serial, data, version_ignored, transaction):
self.server.storea(oid, serial, data, id(transaction))
def send_reply(self, *args): # Masquerade as conn
pass
def tpc_abort(self, transaction):
self.server.tpc_abort(id(transaction))
def tpc_finish(self, transaction, func = lambda: None):
self.server.tpc_finish(id(transaction)).set_sender(0, self)
def multiple_storages_invalidation_queue_is_not_insane():
"""
>>> from ZEO.StorageServer import StorageServer, ZEOStorage
>>> from ZODB.FileStorage import FileStorage
>>> from ZODB.DB import DB
>>> from persistent.mapping import PersistentMapping
>>> from transaction import commit
>>> fs1 = FileStorage('t1.fs')
>>> fs2 = FileStorage('t2.fs')
>>> server = StorageServer(('', get_port()), dict(fs1=fs1, fs2=fs2))
>>> s1 = StorageServerWrapper(server, 'fs1')
>>> s2 = StorageServerWrapper(server, 'fs2')
>>> db1 = DB(s1); conn1 = db1.open()
>>> db2 = DB(s2); conn2 = db2.open()
>>> commit()
>>> o1 = conn1.root()
>>> for i in range(10):
... o1.x = PersistentMapping(); o1 = o1.x
... commit()
>>> last = fs1.lastTransaction()
>>> for i in range(5):
... o1.x = PersistentMapping(); o1 = o1.x
... commit()
>>> o2 = conn2.root()
>>> for i in range(20):
... o2.x = PersistentMapping(); o2 = o2.x
... commit()
>>> trans, oids = s1.getInvalidations(last)
>>> from ZODB.utils import u64
>>> sorted([int(u64(oid)) for oid in oids])
[10, 11, 12, 13, 14]
>>> server.close()
"""
def getInvalidationsAfterServerRestart():
"""
Clients were often forced to verify their caches after a server
restart even if there weren't many transactions between the server
restart and the client connect.
Let's create a file storage and stuff some data into it:
>>> from ZEO.StorageServer import StorageServer, ZEOStorage
>>> from ZODB.FileStorage import FileStorage
>>> from ZODB.DB import DB
>>> from persistent.mapping import PersistentMapping
>>> fs = FileStorage('t.fs')
>>> db = DB(fs)
>>> conn = db.open()
>>> from transaction import commit
>>> last = []
>>> for i in range(100):
... conn.root()[i] = PersistentMapping()
... commit()
... last.append(fs.lastTransaction())
>>> db.close()
Now we'll open a storage server on the data, simulating a restart:
>>> fs = FileStorage('t.fs')
>>> sv = StorageServer(('', get_port()), dict(fs=fs))
>>> s = ZEOStorage(sv, sv.read_only)
>>> s.notifyConnected(FauxConn())
>>> s.register('fs', False)
If we ask for the last transaction, we should get the last transaction
we saved:
>>> s.lastTransaction() == last[-1]
True
If a storage implements the method lastInvalidations, as FileStorage
does, then the stroage server will populate its invalidation data
structure using lastTransactions.
>>> tid, oids = s.getInvalidations(last[-10])
>>> tid == last[-1]
True
>>> from ZODB.utils import u64
>>> sorted([int(u64(oid)) for oid in oids])
[0, 92, 93, 94, 95, 96, 97, 98, 99, 100]
(Note that the fact that we get oids for 92-100 is actually an
artifact of the fact that the FileStorage lastInvalidations method
returns all OIDs written by transactions, even if the OIDs were
created and not modified. FileStorages don't record whether objects
were created rather than modified. Objects that are just created don't
need to be invalidated. This means we'll invalidate objects that
dont' need to be invalidated, however, that's better than verifying
caches.)
>>> sv.close()
>>> fs.close()
If a storage doesn't implement lastInvalidations, a client can still
avoid verifying its cache if it was up to date when the server
restarted. To illustrate this, we'll create a subclass of FileStorage
without this method:
>>> class FS(FileStorage):
... lastInvalidations = property()
>>> fs = FS('t.fs')
>>> sv = StorageServer(('', get_port()), dict(fs=fs))
>>> st = StorageServerWrapper(sv, 'fs')
>>> s = st.server
Now, if we ask for the invalidations since the last committed
transaction, we'll get a result:
>>> tid, oids = s.getInvalidations(last[-1])
>>> tid == last[-1]
True
>>> oids
[]
>>> db = DB(st); conn = db.open()
>>> ob = conn.root()
>>> for i in range(5):
... ob.x = PersistentMapping(); ob = ob.x
... commit()
... last.append(fs.lastTransaction())
>>> ntid, oids = s.getInvalidations(tid)
>>> ntid == last[-1]
True
>>> sorted([int(u64(oid)) for oid in oids])
[0, 101, 102, 103, 104]
>>> fs.close()
"""
def tpc_finish_error():
r"""Server errors in tpc_finish weren't handled properly.
>>> import ZEO.ClientStorage, ZEO.zrpc.connection
>>> class Connection:
... peer_protocol_version = (
... ZEO.zrpc.connection.Connection.current_protocol)
... def __init__(self, client):
... self.client = client
... def get_addr(self):
... return 'server'
... def is_async(self):
... return True
... def register_object(self, ob):
... pass
... def close(self):
... print 'connection closed'
... trigger = property(lambda self: self)
... pull_trigger = lambda self, func, *args: func(*args)
>>> class ConnectionManager:
... def __init__(self, addr, client, tmin, tmax):
... self.client = client
... def connect(self, sync=1):
... self.client.notifyConnected(Connection(self.client))
... def close(self):
... pass
>>> class StorageServer:
... should_fail = True
... def __init__(self, conn):
... self.conn = conn
... self.t = None
... def get_info(self):
... return {}
... def endZeoVerify(self):
... self.conn.client.endVerify()
... def lastTransaction(self):
... return '\0'*8
... def tpc_begin(self, t, *args):
... if self.t is not None:
... raise TypeError('already trans')
... self.t = t
... print 'begin', args
... def vote(self, t):
... if self.t != t:
... raise TypeError('bad trans')
... print 'vote'
... def tpc_finish(self, *args):
... if self.should_fail:
... raise TypeError()
... print 'finish'
... def tpc_abort(self, t):
... if self.t != t:
... raise TypeError('bad trans')
... self.t = None
... print 'abort'
... def iterator_gc(*args):
... pass
>>> class ClientStorage(ZEO.ClientStorage.ClientStorage):
... ConnectionManagerClass = ConnectionManager
... StorageServerStubClass = StorageServer
>>> class Transaction:
... user = 'test'
... description = ''
... _extension = {}
>>> cs = ClientStorage(('', ''))
>>> t1 = Transaction()
>>> cs.tpc_begin(t1)
begin ('test', '', {}, None, ' ')
>>> cs.tpc_vote(t1)
vote
>>> cs.tpc_finish(t1)
Traceback (most recent call last):
...
TypeError
>>> cs.tpc_abort(t1)
abort
>>> t2 = Transaction()
>>> cs.tpc_begin(t2)
begin ('test', '', {}, None, ' ')
>>> cs.tpc_vote(t2)
vote
If client storage has an internal error after the storage finish
succeeeds, it will close the connection, which will force a
restart and reverification.
>>> StorageServer.should_fail = False
>>> cs._update_cache = lambda : None
>>> try: cs.tpc_finish(t2)
... except: pass
... else: print "Should have failed"
finish
connection closed
>>> cs.close()
"""
def client_has_newer_data_than_server():
"""It is bad if a client has newer data than the server.
>>> db = ZODB.DB('Data.fs')
>>> db.close()
>>> shutil.copyfile('Data.fs', 'Data.save')
>>> addr, admin = start_server(keep=1)
>>> db = ZEO.DB(addr, name='client', max_disconnect_poll=.01)
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root().x = 1
>>> transaction.commit()
OK, we've added some data to the storage and the client cache has
the new data. Now, we'll stop the server, put back the old data, and
see what happens. :)
>>> stop_server(admin)
>>> shutil.copyfile('Data.save', 'Data.fs')
>>> import zope.testing.loggingsupport
>>> handler = zope.testing.loggingsupport.InstalledHandler(
... 'ZEO', level=logging.ERROR)
>>> formatter = logging.Formatter('%(name)s %(levelname)s %(message)s')
>>> _, admin = start_server(addr=addr)
>>> for i in range(1000):
... while len(handler.records) < 5:
... time.sleep(.01)
>>> db.close()
>>> for record in handler.records[:5]:
... print formatter.format(record)
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
ZEO.ClientStorage CRITICAL client
Client has seen newer transactions than server!
ZEO.zrpc ERROR (...) CW: error in notifyConnected (('127.0.0.1', ...))
Traceback (most recent call last):
...
ClientStorageError: client Client has seen newer transactions than server!
ZEO.ClientStorage CRITICAL client
Client has seen newer transactions than server!
ZEO.zrpc ERROR (...) CW: error in notifyConnected (('127.0.0.1', ...))
Traceback (most recent call last):
...
ClientStorageError: client Client has seen newer transactions than server!
...
Note that the errors repeat because the client keeps on trying to connect.
>>> handler.uninstall()
>>> stop_server(admin)
"""
def history_over_zeo():
"""
>>> addr, _ = start_server()
>>> db = ZEO.DB(addr)
>>> wait_connected(db.storage)
>>> conn = db.open()
>>> conn.root().x = 0
>>> transaction.commit()
>>> len(db.history(conn.root()._p_oid, 99))
2
>>> db.close()
"""
def dont_log_poskeyerrors_on_server():
"""
>>> addr, admin = start_server()
>>> cs = ClientStorage(addr)
>>> cs.load(ZODB.utils.p64(1))
Traceback (most recent call last):
...
POSKeyError: 0x01
>>> cs.close()
>>> stop_server(admin)
>>> 'POSKeyError' in open('server-%s.log' % addr[1]).read()
False
"""
def open_convenience():
"""Often, we just want to open a single connection.
>>> addr, _ = start_server(path='data.fs')
>>> conn = ZEO.connection(addr)
>>> conn.root()
{}
>>> conn.root()['x'] = 1
>>> transaction.commit()
>>> conn.close()
Let's make sure the database was cloased when we closed the
connection, and that the data is there.
>>> db = ZEO.DB(addr)
>>> conn = db.open()
>>> conn.root()
{'x': 1}
>>> db.close()
"""
def client_asyncore_thread_has_name():
"""
>>> addr, _ = start_server()
>>> db = ZEO.DB(addr)
>>> len([t for t in threading.enumerate()
... if ' zeo client networking thread' in t.getName()])
1
>>> db.close()
"""
def runzeo_without_configfile():
"""
>>> open('runzeo', 'w').write('''
... import sys
... sys.path[:] = %r
... import ZEO.runzeo
... ZEO.runzeo.main(sys.argv[1:])
... ''' % sys.path)
>>> import subprocess, re
>>> print re.sub('\d\d+|[:]', '', subprocess.Popen(
... [sys.executable, 'runzeo', '-a:%s' % get_port(), '-ft', '--test'],
... stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
... ).stdout.read()), # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
------
--T INFO ZEO.runzeo () opening storage '1' using FileStorage
------
--T INFO ZEO.StorageServer StorageServer created RW with storages 1RWt
------
--T INFO ZEO.zrpc () listening on ...
------
--T INFO ZEO.StorageServer closing storage '1'
testing exit immediately
"""
def close_client_storage_w_invalidations():
r"""
Invalidations could cause errors when closing client storages,
>>> addr, _ = start_server()
>>> writing = threading.Event()
>>> def mad_write_thread():
... global writing
... conn = ZEO.connection(addr)
... writing.set()
... while writing.isSet():
... conn.root.x = 1
... transaction.commit()
... conn.close()
>>> thread = threading.Thread(target=mad_write_thread)
>>> thread.setDaemon(True)
>>> thread.start()
>>> _ = writing.wait()
>>> time.sleep(.01)
>>> for i in range(10):
... conn = ZEO.connection(addr)
... _ = conn._storage.load('\0'*8)
... conn.close()
>>> writing.clear()
>>> thread.join(1)
"""
def convenient_to_pass_port_to_client_and_ZEO_dot_client():
"""Jim hates typing
>>> addr, _ = start_server()
>>> client = ZEO.client(addr[1])
>>> client.__name__ == "('127.0.0.1', %s)" % addr[1]
True
>>> client.close()
"""
def test_server_status():
"""
You can get server status using the server_status method.
>>> addr, _ = start_server(zeo_conf=dict(transaction_timeout=1))
>>> db = ZEO.DB(addr)
>>> import pprint
>>> pprint.pprint(db.storage.server_status(), width=1)
{'aborts': 0,
'active_txns': 0,
'commits': 1,
'conflicts': 0,
'conflicts_resolved': 0,
'connections': 1,
'loads': 1,
'lock_time': None,
'start': 'Tue May 4 10:55:20 2010',
'stores': 1,
'timeout-thread-is-alive': True,
'verifying_clients': 0,
'waiting': 0}
>>> db.close()
"""
def client_labels():
"""
When looking at server logs, for servers with lots of clients coming
from the same machine, it can be very difficult to correlate server
log entries with actual clients. It's possible, sort of, but tedious.
You can make this easier by passing a label to the ClientStorage
constructor.
>>> addr, _ = start_server()
>>> db = ZEO.DB(addr, client_label='test-label-1')
>>> db.close()
>>> @wait_until
... def check_for_test_label_1():
... for line in open('server-%s.log' % addr[1]):
... if 'test-label-1' in line:
... print line.split()[1:4]
... return True
['INFO', 'ZEO.StorageServer', '(test-label-1']
You can specify the client label via a configuration file as well:
>>> import ZODB.config
>>> db = ZODB.config.databaseFromString('''
... <zodb>
... <zeoclient>
... server :%s
... client-label test-label-2
... </zeoclient>
... </zodb>
... ''' % addr[1])
>>> db.close()
>>> @wait_until
... def check_for_test_label_2():
... for line in open('server-%s.log' % addr[1]):
... if 'test-label-2' in line:
... print line.split()[1:4]
... return True
['INFO', 'ZEO.StorageServer', '(test-label-2']
"""
def invalidate_client_cache_entry_on_server_commit_error():
"""
When the serials returned during commit includes an error, typically a
conflict error, invalidate the cache entry. This is important when
the cache is messed up.
>>> addr, _ = start_server()
>>> conn1 = ZEO.connection(addr)
>>> conn1.root.x = conn1.root().__class__()
>>> transaction.commit()
>>> conn1.root.x
{}
>>> cs = ZEO.ClientStorage.ClientStorage(addr, client='cache')
>>> conn2 = ZODB.connection(cs)
>>> conn2.root.x
{}
>>> conn2.close()
>>> cs.close()
>>> conn1.root.x['x'] = 1
>>> transaction.commit()
>>> conn1.root.x
{'x': 1}
Now, let's screw up the cache by making it have a last tid that is later than
the root serial.
>>> import ZEO.cache
>>> cache = ZEO.cache.ClientCache('cache-1.zec')
>>> cache.setLastTid(p64(u64(conn1.root.x._p_serial)+1))
>>> cache.close()
We'll also update the server so that it's last tid is newer than the cache's:
>>> conn1.root.y = 1
>>> transaction.commit()
>>> conn1.root.y = 2
>>> transaction.commit()
Now, if we reopen the client storage, we'll get the wrong root:
>>> cs = ZEO.ClientStorage.ClientStorage(addr, client='cache')
>>> conn2 = ZODB.connection(cs)
>>> conn2.root.x
{}
And, we'll get a conflict error if we try to modify it:
>>> conn2.root.x['y'] = 1
>>> transaction.commit() # doctest: +ELLIPSIS
Traceback (most recent call last):
...
ConflictError: ...
But, if we abort, we'll get up to date data and we'll see the changes.
>>> transaction.abort()
>>> conn2.root.x
{'x': 1}
>>> conn2.root.x['y'] = 1
>>> transaction.commit()
>>> sorted(conn2.root.x.items())
[('x', 1), ('y', 1)]
>>> cs.close()
>>> conn1.close()
"""
script_template = """
import sys
sys.path[:] = %(path)r
%(src)s
"""
def generate_script(name, src):
open(name, 'w').write(script_template % dict(
exe=sys.executable,
path=sys.path,
src=src,
))
def runzeo_logrotate_on_sigusr2():
"""
>>> port = get_port()
>>> open('c', 'w').write('''
... <zeo>
... address %s
... </zeo>
... <mappingstorage>
... </mappingstorage>
... <eventlog>
... <logfile>
... path l
... </logfile>
... </eventlog>
... ''' % port)
>>> generate_script('s', '''
... import ZEO.runzeo
... ZEO.runzeo.main()
... ''')
>>> import subprocess, signal
>>> p = subprocess.Popen([sys.executable, 's', '-Cc'], close_fds=True)
>>> wait_until('started',
... lambda : os.path.exists('l') and ('listening on' in open('l').read())
... )
>>> oldlog = open('l').read()
>>> os.rename('l', 'o')
>>> os.kill(p.pid, signal.SIGUSR2)
>>> wait_until('new file', lambda : os.path.exists('l'))
>>> s = ClientStorage(port)
>>> s.close()
>>> wait_until('See logging', lambda : ('Log files ' in open('l').read()))
>>> open('o').read() == oldlog # No new data in old log
True
# Cleanup:
>>> os.kill(p.pid, signal.SIGKILL)
>>> _ = p.wait()
"""
def unix_domain_sockets():
"""Make sure unix domain sockets work
>>> addr, _ = start_server(port='./sock')
>>> c = ZEO.connection(addr)
>>> c.root.x = 1
>>> transaction.commit()
>>> c.close()
"""
def gracefully_handle_abort_while_storing_many_blobs():
r"""
>>> import logging, sys
>>> old_level = logging.getLogger().getEffectiveLevel()
>>> logging.getLogger().setLevel(logging.ERROR)
>>> handler = logging.StreamHandler(sys.stdout)
>>> logging.getLogger().addHandler(handler)
>>> addr, _ = start_server(blob_dir='blobs')
>>> c = ZEO.connection(addr, blob_dir='cblobs')
>>> c.root.x = ZODB.blob.Blob('z'*(1<<20))
>>> c.root.y = ZODB.blob.Blob('z'*(1<<2))
>>> t = c.transaction_manager.get()
>>> c.tpc_begin(t)
>>> c.commit(t)
We've called commit, but the blob sends are queued. We'll call abort
right away, which will delete the temporary blob files. The queued
iterators will try to open these files.
>>> c.tpc_abort(t)
Now we'll try to use the connection, mainly to wait for everything to
get processed. Before we fixed this by making tpc_finish a synchronous
call to the server. we'd get some sort of error here.
>>> _ = c._storage._server.loadEx('\0'*8)
>>> c.close()
>>> logging.getLogger().removeHandler(handler)
>>> logging.getLogger().setLevel(old_level)
"""
if sys.platform.startswith('win'):
del runzeo_logrotate_on_sigusr2
del unix_domain_sockets
if sys.version_info >= (2, 6):
import multiprocessing
def work_with_multiprocessing_process(name, addr, q):
conn = ZEO.connection(addr)
q.put((name, conn.root.x))
conn.close()
class MultiprocessingTests(unittest.TestCase):
layer = ZODB.tests.util.MininalTestLayer('work_with_multiprocessing')
def test_work_with_multiprocessing(self):
"Client storage should work with multi-processing."
# Gaaa, zope.testing.runner.FakeInputContinueGenerator has no close
if not hasattr(sys.stdin, 'close'):
sys.stdin.close = lambda : None
if not hasattr(sys.stdin, 'fileno'):
sys.stdin.fileno = lambda : -1
self.globs = {}
forker.setUp(self)
addr, adminaddr = self.globs['start_server']()
conn = ZEO.connection(addr)
conn.root.x = 1
transaction.commit()
q = multiprocessing.Queue()
processes = [multiprocessing.Process(
target=work_with_multiprocessing_process,
args=(i, addr, q))
for i in range(3)]
_ = [p.start() for p in processes]
self.assertEqual(sorted(q.get(timeout=300) for p in processes),
[(0, 1), (1, 1), (2, 1)])
_ = [p.join(30) for p in processes]
conn.close()
zope.testing.setupstack.tearDown(self)
else:
class MultiprocessingTests(unittest.TestCase):
pass
def quick_close_doesnt_kill_server():
r"""
Start a server:
>>> addr, _ = start_server()
Now connect and immediately disconnect. This caused the server to
die in the past:
>>> import socket, struct
>>> for i in range(5):
... s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
... s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
... struct.pack('ii', 1, 0))
... s.connect(addr)
... s.close()
Now we should be able to connect as normal:
>>> db = ZEO.DB(addr)
>>> db.storage.is_connected()
True
>>> db.close()
"""
def sync_connect_doesnt_hang():
r"""
>>> import threading
>>> import ZEO.zrpc.client
>>> ConnectThread = ZEO.zrpc.client.ConnectThread
>>> ZEO.zrpc.client.ConnectThread = lambda *a, **kw: threading.Thread()
>>> class CM(ZEO.zrpc.client.ConnectionManager):
... sync_wait = 1
... _start_asyncore_loop = lambda self: None
>>> cm = CM(('', 0), object())
Calling connect results in an exception being raised, instead of hanging
indefinitely when the thread dies without setting up the connection.
>>> cm.connect(sync=1)
Traceback (most recent call last):
...
AssertionError
>>> cm.thread.isAlive()
False
>>> ZEO.zrpc.client.ConnectThread = ConnectThread
"""
def lp143344_extension_methods_not_lost_on_server_restart():
r"""
Make sure we don't lose exension methods on server restart.
>>> addr, adminaddr = start_server(keep=True)
>>> conn = ZEO.connection(addr)
>>> conn.root.x = 1
>>> transaction.commit()
>>> conn.db().storage.answer_to_the_ultimate_question()
42
>>> stop_server(adminaddr)
>>> wait_until('not connected',
... lambda : not conn.db().storage.is_connected())
>>> _ = start_server(addr=addr)
>>> wait_until('connected', conn.db().storage.is_connected)
>>> conn.root.x
1
>>> conn.db().storage.answer_to_the_ultimate_question()
42
>>> conn.close()
"""
def can_use_empty_string_for_local_host_on_client():
"""We should be able to spell localhost with ''.
>>> (_, port), _ = start_server()
>>> conn = ZEO.connection(('', port))
>>> conn.root()
{}
>>> conn.root.x = 1
>>> transaction.commit()
>>> conn.close()
"""
slow_test_classes = [
BlobAdaptedFileStorageTests, BlobWritableCacheTests,
MappingStorageTests, DemoStorageTests,
FileStorageTests, FileStorageHexTests, FileStorageClientHexTests,
]
quick_test_classes = [
FileStorageRecoveryTests, ConfigurationTests, HeartbeatTests,
ZRPCConnectionTests,
]
class ServerManagingClientStorage(ClientStorage):
class StorageServerStubClass(ZEO.ServerStub.StorageServer):
# Wait for abort for the benefit of blob_transaction.txt
def tpc_abort(self, id):
self.rpc.call('tpc_abort', id)
def __init__(self, name, blob_dir, shared=False, extrafsoptions=''):
if shared:
server_blob_dir = blob_dir
else:
server_blob_dir = 'server-'+blob_dir
self.globs = {}
port = forker.get_port2(self)
addr, admin, pid, config = forker.start_zeo_server(
"""
<blobstorage>
blob-dir %s
<filestorage>
path %s
%s
</filestorage>
</blobstorage>
""" % (server_blob_dir, name+'.fs', extrafsoptions),
port=port,
)
os.remove(config)
zope.testing.setupstack.register(self, os.waitpid, pid, 0)
zope.testing.setupstack.register(
self, forker.shutdown_zeo_server, admin)
if shared:
ClientStorage.__init__(self, addr, blob_dir=blob_dir,
shared_blob_dir=True)
else:
ClientStorage.__init__(self, addr, blob_dir=blob_dir)
def close(self):
ClientStorage.close(self)
zope.testing.setupstack.tearDown(self)
def create_storage_shared(name, blob_dir):
return ServerManagingClientStorage(name, blob_dir, True)
class ServerManagingClientStorageForIExternalGCTest(
ServerManagingClientStorage):
def pack(self, t=None, referencesf=None):
ServerManagingClientStorage.pack(self, t, referencesf, wait=True)
# Packing doesn't clear old versions out of zeo client caches,
# so we'll clear the caches.
self._cache.clear()
ZEO.ClientStorage._check_blob_cache_size(self.blob_dir, 0)
def test_suite():
suite = unittest.TestSuite()
# Collect misc tests into their own layer to educe size of
# unit test layer
zeo = unittest.TestSuite()
zeo.addTest(unittest.makeSuite(ZODB.tests.util.AAAA_Test_Runner_Hack))
zeo.addTest(doctest.DocTestSuite(
setUp=forker.setUp, tearDown=zope.testing.setupstack.tearDown,
checker=renormalizing.RENormalizing([
(re.compile(r"'start': '[^\n]+'"), 'start'),
]),
))
zeo.addTest(doctest.DocTestSuite(ZEO.tests.IterationTests,
setUp=forker.setUp, tearDown=zope.testing.setupstack.tearDown))
zeo.addTest(doctest.DocFileSuite('registerDB.test'))
zeo.addTest(
doctest.DocFileSuite(
'zeo-fan-out.test', 'zdoptions.test',
'drop_cache_rather_than_verify.txt', 'client-config.test',
'protocols.test', 'zeo_blob_cache.test', 'invalidation-age.txt',
'dynamic_server_ports.test', 'new_addr.test',
setUp=forker.setUp, tearDown=zope.testing.setupstack.tearDown,
),
)
zeo.addTest(PackableStorage.IExternalGC_suite(
lambda :
ServerManagingClientStorageForIExternalGCTest(
'data.fs', 'blobs', extrafsoptions='pack-gc false')
))
for klass in quick_test_classes:
zeo.addTest(unittest.makeSuite(klass, "check"))
zeo.layer = ZODB.tests.util.MininalTestLayer('testZeo-misc')
suite.addTest(zeo)
suite.addTest(unittest.makeSuite(MultiprocessingTests))
# Put the heavyweights in their own layers
for klass in slow_test_classes:
sub = unittest.makeSuite(klass, "check")
sub.layer = ZODB.tests.util.MininalTestLayer(klass.__name__)
suite.addTest(sub)
suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
'ClientStorageNonSharedBlobs', ServerManagingClientStorage))
suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
'ClientStorageSharedBlobs', create_storage_shared))
return suite
if __name__ == "__main__":
unittest.main(defaultTest="test_suite")
src/ZEO/tests/testZEO2.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from zope.testing import setupstack, renormalizing
import doctest
import logging
import pprint
import re
import sys
import transaction
import unittest
import ZEO.StorageServer
import ZEO.tests.servertesting
import ZODB.blob
import ZODB.FileStorage
import ZODB.tests.util
import ZODB.utils
def proper_handling_of_blob_conflicts():
r"""
Conflict errors weren't properly handled when storing blobs, the
result being that the storage was left in a transaction.
We originally saw this when restarting a block transaction, although
it doesn't really matter.
Set up the storage with some initial blob data.
>>> fs = ZODB.FileStorage.FileStorage('t.fs', blob_dir='t.blobs')
>>> db = ZODB.DB(fs)
>>> conn = db.open()
>>> conn.root.b = ZODB.blob.Blob('x')
>>> transaction.commit()
Get the iod and first serial. We'll use the serial later to provide
out-of-date data.
>>> oid = conn.root.b._p_oid
>>> serial = conn.root.b._p_serial
>>> conn.root.b.open('w').write('y')
>>> transaction.commit()
>>> data = fs.load(oid)[0]
Create the server:
>>> server = ZEO.tests.servertesting.StorageServer('x', {'1': fs})
And an initial client.
>>> zs1 = ZEO.StorageServer.ZEOStorage(server)
>>> conn1 = ZEO.tests.servertesting.Connection(1)
>>> zs1.notifyConnected(conn1)
>>> zs1.register('1', 0)
>>> zs1.tpc_begin('0', '', '', {})
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', '0')
>>> _ = zs1.vote('0') # doctest: +ELLIPSIS
1 callAsync serialnos ...
In a second client, we'll try to commit using the old serial. This
will conflict. It will be blocked at the vote call.
>>> zs2 = ZEO.StorageServer.ZEOStorage(server)
>>> conn2 = ZEO.tests.servertesting.Connection(2)
>>> zs2.notifyConnected(conn2)
>>> zs2.register('1', 0)
>>> zs2.tpc_begin('1', '', '', {})
>>> zs2.storeBlobStart()
>>> zs2.storeBlobChunk('z')
>>> zs2.storeBlobEnd(oid, serial, data, '1')
>>> delay = zs2.vote('1')
>>> class Sender:
... def send_reply(self, id, reply):
... print 'reply', id, reply
>>> delay.set_sender(1, Sender())
>>> logger = logging.getLogger('ZEO')
>>> handler = logging.StreamHandler(sys.stdout)
>>> logger.setLevel(logging.INFO)
>>> logger.addHandler(handler)
Now, when we abort the transaction for the first client. the second
client will be restarted. It will get a conflict error, that is
handled correctly:
>>> zs1.tpc_abort('0') # doctest: +ELLIPSIS
2 callAsync serialnos ...
reply 1 None
>>> fs.tpc_transaction() is not None
True
>>> conn2.connected
True
>>> logger.setLevel(logging.NOTSET)
>>> logger.removeHandler(handler)
>>> zs2.tpc_abort('1')
>>> fs.close()
"""
def proper_handling_of_errors_in_restart():
r"""
It's critical that if there is an error in vote that the
storage isn't left in tpc.
>>> fs = ZODB.FileStorage.FileStorage('t.fs', blob_dir='t.blobs')
>>> server = ZEO.tests.servertesting.StorageServer('x', {'1': fs})
And an initial client.
>>> zs1 = ZEO.StorageServer.ZEOStorage(server)
>>> conn1 = ZEO.tests.servertesting.Connection(1)
>>> zs1.notifyConnected(conn1)
>>> zs1.register('1', 0)
>>> zs1.tpc_begin('0', '', '', {})
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', '0')
Intentionally break zs1:
>>> zs1._store = lambda : None
>>> _ = zs1.vote('0') # doctest: +ELLIPSIS
Traceback (most recent call last):
...
TypeError: <lambda>() takes no arguments (3 given)
We're not in a transaction:
>>> fs.tpc_transaction() is None
True
We can start another client and get the storage lock.
>>> zs1 = ZEO.StorageServer.ZEOStorage(server)
>>> conn1 = ZEO.tests.servertesting.Connection(1)
>>> zs1.notifyConnected(conn1)
>>> zs1.register('1', 0)
>>> zs1.tpc_begin('1', '', '', {})
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', '1')
>>> _ = zs1.vote('1') # doctest: +ELLIPSIS
1 callAsync serialnos ...
>>> zs1.tpc_finish('1').set_sender(0, conn1)
>>> fs.close()
"""
def errors_in_vote_should_clear_lock():
"""
So, we arrange to get an error in vote:
>>> import ZODB.MappingStorage
>>> vote_should_fail = True
>>> class MappingStorage(ZODB.MappingStorage.MappingStorage):
... def tpc_vote(*args):
... if vote_should_fail:
... raise ValueError
... return ZODB.MappingStorage.MappingStorage.tpc_vote(*args)
>>> server = ZEO.tests.servertesting.StorageServer(
... 'x', {'1': MappingStorage()})
>>> zs = ZEO.StorageServer.ZEOStorage(server)
>>> conn = ZEO.tests.servertesting.Connection(1)
>>> zs.notifyConnected(conn)
>>> zs.register('1', 0)
>>> zs.tpc_begin('0', '', '', {})
>>> zs.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', '0')
>>> zs.vote('0')
Traceback (most recent call last):
...
ValueError
When we do, the storage server's transaction lock shouldn't be held:
>>> '1' in server._commit_locks
False
Of course, if vote suceeds, the lock will be held:
>>> vote_should_fail = False
>>> zs.tpc_begin('1', '', '', {})
>>> zs.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', '1')
>>> _ = zs.vote('1') # doctest: +ELLIPSIS
1 callAsync serialnos ...
>>> '1' in server._commit_locks
True
>>> zs.tpc_abort('1')
"""
def some_basic_locking_tests():
r"""
>>> itid = 0
>>> def start_trans(zs):
... global itid
... itid += 1
... tid = str(itid)
... zs.tpc_begin(tid, '', '', {})
... zs.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', tid)
... return tid
>>> server = ZEO.tests.servertesting.StorageServer()
>>> handler = logging.StreamHandler(sys.stdout)
>>> handler.setFormatter(logging.Formatter(
... '%(name)s %(levelname)s\n%(message)s'))
>>> logging.getLogger('ZEO').addHandler(handler)
>>> logging.getLogger('ZEO').setLevel(logging.DEBUG)
We start a transaction and vote, this leads to getting the lock.
>>> zs1 = ZEO.tests.servertesting.client(server, '1')
>>> tid1 = start_trans(zs1)
>>> zs1.vote(tid1) # doctest: +ELLIPSIS
ZEO.StorageServer DEBUG
(test-addr-1) ('1') lock: transactions waiting: 0
ZEO.StorageServer BLATHER
(test-addr-1) Preparing to commit transaction: 1 objects, 36 bytes
1 callAsync serialnos ...
If another client tried to vote, it's lock request will be queued and
a delay will be returned:
>>> zs2 = ZEO.tests.servertesting.client(server, '2')
>>> tid2 = start_trans(zs2)
>>> delay = zs2.vote(tid2)
ZEO.StorageServer DEBUG
(test-addr-2) ('1') queue lock: transactions waiting: 1
>>> delay.set_sender(0, zs2.connection)
When we end the first transaction, the queued vote gets the lock.
>>> zs1.tpc_abort(tid1) # doctest: +ELLIPSIS
ZEO.StorageServer DEBUG
(test-addr-1) ('1') unlock: transactions waiting: 1
ZEO.StorageServer DEBUG
(test-addr-2) ('1') lock: transactions waiting: 0
ZEO.StorageServer BLATHER
(test-addr-2) Preparing to commit transaction: 1 objects, 36 bytes
2 callAsync serialnos ...
Let's try again with the first client. The vote will be queued:
>>> tid1 = start_trans(zs1)
>>> delay = zs1.vote(tid1)
ZEO.StorageServer DEBUG
(test-addr-1) ('1') queue lock: transactions waiting: 1
If the queued transaction is aborted, it will be dequeued:
>>> zs1.tpc_abort(tid1) # doctest: +ELLIPSIS
ZEO.StorageServer DEBUG
(test-addr-1) ('1') dequeue lock: transactions waiting: 0
BTW, voting multiple times will error:
>>> zs2.vote(tid2)
Traceback (most recent call last):
...
StorageTransactionError: Already voting (locked)
>>> tid1 = start_trans(zs1)
>>> delay = zs1.vote(tid1)
ZEO.StorageServer DEBUG
(test-addr-1) ('1') queue lock: transactions waiting: 1
>>> delay.set_sender(0, zs1.connection)
>>> zs1.vote(tid1)
Traceback (most recent call last):
...
StorageTransactionError: Already voting (waiting)
Note that the locking activity is logged at debug level to avoid
cluttering log files, however, as the number of waiting votes
increased, so does the logging level:
>>> clients = []
>>> for i in range(9):
... client = ZEO.tests.servertesting.client(server, str(i+10))
... tid = start_trans(client)
... delay = client.vote(tid)
... clients.append(client)
ZEO.StorageServer DEBUG
(test-addr-10) ('1') queue lock: transactions waiting: 2
ZEO.StorageServer DEBUG
(test-addr-11) ('1') queue lock: transactions waiting: 3
ZEO.StorageServer WARNING
(test-addr-12) ('1') queue lock: transactions waiting: 4
ZEO.StorageServer WARNING
(test-addr-13) ('1') queue lock: transactions waiting: 5
ZEO.StorageServer WARNING
(test-addr-14) ('1') queue lock: transactions waiting: 6
ZEO.StorageServer WARNING
(test-addr-15) ('1') queue lock: transactions waiting: 7
ZEO.StorageServer WARNING
(test-addr-16) ('1') queue lock: transactions waiting: 8
ZEO.StorageServer WARNING
(test-addr-17) ('1') queue lock: transactions waiting: 9
ZEO.StorageServer CRITICAL
(test-addr-18) ('1') queue lock: transactions waiting: 10
If a client with the transaction lock disconnects, it will abort and
release the lock and one of the waiting clients will get the lock.
>>> zs2.notifyDisconnected() # doctest: +ELLIPSIS
ZEO.StorageServer INFO
(test-addr-2) disconnected during locked transaction
ZEO.StorageServer CRITICAL
(test-addr-2) ('1') unlock: transactions waiting: 10
ZEO.StorageServer WARNING
(test-addr-1) ('1') lock: transactions waiting: 9
ZEO.StorageServer BLATHER
(test-addr-1) Preparing to commit transaction: 1 objects, 36 bytes
1 callAsync serialnos ...
(In practice, waiting clients won't necessarily get the lock in order.)
We can find out about the current lock state, and get other server
statistics using the server_status method:
>>> pprint.pprint(zs1.server_status(), width=1)
{'aborts': 3,
'active_txns': 10,
'commits': 0,
'conflicts': 0,
'conflicts_resolved': 0,
'connections': 11,
'loads': 0,
'lock_time': 1272653598.693882,
'start': 'Fri Apr 30 14:53:18 2010',
'stores': 13,
'timeout-thread-is-alive': 'stub',
'verifying_clients': 0,
'waiting': 9}
(Note that the connections count above is off by 1 due to the way the
test infrastructure works.)
If clients disconnect while waiting, they will be dequeued:
>>> for client in clients:
... client.notifyDisconnected()
ZEO.StorageServer INFO
(test-addr-10) disconnected during unlocked transaction
ZEO.StorageServer WARNING
(test-addr-10) ('1') dequeue lock: transactions waiting: 8
ZEO.StorageServer INFO
(test-addr-11) disconnected during unlocked transaction
ZEO.StorageServer WARNING
(test-addr-11) ('1') dequeue lock: transactions waiting: 7
ZEO.StorageServer INFO
(test-addr-12) disconnected during unlocked transaction
ZEO.StorageServer WARNING
(test-addr-12) ('1') dequeue lock: transactions waiting: 6
ZEO.StorageServer INFO
(test-addr-13) disconnected during unlocked transaction
ZEO.StorageServer WARNING
(test-addr-13) ('1') dequeue lock: transactions waiting: 5
ZEO.StorageServer INFO
(test-addr-14) disconnected during unlocked transaction
ZEO.StorageServer WARNING
(test-addr-14) ('1') dequeue lock: transactions waiting: 4
ZEO.StorageServer INFO
(test-addr-15) disconnected during unlocked transaction
ZEO.StorageServer DEBUG
(test-addr-15) ('1') dequeue lock: transactions waiting: 3
ZEO.StorageServer INFO
(test-addr-16) disconnected during unlocked transaction
ZEO.StorageServer DEBUG
(test-addr-16) ('1') dequeue lock: transactions waiting: 2
ZEO.StorageServer INFO
(test-addr-17) disconnected during unlocked transaction
ZEO.StorageServer DEBUG
(test-addr-17) ('1') dequeue lock: transactions waiting: 1
ZEO.StorageServer INFO
(test-addr-18) disconnected during unlocked transaction
ZEO.StorageServer DEBUG
(test-addr-18) ('1') dequeue lock: transactions waiting: 0
>>> zs1.tpc_abort(tid1)
>>> logging.getLogger('ZEO').setLevel(logging.NOTSET)
>>> logging.getLogger('ZEO').removeHandler(handler)
"""
def lock_sanity_check():
r"""
On one occasion with 3.10.0a1 in production, we had a case where a
transaction lock wasn't released properly. One possibility, fron
scant log information, is that the server and ZEOStorage had different
ideas about whether the ZEOStorage was locked. The timeout thread
properly closed the ZEOStorage's connection, but the ZEOStorage didn't
release it's lock, presumably because it thought it wasn't locked. I'm
not sure why this happened. I've refactored the logic quite a bit to
try to deal with this, but the consequences of this failure are so
severe, I'm adding some sanity checking when queueing lock requests.
Helper to manage transactions:
>>> itid = 0
>>> def start_trans(zs):
... global itid
... itid += 1
... tid = str(itid)
... zs.tpc_begin(tid, '', '', {})
... zs.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', tid)
... return tid
Set up server and logging:
>>> server = ZEO.tests.servertesting.StorageServer()
>>> handler = logging.StreamHandler(sys.stdout)
>>> handler.setFormatter(logging.Formatter(
... '%(name)s %(levelname)s\n%(message)s'))
>>> logging.getLogger('ZEO').addHandler(handler)
>>> logging.getLogger('ZEO').setLevel(logging.DEBUG)
Now, we'll start a transaction, get the lock and then mark the
ZEOStorage as closed and see if trying to get a lock cleans it up:
>>> zs1 = ZEO.tests.servertesting.client(server, '1')
>>> tid1 = start_trans(zs1)
>>> zs1.vote(tid1) # doctest: +ELLIPSIS
ZEO.StorageServer DEBUG
(test-addr-1) ('1') lock: transactions waiting: 0
ZEO.StorageServer BLATHER
(test-addr-1) Preparing to commit transaction: 1 objects, 36 bytes
1 callAsync serialnos ...
>>> zs1.connection = None
>>> zs2 = ZEO.tests.servertesting.client(server, '2')
>>> tid2 = start_trans(zs2)
>>> zs2.vote(tid2) # doctest: +ELLIPSIS
ZEO.StorageServer CRITICAL
(test-addr-1) Still locked after disconnected. Unlocking.
ZEO.StorageServer DEBUG
(test-addr-2) ('1') lock: transactions waiting: 0
ZEO.StorageServer BLATHER
(test-addr-2) Preparing to commit transaction: 1 objects, 36 bytes
2 callAsync serialnos ...
>>> zs1.txnlog.close()
>>> zs2.tpc_abort(tid2)
>>> logging.getLogger('ZEO').setLevel(logging.NOTSET)
>>> logging.getLogger('ZEO').removeHandler(handler)
"""
def test_suite():
return unittest.TestSuite((
doctest.DocTestSuite(
setUp=ZODB.tests.util.setUp, tearDown=setupstack.tearDown,
checker=renormalizing.RENormalizing([
(re.compile('\d+/test-addr'), ''),
(re.compile("'lock_time': \d+.\d+"), 'lock_time'),
(re.compile(r"'start': '[^\n]+'"), 'start'),
]),
),
))
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
src/ZEO/tests/testZEOOptions.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for ZEO.runzeo.ZEOOptions."""
import os
import tempfile
import unittest
import ZODB.config
from ZEO.runzeo import ZEOOptions
from zdaemon.tests.testzdoptions import TestZDOptions
# When a hostname isn't specified in a socket binding address, ZConfig
# supplies the empty string.
DEFAULT_BINDING_HOST = ""
class TestZEOOptions(TestZDOptions):
OptionsClass = ZEOOptions
input_args = ["-f", "Data.fs", "-a", "5555"]
output_opts = [("-f", "Data.fs"), ("-a", "5555")]
output_args = []
configdata = """
<zeo>
address 5555
</zeo>
<filestorage fs>
path Data.fs
</filestorage>
"""
def setUp(self):
self.tempfilename = tempfile.mktemp()
f = open(self.tempfilename, "w")
f.write(self.configdata)
f.close()
def tearDown(self):
try:
os.remove(self.tempfilename)
except os.error:
pass
def test_configure(self):
# Hide the base class test_configure
pass
def test_default_help(self): pass # disable silly test w spurious failures
def test_defaults_with_schema(self):
options = self.OptionsClass()
options.realize(["-C", self.tempfilename])
self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 5555))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.name, "fs")
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_defaults_without_schema(self):
options = self.OptionsClass()
options.realize(["-a", "5555", "-f", "Data.fs"])
self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 5555))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.name, "1")
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(opener.config.path, "Data.fs")
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_commandline_overrides(self):
options = self.OptionsClass()
options.realize(["-C", self.tempfilename,
"-a", "6666", "-f", "Wisdom.fs"])
self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 6666))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(opener.config.path, "Wisdom.fs")
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_suite():
suite = unittest.TestSuite()
for cls in [TestZEOOptions]:
suite.addTest(unittest.makeSuite(cls))
return suite
if __name__ == "__main__":
unittest.main(defaultTest='test_suite')
src/ZEO/tests/test_cache.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Basic unit tests for a client cache."""
from ZODB.utils import p64, repr_to_oid
import doctest
import os
import re
import string
import struct
import sys
import tempfile
import unittest
import ZEO.cache
import ZODB.tests.util
import zope.testing.setupstack
import zope.testing.renormalizing
import ZEO.cache
from ZODB.utils import p64, u64, z64
n1 = p64(1)
n2 = p64(2)
n3 = p64(3)
n4 = p64(4)
n5 = p64(5)
def hexprint(file):
file.seek(0)
data = file.read()
offset = 0
while data:
line, data = data[:16], data[16:]
printable = ""
hex = ""
for character in line:
if (character in string.printable
and not ord(character) in [12,13,9]):
printable += character
else:
printable += '.'
hex += character.encode('hex') + ' '
hex = hex[:24] + ' ' + hex[24:]
hex = hex.ljust(49)
printable = printable.ljust(16)
print '%08x %s |%s|' % (offset, hex, printable)
offset += 16
def oid(o):
repr = '%016x' % o
return repr_to_oid(repr)
tid = oid
class CacheTests(ZODB.tests.util.TestCase):
def setUp(self):
# The default cache size is much larger than we need here. Since
# testSerialization reads the entire file into a string, it's not
# good to leave it that big.
ZODB.tests.util.TestCase.setUp(self)
self.cache = ZEO.cache.ClientCache(size=1024**2)
def tearDown(self):
self.cache.close()
if self.cache.path:
os.remove(self.cache.path)
ZODB.tests.util.TestCase.tearDown(self)
def testLastTid(self):
self.assertEqual(self.cache.getLastTid(), z64)
self.cache.setLastTid(n2)
self.assertEqual(self.cache.getLastTid(), n2)
self.assertEqual(self.cache.getLastTid(), n2)
self.cache.setLastTid(n3)
self.assertEqual(self.cache.getLastTid(), n3)
# Check that setting tids out of order gives an error:
# the cache complains only when it's non-empty
self.cache.store(n1, n3, None, 'x')
self.assertRaises(ValueError, self.cache.setLastTid, n2)
def testLoad(self):
data1 = "data for n1"
self.assertEqual(self.cache.load(n1), None)
self.cache.store(n1, n3, None, data1)
self.assertEqual(self.cache.load(n1), (data1, n3))
def testInvalidate(self):
data1 = "data for n1"
self.cache.store(n1, n3, None, data1)
self.cache.invalidate(n2, n2)
self.cache.invalidate(n1, n4)
self.assertEqual(self.cache.load(n1), None)
self.assertEqual(self.cache.loadBefore(n1, n4),
(data1, n3, n4))
def testNonCurrent(self):
data1 = "data for n1"
data2 = "data for n2"
self.cache.store(n1, n4, None, data1)
self.cache.store(n1, n2, n3, data2)
# can't say anything about state before n2
self.assertEqual(self.cache.loadBefore(n1, n2), None)
# n3 is the upper bound of non-current record n2
self.assertEqual(self.cache.loadBefore(n1, n3), (data2, n2, n3))
# no data for between n2 and n3
self.assertEqual(self.cache.loadBefore(n1, n4), None)
self.cache.invalidate(n1, n5)
self.assertEqual(self.cache.loadBefore(n1, n5), (data1, n4, n5))
self.assertEqual(self.cache.loadBefore(n2, n4), None)
def testException(self):
self.cache.store(n1, n2, None, "data")
self.cache.store(n1, n2, None, "data")
self.assertRaises(ValueError,
self.cache.store,
n1, n3, None, "data")
def testEviction(self):
# Manually override the current maxsize
cache = ZEO.cache.ClientCache(None, 3395)
# Trivial test of eviction code. Doesn't test non-current
# eviction.
data = ["z" * i for i in range(100)]
for i in range(50):
n = p64(i)
cache.store(n, n, None, data[i])
self.assertEquals(len(cache), i + 1)
# The cache is now almost full. The next insert
# should delete some objects.
n = p64(50)
cache.store(n, n, None, data[51])
self.assert_(len(cache) < 51)
# TODO: Need to make sure eviction of non-current data
# are handled correctly.
def testSerialization(self):
self.cache.store(n1, n2, None, "data for n1")
self.cache.store(n3, n3, n4, "non-current data for n3")
self.cache.store(n3, n4, n5, "more non-current data for n3")
path = tempfile.mktemp()
# Copy data from self.cache into path, reaching into the cache
# guts to make the copy.
dst = open(path, "wb+")
src = self.cache.f
src.seek(0)
dst.write(src.read(self.cache.maxsize))
dst.close()
copy = ZEO.cache.ClientCache(path)
# Verify that internals of both objects are the same.
# Could also test that external API produces the same results.
eq = self.assertEqual
eq(copy.getLastTid(), self.cache.getLastTid())
eq(len(copy), len(self.cache))
eq(dict(copy.current), dict(self.cache.current))
eq(dict([(k, dict(v)) for (k, v) in copy.noncurrent.items()]),
dict([(k, dict(v)) for (k, v) in self.cache.noncurrent.items()]),
)
def testCurrentObjectLargerThanCache(self):
if self.cache.path:
os.remove(self.cache.path)
self.cache = ZEO.cache.ClientCache(size=50)
# We store an object that is a bit larger than the cache can handle.
self.cache.store(n1, n2, None, "x"*64)
# We can see that it was not stored.
self.assertEquals(None, self.cache.load(n1))
# If an object cannot be stored in the cache, it must not be
# recorded as current.
self.assert_(n1 not in self.cache.current)
# Regression test: invalidation must still work.
self.cache.invalidate(n1, n2)
def testOldObjectLargerThanCache(self):
if self.cache.path:
os.remove(self.cache.path)
cache = ZEO.cache.ClientCache(size=50)
# We store an object that is a bit larger than the cache can handle.
cache.store(n1, n2, n3, "x"*64)
# We can see that it was not stored.
self.assertEquals(None, cache.load(n1))
# If an object cannot be stored in the cache, it must not be
# recorded as non-current.
self.assert_(1 not in cache.noncurrent)
def testVeryLargeCaches(self):
cache = ZEO.cache.ClientCache('cache', size=(1<<32)+(1<<20))
cache.store(n1, n2, None, "x")
cache.close()
cache = ZEO.cache.ClientCache('cache', size=(1<<33)+(1<<20))
self.assertEquals(cache.load(n1), ('x', n2))
cache.close()
def testConversionOfLargeFreeBlocks(self):
f = open('cache', 'wb')
f.write(ZEO.cache.magic+
'\0'*8 +
'f'+struct.pack(">I", (1<<32)-12)
)
f.seek((1<<32)-1)
f.write('x')
f.close()
cache = ZEO.cache.ClientCache('cache', size=1<<32)
cache.close()
cache = ZEO.cache.ClientCache('cache', size=1<<32)
cache.close()
f = open('cache', 'rb')
f.seek(12)
self.assertEquals(f.read(1), 'f')
self.assertEquals(struct.unpack(">I", f.read(4))[0],
ZEO.cache.max_block_size)
f.close()
if not sys.platform.startswith('linux'):
# On platforms without sparse files, these tests are just way
# too hard on the disk and take too long (especially in a windows
# VM).
del testVeryLargeCaches
del testConversionOfLargeFreeBlocks
def test_clear_zeo_cache(self):
cache = self.cache
for i in range(10):
cache.store(p64(i), n2, None, str(i))
cache.store(p64(i), n1, n2, str(i)+'old')
self.assertEqual(len(cache), 20)
self.assertEqual(cache.load(n3), ('3', n2))
self.assertEqual(cache.loadBefore(n3, n2), ('3old', n1, n2))
cache.clear()
self.assertEqual(len(cache), 0)
self.assertEqual(cache.load(n3), None)
self.assertEqual(cache.loadBefore(n3, n2), None)
def testChangingCacheSize(self):
# start with a small cache
data = 'x'
recsize = ZEO.cache.allocated_record_overhead+len(data)
for extra in (2, recsize-2):
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+100*recsize+extra)
for i in range(100):
cache.store(p64(i), n1, None, data)
self.assertEquals(len(cache), 100)
self.assertEquals(os.path.getsize(
'cache'), ZEO.cache.ZEC_HEADER_SIZE+100*recsize+extra)
# Now make it smaller
cache.close()
small = 50
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+small*recsize+extra)
self.assertEquals(len(cache), small)
self.assertEquals(os.path.getsize(
'cache'), ZEO.cache.ZEC_HEADER_SIZE+small*recsize+extra)
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
set(range(small)))
for i in range(100, 110):
cache.store(p64(i), n1, None, data)
# We use small-1 below because an extra object gets
# evicted because of the optimization to assure that we
# always get a free block after a new allocated block.
expected_len = small - 1
self.assertEquals(len(cache), expected_len)
expected_oids = set(range(11, 50)+range(100, 110))
self.assertEquals(
set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
# Make sure we can reopen with same size
cache.close()
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+small*recsize+extra)
self.assertEquals(len(cache), expected_len)
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
# Now make it bigger
cache.close()
large = 150
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+large*recsize+extra)
self.assertEquals(len(cache), expected_len)
self.assertEquals(os.path.getsize(
'cache'), ZEO.cache.ZEC_HEADER_SIZE+large*recsize+extra)
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
for i in range(200, 305):
cache.store(p64(i), n1, None, data)
# We use large-2 for the same reason we used small-1 above.
expected_len = large-2
self.assertEquals(len(cache), expected_len)
expected_oids = set(range(11, 50)+range(106, 110)+range(200, 305))
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
# Make sure we can reopen with same size
cache.close()
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+large*recsize+extra)
self.assertEquals(len(cache), expected_len)
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
# Cleanup
cache.close()
os.remove('cache')
def testSetAnyLastTidOnEmptyCache(self):
self.cache.setLastTid(p64(5))
self.cache.setLastTid(p64(5))
self.cache.setLastTid(p64(3))
self.cache.setLastTid(p64(4))
def kill_does_not_cause_cache_corruption():
r"""
If we kill a process while a cache is being written to, the cache
isn't corrupted. To see this, we'll write a little script that
writes records to a cache file repeatedly.
>>> import os, random, sys, time
>>> open('t', 'w').write('''
... import os, random, sys, thread, time
... sys.path = %r
...
... def suicide():
... time.sleep(random.random()/10)
... os._exit(0)
...
... import ZEO.cache
... from ZODB.utils import p64
... cache = ZEO.cache.ClientCache('cache')
... oid = 0
... t = 0
... thread.start_new_thread(suicide, ())
... while 1:
... oid += 1
... t += 1
... data = 'X' * random.randint(5000,25000)
... cache.store(p64(oid), p64(t), None, data)
...
... ''' % sys.path)
>>> for i in range(10):
... _ = os.spawnl(os.P_WAIT, sys.executable, sys.executable, 't')
... if os.path.exists('cache'):
... cache = ZEO.cache.ClientCache('cache')
... cache.close()
... os.remove('cache')
... os.remove('cache.lock')
"""
def full_cache_is_valid():
r"""
If we fill up the cache without any free space, the cache can
still be used.
>>> import ZEO.cache
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> data = 'X' * (1000 - ZEO.cache.ZEC_HEADER_SIZE - 41)
>>> cache.store(p64(1), p64(1), None, data)
>>> cache.close()
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache.store(p64(2), p64(2), None, 'XXX')
>>> cache.close()
"""
def cannot_open_same_cache_file_twice():
r"""
>>> import ZEO.cache
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache2 = ZEO.cache.ClientCache('cache', 1000)
Traceback (most recent call last):
...
LockError: Couldn't lock 'cache.lock'
>>> cache.close()
"""
def thread_safe():
r"""
>>> import ZEO.cache, ZODB.utils
>>> cache = ZEO.cache.ClientCache('cache', 1000000)
>>> for i in range(100):
... cache.store(ZODB.utils.p64(i), ZODB.utils.p64(1), None, '0')
>>> import random, sys, threading
>>> random = random.Random(0)
>>> stop = False
>>> read_failure = None
>>> def read_thread():
... def pick_oid():
... return ZODB.utils.p64(random.randint(0,99))
...
... try:
... while not stop:
... cache.load(pick_oid())
... cache.loadBefore(pick_oid(), ZODB.utils.p64(2))
... except:
... global read_failure
... read_failure = sys.exc_info()
>>> thread = threading.Thread(target=read_thread)
>>> thread.start()
>>> for tid in range(2,10):
... for oid in range(100):
... oid = ZODB.utils.p64(oid)
... cache.invalidate(oid, ZODB.utils.p64(tid))
... cache.store(oid, ZODB.utils.p64(tid), None, str(tid))
>>> stop = True
>>> thread.join()
>>> if read_failure:
... print 'Read failure:'
... import traceback
... traceback.print_exception(*read_failure)
>>> expected = '9', ZODB.utils.p64(9)
>>> for oid in range(100):
... loaded = cache.load(ZODB.utils.p64(oid))
... if loaded != expected:
... print oid, loaded
>>> cache.close()
"""
def broken_non_current():
r"""
In production, we saw a situation where an _del_noncurrent raused
a key error when trying to free space, causing the cache to become
unusable. I can't see why this would occur, but added a logging
exception handler so, in the future, we'll still see cases in the
log, but will ignore the error and keep going.
>>> import ZEO.cache, ZODB.utils, logging, sys
>>> logger = logging.getLogger('ZEO.cache')
>>> logger.setLevel(logging.ERROR)
>>> handler = logging.StreamHandler(sys.stdout)
>>> logger.addHandler(handler)
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache.store(ZODB.utils.p64(1), ZODB.utils.p64(1), None, '0')
>>> cache.invalidate(ZODB.utils.p64(1), ZODB.utils.p64(2))
>>> cache._del_noncurrent(ZODB.utils.p64(1), ZODB.utils.p64(2))
... # doctest: +NORMALIZE_WHITESPACE
Couldn't find non-current
('\x00\x00\x00\x00\x00\x00\x00\x01', '\x00\x00\x00\x00\x00\x00\x00\x02')
>>> cache._del_noncurrent(ZODB.utils.p64(1), ZODB.utils.p64(1))
>>> cache._del_noncurrent(ZODB.utils.p64(1), ZODB.utils.p64(1)) #
... # doctest: +NORMALIZE_WHITESPACE
Couldn't find non-current
('\x00\x00\x00\x00\x00\x00\x00\x01', '\x00\x00\x00\x00\x00\x00\x00\x01')
>>> logger.setLevel(logging.NOTSET)
>>> logger.removeHandler(handler)
>>> cache.close()
"""
# def bad_magic_number(): See rename_bad_cache_file
def cache_trace_analysis():
r"""
Check to make sure the cache analysis scripts work.
>>> import time
>>> timetime = time.time
>>> now = 1278864701.5
>>> time.time = lambda : now
>>> os.environ["ZEO_CACHE_TRACE"] = 'yes'
>>> import random
>>> random = random.Random(42)
>>> history = []
>>> serial = 1
>>> for i in range(1000):
... serial += 1
... oid = random.randint(i+1000, i+6000)
... history.append(('s', p64(oid), p64(serial),
... 'x'*random.randint(200,2000)))
... for j in range(10):
... oid = random.randint(i+1000, i+6000)
... history.append(('l', p64(oid), p64(serial),
... 'x'*random.randint(200,2000)))
>>> def cache_run(name, size):
... serial = 1
... random.seed(42)
... global now
... now = 1278864701.5
... cache = ZEO.cache.ClientCache(name, size*(1<<20))
... for action, oid, serial, data in history:
... now += 1
... if action == 's':
... cache.invalidate(oid, serial)
... cache.store(oid, serial, None, data)
... else:
... v = cache.load(oid)
... if v is None:
... cache.store(oid, serial, None, data)
... cache.close()
>>> cache_run('cache', 2)
>>> import ZEO.scripts.cache_stats, ZEO.scripts.cache_simul
>>> def ctime(t):
... return time.asctime(time.gmtime(t-3600*4))
>>> ZEO.scripts.cache_stats.ctime = ctime
>>> ZEO.scripts.cache_simul.ctime = ctime
############################################################
Stats
>>> ZEO.scripts.cache_stats.main(['cache.trace'])
loads hits inv(h) writes hitrate
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11-14 180 1 2 197 0.6%
Jul 11 12:15-29 818 107 9 793 13.1%
Jul 11 12:30-44 818 213 22 687 26.0%
Jul 11 12:45-59 818 291 19 609 35.6%
Jul 11 13:00-14 818 295 36 605 36.1%
Jul 11 13:15-29 818 277 31 623 33.9%
Jul 11 13:30-44 819 276 29 624 33.7%
Jul 11 13:45-59 818 251 25 649 30.7%
Jul 11 14:00-14 818 295 27 605 36.1%
Jul 11 14:15-29 818 262 33 638 32.0%
Jul 11 14:30-44 818 297 32 603 36.3%
Jul 11 14:45-59 819 268 23 632 32.7%
Jul 11 15:00-14 818 291 30 609 35.6%
Jul 11 15:15-15 2 1 0 1 50.0%
<BLANKLINE>
Read 18,876 trace records (641,776 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (58.3%), average size 1108 bytes
Hit rate: 31.2% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
682 10 invalidate (miss)
318 1c invalidate (hit, saving non-current)
6,875 20 load (miss)
3,125 22 load (hit)
7,875 52 store (current, non-version)
>>> ZEO.scripts.cache_stats.main('-q cache.trace'.split())
loads hits inv(h) writes hitrate
<BLANKLINE>
Read 18,876 trace records (641,776 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (58.3%), average size 1108 bytes
Hit rate: 31.2% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
682 10 invalidate (miss)
318 1c invalidate (hit, saving non-current)
6,875 20 load (miss)
3,125 22 load (hit)
7,875 52 store (current, non-version)
>>> ZEO.scripts.cache_stats.main('-v cache.trace'.split())
... # doctest: +ELLIPSIS
loads hits inv(h) writes hitrate
Jul 11 12:11:41 00 '' 0000000000000000 0000000000000000 -
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11:42 10 1065 0000000000000002 0000000000000000 -
Jul 11 12:11:42 52 1065 0000000000000002 0000000000000000 - 245
Jul 11 12:11:43 20 947 0000000000000000 0000000000000000 -
Jul 11 12:11:43 52 947 0000000000000002 0000000000000000 - 602
Jul 11 12:11:44 20 124b 0000000000000000 0000000000000000 -
Jul 11 12:11:44 52 124b 0000000000000002 0000000000000000 - 1418
...
Jul 11 15:14:55 52 10cc 00000000000003e9 0000000000000000 - 1306
Jul 11 15:14:56 20 18a7 0000000000000000 0000000000000000 -
Jul 11 15:14:56 52 18a7 00000000000003e9 0000000000000000 - 1610
Jul 11 15:14:57 22 18b5 000000000000031d 0000000000000000 - 1636
Jul 11 15:14:58 20 b8a 0000000000000000 0000000000000000 -
Jul 11 15:14:58 52 b8a 00000000000003e9 0000000000000000 - 838
Jul 11 15:14:59 22 1085 0000000000000357 0000000000000000 - 217
Jul 11 15:00-14 818 291 30 609 35.6%
Jul 11 15:15:00 22 1072 000000000000037e 0000000000000000 - 204
Jul 11 15:15:01 20 16c5 0000000000000000 0000000000000000 -
Jul 11 15:15:01 52 16c5 00000000000003e9 0000000000000000 - 1712
Jul 11 15:15-15 2 1 0 1 50.0%
<BLANKLINE>
Read 18,876 trace records (641,776 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (58.3%), average size 1108 bytes
Hit rate: 31.2% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
682 10 invalidate (miss)
318 1c invalidate (hit, saving non-current)
6,875 20 load (miss)
3,125 22 load (hit)
7,875 52 store (current, non-version)
>>> ZEO.scripts.cache_stats.main('-h cache.trace'.split())
loads hits inv(h) writes hitrate
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11-14 180 1 2 197 0.6%
Jul 11 12:15-29 818 107 9 793 13.1%
Jul 11 12:30-44 818 213 22 687 26.0%
Jul 11 12:45-59 818 291 19 609 35.6%
Jul 11 13:00-14 818 295 36 605 36.1%
Jul 11 13:15-29 818 277 31 623 33.9%
Jul 11 13:30-44 819 276 29 624 33.7%
Jul 11 13:45-59 818 251 25 649 30.7%
Jul 11 14:00-14 818 295 27 605 36.1%
Jul 11 14:15-29 818 262 33 638 32.0%
Jul 11 14:30-44 818 297 32 603 36.3%
Jul 11 14:45-59 819 268 23 632 32.7%
Jul 11 15:00-14 818 291 30 609 35.6%
Jul 11 15:15-15 2 1 0 1 50.0%
<BLANKLINE>
Read 18,876 trace records (641,776 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (58.3%), average size 1108 bytes
Hit rate: 31.2% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
682 10 invalidate (miss)
318 1c invalidate (hit, saving non-current)
6,875 20 load (miss)
3,125 22 load (hit)
7,875 52 store (current, non-version)
<BLANKLINE>
Histogram of object load frequency
Unique oids: 4,585
Total loads: 10,000
loads objects %obj %load %cum
1 1,645 35.9% 16.4% 16.4%
2 1,465 32.0% 29.3% 45.8%
3 809 17.6% 24.3% 70.0%
4 430 9.4% 17.2% 87.2%
5 167 3.6% 8.3% 95.6%
6 49 1.1% 2.9% 98.5%
7 12 0.3% 0.8% 99.3%
8 7 0.2% 0.6% 99.9%
9 1 0.0% 0.1% 100.0%
>>> ZEO.scripts.cache_stats.main('-s cache.trace'.split())
... # doctest: +ELLIPSIS
loads hits inv(h) writes hitrate
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11-14 180 1 2 197 0.6%
Jul 11 12:15-29 818 107 9 793 13.1%
Jul 11 12:30-44 818 213 22 687 26.0%
Jul 11 12:45-59 818 291 19 609 35.6%
Jul 11 13:00-14 818 295 36 605 36.1%
Jul 11 13:15-29 818 277 31 623 33.9%
Jul 11 13:30-44 819 276 29 624 33.7%
Jul 11 13:45-59 818 251 25 649 30.7%
Jul 11 14:00-14 818 295 27 605 36.1%
Jul 11 14:15-29 818 262 33 638 32.0%
Jul 11 14:30-44 818 297 32 603 36.3%
Jul 11 14:45-59 819 268 23 632 32.7%
Jul 11 15:00-14 818 291 30 609 35.6%
Jul 11 15:15-15 2 1 0 1 50.0%
<BLANKLINE>
Read 18,876 trace records (641,776 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (58.3%), average size 1108 bytes
Hit rate: 31.2% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
682 10 invalidate (miss)
318 1c invalidate (hit, saving non-current)
6,875 20 load (miss)
3,125 22 load (hit)
7,875 52 store (current, non-version)
<BLANKLINE>
Histograms of object sizes
<BLANKLINE>
<BLANKLINE>
Unique sizes written: 1,782
size objs writes
200 5 5
201 4 4
202 4 4
203 1 1
204 1 1
205 6 6
206 8 8
...
1,995 1 2
1,996 2 2
1,997 1 1
1,998 2 2
1,999 2 4
2,000 1 1
>>> ZEO.scripts.cache_stats.main('-S cache.trace'.split())
loads hits inv(h) writes hitrate
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11-14 180 1 2 197 0.6%
Jul 11 12:15-29 818 107 9 793 13.1%
Jul 11 12:30-44 818 213 22 687 26.0%
Jul 11 12:45-59 818 291 19 609 35.6%
Jul 11 13:00-14 818 295 36 605 36.1%
Jul 11 13:15-29 818 277 31 623 33.9%
Jul 11 13:30-44 819 276 29 624 33.7%
Jul 11 13:45-59 818 251 25 649 30.7%
Jul 11 14:00-14 818 295 27 605 36.1%
Jul 11 14:15-29 818 262 33 638 32.0%
Jul 11 14:30-44 818 297 32 603 36.3%
Jul 11 14:45-59 819 268 23 632 32.7%
Jul 11 15:00-14 818 291 30 609 35.6%
Jul 11 15:15-15 2 1 0 1 50.0%
>>> ZEO.scripts.cache_stats.main('-X cache.trace'.split())
loads hits inv(h) writes hitrate
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11-14 180 1 2 197 0.6%
Jul 11 12:15-29 818 107 9 793 13.1%
Jul 11 12:30-44 818 213 22 687 26.0%
Jul 11 12:45-59 818 291 19 609 35.6%
Jul 11 13:00-14 818 295 36 605 36.1%
Jul 11 13:15-29 818 277 31 623 33.9%
Jul 11 13:30-44 819 276 29 624 33.7%
Jul 11 13:45-59 818 251 25 649 30.7%
Jul 11 14:00-14 818 295 27 605 36.1%
Jul 11 14:15-29 818 262 33 638 32.0%
Jul 11 14:30-44 818 297 32 603 36.3%
Jul 11 14:45-59 819 268 23 632 32.7%
Jul 11 15:00-14 818 291 30 609 35.6%
Jul 11 15:15-15 2 1 0 1 50.0%
<BLANKLINE>
Read 18,876 trace records (641,776 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (58.3%), average size 1108 bytes
Hit rate: 31.2% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
682 10 invalidate (miss)
318 1c invalidate (hit, saving non-current)
6,875 20 load (miss)
3,125 22 load (hit)
7,875 52 store (current, non-version)
>>> ZEO.scripts.cache_stats.main('-i 5 cache.trace'.split())
loads hits inv(h) writes hitrate
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11-14 180 1 2 197 0.6%
Jul 11 12:15-19 272 19 2 281 7.0%
Jul 11 12:20-24 273 35 5 265 12.8%
Jul 11 12:25-29 273 53 2 247 19.4%
Jul 11 12:30-34 272 60 8 240 22.1%
Jul 11 12:35-39 273 68 6 232 24.9%
Jul 11 12:40-44 273 85 8 215 31.1%
Jul 11 12:45-49 273 84 6 216 30.8%
Jul 11 12:50-54 272 104 9 196 38.2%
Jul 11 12:55-59 273 103 4 197 37.7%
Jul 11 13:00-04 273 92 12 208 33.7%
Jul 11 13:05-09 273 103 8 197 37.7%
Jul 11 13:10-14 272 100 16 200 36.8%
Jul 11 13:15-19 273 91 11 209 33.3%
Jul 11 13:20-24 273 96 9 204 35.2%
Jul 11 13:25-29 272 90 11 210 33.1%
Jul 11 13:30-34 273 82 14 218 30.0%
Jul 11 13:35-39 273 102 9 198 37.4%
Jul 11 13:40-44 273 92 6 208 33.7%
Jul 11 13:45-49 272 82 6 218 30.1%
Jul 11 13:50-54 273 83 8 217 30.4%
Jul 11 13:55-59 273 86 11 214 31.5%
Jul 11 14:00-04 273 95 11 205 34.8%
Jul 11 14:05-09 272 91 10 209 33.5%
Jul 11 14:10-14 273 109 6 191 39.9%
Jul 11 14:15-19 273 89 9 211 32.6%
Jul 11 14:20-24 272 84 16 216 30.9%
Jul 11 14:25-29 273 89 8 211 32.6%
Jul 11 14:30-34 273 97 12 203 35.5%
Jul 11 14:35-39 273 93 10 207 34.1%
Jul 11 14:40-44 272 107 10 193 39.3%
Jul 11 14:45-49 273 80 8 220 29.3%
Jul 11 14:50-54 273 100 8 200 36.6%
Jul 11 14:55-59 273 88 7 212 32.2%
Jul 11 15:00-04 272 99 8 201 36.4%
Jul 11 15:05-09 273 95 11 205 34.8%
Jul 11 15:10-14 273 97 11 203 35.5%
Jul 11 15:15-15 2 1 0 1 50.0%
<BLANKLINE>
Read 18,876 trace records (641,776 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (58.3%), average size 1108 bytes
Hit rate: 31.2% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
682 10 invalidate (miss)
318 1c invalidate (hit, saving non-current)
6,875 20 load (miss)
3,125 22 load (hit)
7,875 52 store (current, non-version)
>>> ZEO.scripts.cache_simul.main('-s 2 -i 5 cache.trace'.split())
CircularCacheSimulation, cache size 2,097,152 bytes
START TIME DUR. LOADS HITS INVALS WRITES HITRATE EVICTS INUSE
Jul 11 12:11 3:17 180 1 2 197 0.6% 0 10.7
Jul 11 12:15 4:59 272 19 2 281 7.0% 0 26.4
Jul 11 12:20 4:59 273 35 5 265 12.8% 0 40.4
Jul 11 12:25 4:59 273 53 2 247 19.4% 0 54.8
Jul 11 12:30 4:59 272 60 8 240 22.1% 0 67.1
Jul 11 12:35 4:59 273 68 6 232 24.9% 0 79.8
Jul 11 12:40 4:59 273 85 8 215 31.1% 0 91.4
Jul 11 12:45 4:59 273 84 6 216 30.8% 77 99.1
Jul 11 12:50 4:59 272 104 9 196 38.2% 196 98.9
Jul 11 12:55 4:59 273 104 4 196 38.1% 188 99.1
Jul 11 13:00 4:59 273 92 12 208 33.7% 213 99.3
Jul 11 13:05 4:59 273 103 8 197 37.7% 190 99.0
Jul 11 13:10 4:59 272 100 16 200 36.8% 203 99.2
Jul 11 13:15 4:59 273 91 11 209 33.3% 222 98.7
Jul 11 13:20 4:59 273 96 9 204 35.2% 210 99.2
Jul 11 13:25 4:59 272 89 11 211 32.7% 212 99.1
Jul 11 13:30 4:59 273 82 14 218 30.0% 220 99.1
Jul 11 13:35 4:59 273 101 9 199 37.0% 191 99.5
Jul 11 13:40 4:59 273 92 6 208 33.7% 214 99.4
Jul 11 13:45 4:59 272 80 6 220 29.4% 217 99.3
Jul 11 13:50 4:59 273 81 8 219 29.7% 214 99.2
Jul 11 13:55 4:59 273 86 11 214 31.5% 208 98.8
Jul 11 14:00 4:59 273 95 11 205 34.8% 188 99.3
Jul 11 14:05 4:59 272 93 10 207 34.2% 207 99.3
Jul 11 14:10 4:59 273 110 6 190 40.3% 198 98.8
Jul 11 14:15 4:59 273 91 9 209 33.3% 209 99.1
Jul 11 14:20 4:59 272 85 16 215 31.2% 210 99.3
Jul 11 14:25 4:59 273 89 8 211 32.6% 226 99.3
Jul 11 14:30 4:59 273 96 12 204 35.2% 214 99.3
Jul 11 14:35 4:59 273 90 10 210 33.0% 213 99.3
Jul 11 14:40 4:59 272 106 10 194 39.0% 196 98.8
Jul 11 14:45 4:59 273 80 8 220 29.3% 230 99.0
Jul 11 14:50 4:59 273 99 8 201 36.3% 202 99.0
Jul 11 14:55 4:59 273 87 8 213 31.9% 205 99.4
Jul 11 15:00 4:59 272 98 8 202 36.0% 211 99.3
Jul 11 15:05 4:59 273 93 11 207 34.1% 198 99.2
Jul 11 15:10 4:59 273 96 11 204 35.2% 184 99.2
Jul 11 15:15 1 2 1 0 1 50.0% 1 99.2
--------------------------------------------------------------------------
Jul 11 12:45 2:30:01 8184 2794 286 6208 34.1% 6067 99.2
>>> cache_run('cache4', 4)
>>> ZEO.scripts.cache_stats.main('cache4.trace'.split())
loads hits inv(h) writes hitrate
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11-14 180 1 2 197 0.6%
Jul 11 12:15-29 818 107 9 793 13.1%
Jul 11 12:30-44 818 213 22 687 26.0%
Jul 11 12:45-59 818 322 23 578 39.4%
Jul 11 13:00-14 818 381 43 519 46.6%
Jul 11 13:15-29 818 450 44 450 55.0%
Jul 11 13:30-44 819 503 47 397 61.4%
Jul 11 13:45-59 818 496 49 404 60.6%
Jul 11 14:00-14 818 516 48 384 63.1%
Jul 11 14:15-29 818 532 59 368 65.0%
Jul 11 14:30-44 818 516 51 384 63.1%
Jul 11 14:45-59 819 529 53 371 64.6%
Jul 11 15:00-14 818 515 49 385 63.0%
Jul 11 15:15-15 2 2 0 0 100.0%
<BLANKLINE>
Read 16,918 trace records (575,204 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (65.0%), average size 1104 bytes
Hit rate: 50.8% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
501 10 invalidate (miss)
499 1c invalidate (hit, saving non-current)
4,917 20 load (miss)
5,083 22 load (hit)
5,917 52 store (current, non-version)
>>> ZEO.scripts.cache_simul.main('-s 4 cache.trace'.split())
CircularCacheSimulation, cache size 4,194,304 bytes
START TIME DUR. LOADS HITS INVALS WRITES HITRATE EVICTS INUSE
Jul 11 12:11 3:17 180 1 2 197 0.6% 0 5.4
Jul 11 12:15 14:59 818 107 9 793 13.1% 0 27.4
Jul 11 12:30 14:59 818 213 22 687 26.0% 0 45.7
Jul 11 12:45 14:59 818 322 23 578 39.4% 0 61.4
Jul 11 13:00 14:59 818 381 43 519 46.6% 0 75.8
Jul 11 13:15 14:59 818 450 44 450 55.0% 0 88.2
Jul 11 13:30 14:59 819 503 47 397 61.4% 36 98.2
Jul 11 13:45 14:59 818 496 49 404 60.6% 388 98.5
Jul 11 14:00 14:59 818 515 48 385 63.0% 376 98.3
Jul 11 14:15 14:59 818 529 58 371 64.7% 391 98.1
Jul 11 14:30 14:59 818 511 51 389 62.5% 376 98.5
Jul 11 14:45 14:59 819 529 53 371 64.6% 410 97.9
Jul 11 15:00 14:59 818 512 49 388 62.6% 379 97.7
Jul 11 15:15 1 2 2 0 0 100.0% 0 97.7
--------------------------------------------------------------------------
Jul 11 13:30 1:45:01 5730 3597 355 2705 62.8% 2356 97.7
>>> cache_run('cache1', 1)
>>> ZEO.scripts.cache_stats.main('cache1.trace'.split())
loads hits inv(h) writes hitrate
Jul 11 12:11-11 0 0 0 0 n/a
Jul 11 12:11:41 ==================== Restart ====================
Jul 11 12:11-14 180 1 2 197 0.6%
Jul 11 12:15-29 818 107 9 793 13.1%
Jul 11 12:30-44 818 160 16 740 19.6%
Jul 11 12:45-59 818 158 8 742 19.3%
Jul 11 13:00-14 818 141 21 759 17.2%
Jul 11 13:15-29 818 128 17 772 15.6%
Jul 11 13:30-44 819 151 13 749 18.4%
Jul 11 13:45-59 818 120 17 780 14.7%
Jul 11 14:00-14 818 159 17 741 19.4%
Jul 11 14:15-29 818 141 13 759 17.2%
Jul 11 14:30-44 818 157 16 743 19.2%
Jul 11 14:45-59 819 133 13 767 16.2%
Jul 11 15:00-14 818 158 10 742 19.3%
Jul 11 15:15-15 2 1 0 1 50.0%
<BLANKLINE>
Read 20,286 trace records (689,716 bytes) in 0.0 seconds
Versions: 0 records used a version
First time: Sun Jul 11 12:11:41 2010
Last time: Sun Jul 11 15:15:01 2010
Duration: 11,000 seconds
Data recs: 11,000 (54.2%), average size 1105 bytes
Hit rate: 17.1% (load hits / loads)
<BLANKLINE>
Count Code Function (action)
1 00 _setup_trace (initialization)
828 10 invalidate (miss)
172 1c invalidate (hit, saving non-current)
8,285 20 load (miss)
1,715 22 load (hit)
9,285 52 store (current, non-version)
>>> ZEO.scripts.cache_simul.main('-s 1 cache.trace'.split())
CircularCacheSimulation, cache size 1,048,576 bytes
START TIME DUR. LOADS HITS INVALS WRITES HITRATE EVICTS INUSE
Jul 11 12:11 3:17 180 1 2 197 0.6% 0 21.5
Jul 11 12:15 14:59 818 107 9 793 13.1% 96 99.6
Jul 11 12:30 14:59 818 160 16 740 19.6% 724 99.6
Jul 11 12:45 14:59 818 158 8 742 19.3% 741 99.2
Jul 11 13:00 14:59 818 140 21 760 17.1% 771 99.5
Jul 11 13:15 14:59 818 125 17 775 15.3% 781 99.6
Jul 11 13:30 14:59 819 147 13 753 17.9% 748 99.5
Jul 11 13:45 14:59 818 120 17 780 14.7% 763 99.5
Jul 11 14:00 14:59 818 159 17 741 19.4% 728 99.4
Jul 11 14:15 14:59 818 141 13 759 17.2% 787 99.6
Jul 11 14:30 14:59 818 150 15 750 18.3% 755 99.2
Jul 11 14:45 14:59 819 132 13 768 16.1% 771 99.5
Jul 11 15:00 14:59 818 154 10 746 18.8% 723 99.2
Jul 11 15:15 1 2 1 0 1 50.0% 0 99.3
--------------------------------------------------------------------------
Jul 11 12:15 3:00:01 9820 1694 169 9108 17.3% 8388 99.3
Cleanup:
>>> del os.environ["ZEO_CACHE_TRACE"]
>>> time.time = timetime
>>> ZEO.scripts.cache_stats.ctime = time.ctime
>>> ZEO.scripts.cache_simul.ctime = time.ctime
"""
def cache_simul_properly_handles_load_miss_after_eviction_and_inval():
r"""
Set up evicted and then invalidated oid
>>> os.environ["ZEO_CACHE_TRACE"] = 'yes'
>>> cache = ZEO.cache.ClientCache('cache', 1<<21)
>>> cache.store(p64(1), p64(1), None, 'x')
>>> for i in range(10):
... cache.store(p64(2+i), p64(1), None, 'x'*(1<<19)) # Evict 1
>>> cache.store(p64(1), p64(1), None, 'x')
>>> cache.invalidate(p64(1), p64(2))
>>> cache.load(p64(1))
>>> cache.close()
Now try to do simulation:
>>> import ZEO.scripts.cache_simul
>>> ZEO.scripts.cache_simul.main('-s 1 cache.trace'.split())
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
CircularCacheSimulation, cache size 1,048,576 bytes
START TIME DUR. LOADS HITS INVALS WRITES HITRATE EVICTS INUSE
... 1 0 1 12 0.0% 10 50.0
--------------------------------------------------------------------------
... 1 0 1 12 0.0% 10 50.0
>>> del os.environ["ZEO_CACHE_TRACE"]
"""
def invalidations_with_current_tid_dont_wreck_cache():
"""
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache.store(p64(1), p64(1), None, 'data')
>>> import logging, sys
>>> handler = logging.StreamHandler(sys.stdout)
>>> logging.getLogger().addHandler(handler)
>>> old_level = logging.getLogger().getEffectiveLevel()
>>> logging.getLogger().setLevel(logging.WARNING)
>>> cache.invalidate(p64(1), p64(1))
Ignoring invalidation with same tid as current
>>> cache.close()
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache.close()
>>> logging.getLogger().removeHandler(handler)
>>> logging.getLogger().setLevel(old_level)
"""
def rename_bad_cache_file():
"""
An attempt to open a bad cache file will cause it to be dropped and recreated.
>>> open('cache', 'w').write('x'*100)
>>> import logging, sys
>>> handler = logging.StreamHandler(sys.stdout)
>>> logging.getLogger().addHandler(handler)
>>> old_level = logging.getLogger().getEffectiveLevel()
>>> logging.getLogger().setLevel(logging.WARNING)
>>> cache = ZEO.cache.ClientCache('cache', 1000) # doctest: +ELLIPSIS
Moving bad cache file to 'cache.bad'.
Traceback (most recent call last):
...
ValueError: unexpected magic number: 'xxxx'
>>> cache.store(p64(1), p64(1), None, 'data')
>>> cache.close()
>>> f = open('cache')
>>> f.seek(0, 2)
>>> print f.tell()
1000
>>> f.close()
>>> open('cache', 'w').write('x'*200)
>>> cache = ZEO.cache.ClientCache('cache', 1000) # doctest: +ELLIPSIS
Removing bad cache file: 'cache' (prev bad exists).
Traceback (most recent call last):
...
ValueError: unexpected magic number: 'xxxx'
>>> cache.store(p64(1), p64(1), None, 'data')
>>> cache.close()
>>> f = open('cache')
>>> f.seek(0, 2)
>>> print f.tell()
1000
>>> f.close()
>>> f = open('cache.bad')
>>> f.seek(0, 2)
>>> print f.tell()
100
>>> f.close()
>>> logging.getLogger().removeHandler(handler)
>>> logging.getLogger().setLevel(old_level)
"""
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(CacheTests))
suite.addTest(
doctest.DocTestSuite(
setUp=zope.testing.setupstack.setUpDirectory,
tearDown=zope.testing.setupstack.tearDown,
checker=zope.testing.renormalizing.RENormalizing([
(re.compile(r'31\.3%'), '31.2%'),
]),
)
)
return suite
src/ZEO/tests/zdoptions.test deleted 100644 → 0
View file @ 56c7dd30
Minimal test of Server Options Handling
=======================================
This is initially motivated by a desire to remove the requirement of
specifying a storage name when there is only one storage.
Storage Names
-------------
It is an error not to specify any storages:
>>> import StringIO, sys, ZEO.runzeo
>>> stderr = sys.stderr
>>> open('config', 'w').write("""
... <zeo>
... address 8100
... </zeo>
... """)
>>> sys.stderr = StringIO.StringIO()
>>> options = ZEO.runzeo.ZEOOptions()
>>> options.realize('-C config'.split())
Traceback (most recent call last):
...
SystemExit: 2
>>> print sys.stderr.getvalue() # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Error: not enough values for section type 'zodb.storage';
0 found, 1 required
...
But we can specify a storage without a name:
>>> open('config', 'w').write("""
... <zeo>
... address 8100
... </zeo>
... <mappingstorage>
... </mappingstorage>
... """)
>>> options = ZEO.runzeo.ZEOOptions()
>>> options.realize('-C config'.split())
>>> [storage.name for storage in options.storages]
['1']
We can't have multiple unnamed storages:
>>> sys.stderr = StringIO.StringIO()
>>> open('config', 'w').write("""
... <zeo>
... address 8100
... </zeo>
... <mappingstorage>
... </mappingstorage>
... <mappingstorage>
... </mappingstorage>
... """)
>>> options = ZEO.runzeo.ZEOOptions()
>>> options.realize('-C config'.split())
Traceback (most recent call last):
...
SystemExit: 2
>>> print sys.stderr.getvalue() # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Error: No more than one storage may be unnamed.
...
Or an unnamed storage and one named '1':
>>> sys.stderr = StringIO.StringIO()
>>> open('config', 'w').write("""
... <zeo>
... address 8100
... </zeo>
... <mappingstorage>
... </mappingstorage>
... <mappingstorage 1>
... </mappingstorage>
... """)
>>> options = ZEO.runzeo.ZEOOptions()
>>> options.realize('-C config'.split())
Traceback (most recent call last):
...
SystemExit: 2
>>> print sys.stderr.getvalue() # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Error: Can't have an unnamed storage and a storage named 1.
...
But we can have multiple storages:
>>> open('config', 'w').write("""
... <zeo>
... address 8100
... </zeo>
... <mappingstorage x>
... </mappingstorage>
... <mappingstorage y>
... </mappingstorage>
... """)
>>> options = ZEO.runzeo.ZEOOptions()
>>> options.realize('-C config'.split())
>>> [storage.name for storage in options.storages]
['x', 'y']
As long as the names are unique:
>>> sys.stderr = StringIO.StringIO()
>>> open('config', 'w').write("""
... <zeo>
... address 8100
... </zeo>
... <mappingstorage 1>
... </mappingstorage>
... <mappingstorage 1>
... </mappingstorage>
... """)
>>> options = ZEO.runzeo.ZEOOptions()
>>> options.realize('-C config'.split())
Traceback (most recent call last):
...
SystemExit: 2
>>> print sys.stderr.getvalue() # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Error: section names must not be re-used within the same container:'1'
...
.. Cleanup =====================================================
>>> sys.stderr = stderr
src/ZEO/tests/zeo-fan-out.test deleted 100644 → 0
View file @ 56c7dd30
ZEO Fan Out
===========
We should be able to set up ZEO servers with ZEO clients. Let's see
if we can make it work.
We'll use some helper functions. The first is a helper that starts
ZEO servers for us and another one that picks ports.
We'll start the first server:
>>> (_, port0), adminaddr0 = start_server(
... '<filestorage>\npath fs\nblob-dir blobs\n</filestorage>', keep=1)
Then we'll start 2 others that use this one:
>>> addr1, _ = start_server(
... '<zeoclient>\nserver %s\nblob-dir b1\n</zeoclient>' % port0)
>>> addr2, _ = start_server(
... '<zeoclient>\nserver %s\nblob-dir b2\n</zeoclient>' % port0)
Now, let's create some client storages that connect to these:
>>> import os, ZEO, ZODB.blob, ZODB.POSException, transaction
>>> db0 = ZEO.DB(port0, blob_dir='cb0')
>>> db1 = ZEO.DB(addr1, blob_dir='cb1')
>>> tm1 = transaction.TransactionManager()
>>> c1 = db1.open(transaction_manager=tm1)
>>> r1 = c1.root()
>>> r1
{}
>>> db2 = ZEO.DB(addr2, blob_dir='cb2')
>>> tm2 = transaction.TransactionManager()
>>> c2 = db2.open(transaction_manager=tm2)
>>> r2 = c2.root()
>>> r2
{}
If we update c1, we'll eventually see the change in c2:
>>> import persistent.mapping
>>> r1[1] = persistent.mapping.PersistentMapping()
>>> r1[1].v = 1000
>>> r1[2] = persistent.mapping.PersistentMapping()
>>> r1[2].v = -1000
>>> r1[3] = ZODB.blob.Blob('x'*4111222)
>>> for i in range(1000, 2000):
... r1[i] = persistent.mapping.PersistentMapping()
... r1[i].v = 0
>>> tm1.commit()
>>> blob_id = r1[3]._p_oid, r1[1]._p_serial
>>> import time
>>> for i in range(100):
... t = tm2.begin()
... if 1 in r2:
... break
... time.sleep(0.01)
>>> tm2.abort()
>>> r2[1].v
1000
>>> r2[2].v
-1000
Now, let's see if we can break it. :)
>>> def f():
... c = db1.open(transaction.TransactionManager())
... r = c.root()
... i = 0
... while i < 100:
... r[1].v -= 1
... r[2].v += 1
... try:
... c.transaction_manager.commit()
... i += 1
... except ZODB.POSException.ConflictError:
... c.transaction_manager.abort()
... c.close()
>>> import threading
>>> threadf = threading.Thread(target=f)
>>> threadg = threading.Thread(target=f)
>>> threadf.start()
>>> threadg.start()
>>> s2 = db2.storage
>>> start_time = time.time()
>>> while time.time() - start_time < 999:
... t = tm2.begin()
... if r2[1].v + r2[2].v:
... print 'oops', r2[1], r2[2]
... if r2[1].v == 800:
... break # we caught up
... path = s2.fshelper.getBlobFilename(*blob_id)
... if os.path.exists(path):
... ZODB.blob.remove_committed(path)
... s2._server.sendBlob(*blob_id)
... else: print 'Dang'
>>> threadf.join()
>>> threadg.join()
If we shutdown and restart the source server, the variables will be
invalidated:
>>> stop_server(adminaddr0)
>>> _ = start_server('<filestorage 1>\npath fs\n</filestorage>\n',
... port=port0)
>>> for i in range(1000):
... c1.sync()
... c2.sync()
... if (
... (r1[1]._p_changed is None)
... and
... (r1[2]._p_changed is None)
... and
... (r2[1]._p_changed is None)
... and
... (r2[2]._p_changed is None)
... ):
... print 'Cool'
... break
... time.sleep(0.01)
... else:
... print 'Dang'
Cool
Cleanup:
>>> db0.close()
>>> db1.close()
>>> db2.close()
src/ZEO/tests/zeo_blob_cache.test deleted 100644 → 0
View file @ 56c7dd30
ZEO caching of blob data
========================
ZEO supports 2 modes for providing clients access to blob data:
shared
Blob data are shared via a network file system. The client shares
a common blob directory with the server.
non-shared
Blob data are loaded from the storage server and cached locally.
A maximum size for the blob data can be set and data are removed
when the size is exceeded.
In this test, we'll demonstrate that blobs data are removed from a ZEO
cache when the amount of data stored exceeds a given limit.
Let's start by setting up some data:
>>> addr, _ = start_server(blob_dir='server-blobs')
We'll also create a client.
>>> import ZEO
>>> db = ZEO.DB(addr, blob_dir='blobs', blob_cache_size=3000)
Here, we passed a blob_cache_size parameter, which specifies a target
blob cache size. This is not a hard limit, but rather a target. It
defaults to a very large value. We also passed a blob_cache_size_check
option. The blob_cache_size_check option specifies the number of
bytes, as a percent of the target that can be written or downloaded
from the server before the cache size is checked. The
blob_cache_size_check option defaults to 100. We passed 10, to check
after writing 10% of the target size.
.. We're going to wait for any threads we started to finish, so...
>>> import threading
>>> old_threads = list(threading.enumerate())
We want to check for name collections in the blob cache dir. We'll try
to provoke name collections by reducing the number of cache directory
subdirectories.
>>> import ZEO.ClientStorage
>>> orig_blob_cache_layout_size = ZEO.ClientStorage.BlobCacheLayout.size
>>> ZEO.ClientStorage.BlobCacheLayout.size = 11
Now, let's write some data:
>>> import ZODB.blob, transaction, time
>>> conn = db.open()
>>> for i in range(1, 101):
... conn.root()[i] = ZODB.blob.Blob()
... conn.root()[i].open('w').write(chr(i)*100)
>>> transaction.commit()
We've committed 10000 bytes of data, but our target size is 3000. We
expect to have not much more than the target size in the cache blob
directory.
>>> import os
>>> def cache_size(d):
... size = 0
... for base, dirs, files in os.walk(d):
... for f in files:
... if f.endswith('.blob'):
... try:
... size += os.stat(os.path.join(base, f)).st_size
... except OSError:
... if os.path.exists(os.path.join(base, f)):
... raise
... return size
>>> def check():
... return cache_size('blobs') < 5000
>>> def onfail():
... return cache_size('blobs')
>>> from ZEO.tests.forker import wait_until
>>> wait_until("size is reduced", check, 99, onfail)
If we read all of the blobs, data will be downloaded again, as
necessary, but the cache size will remain not much bigger than the
target:
>>> for i in range(1, 101):
... data = conn.root()[i].open().read()
... if data != chr(i)*100:
... print 'bad data', `chr(i)`, `data`
>>> wait_until("size is reduced", check, 99, onfail)
>>> for i in range(1, 101):
... data = conn.root()[i].open().read()
... if data != chr(i)*100:
... print 'bad data', `chr(i)`, `data`
>>> for i in range(1, 101):
... data = conn.root()[i].open('c').read()
... if data != chr(i)*100:
... print 'bad data', `chr(i)`, `data`
>>> wait_until("size is reduced", check, 99, onfail)
Now let see if we can stress things a bit. We'll create many clients
and get them to pound on the blobs all at once to see if we can
provoke problems:
>>> import threading, random
>>> def run():
... db = ZEO.DB(addr, blob_dir='blobs', blob_cache_size=4000)
... conn = db.open()
... for i in range(300):
... time.sleep(0)
... i = random.randint(1, 100)
... data = conn.root()[i].open().read()
... if data != chr(i)*100:
... print 'bad data', `chr(i)`, `data`
... i = random.randint(1, 100)
... data = conn.root()[i].open('c').read()
... if data != chr(i)*100:
... print 'bad data', `chr(i)`, `data`
... db.close()
>>> threads = [threading.Thread(target=run) for i in range(10)]
>>> for thread in threads:
... thread.setDaemon(True)
>>> for thread in threads:
... thread.start()
>>> for thread in threads:
... thread.join(99)
... if thread.isAlive():
... print "Can't join thread."
>>> wait_until("size is reduced", check, 99, onfail)
.. cleanup
>>> for thread in threading.enumerate():
... if thread not in old_threads:
... thread.join(33)
>>> db.close()
>>> ZEO.ClientStorage.BlobCacheLayout.size = orig_blob_cache_layout_size
src/ZEO/tests/zeoserver.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Helper file used to launch a ZEO server cross platform"""
import asyncore
import errno
import getopt
import logging
import os
import signal
import socket
import sys
import threading
import time
import ZEO.runzeo
import ZEO.zrpc.connection
def cleanup(storage):
# FileStorage and the Berkeley storages have this method, which deletes
# all files and directories used by the storage. This prevents @-files
# from clogging up /tmp
try:
storage.cleanup()
except AttributeError:
pass
logger = logging.getLogger('ZEO.tests.zeoserver')
def log(label, msg, *args):
message = "(%s) %s" % (label, msg)
logger.debug(message, *args)
class ZEOTestServer(asyncore.dispatcher):
"""A server for killing the whole process at the end of a test.
The first time we connect to this server, we write an ack character down
the socket. The other end should block on a recv() of the socket so it
can guarantee the server has started up before continuing on.
The second connect to the port immediately exits the process, via
os._exit(), without writing data on the socket. It does close and clean
up the storage first. The other end will get the empty string from its
recv() which will be enough to tell it that the server has exited.
I think this should prevent us from ever getting a legitimate addr-in-use
error.
"""
__super_init = asyncore.dispatcher.__init__
def __init__(self, addr, server, keep):
self.__super_init()
self._server = server
self._sockets = [self]
self._keep = keep
# Count down to zero, the number of connects
self._count = 1
self._label ='%d @ %s' % (os.getpid(), addr)
if isinstance(addr, str):
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
else:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
# Some ZEO tests attempt a quick start of the server using the same
# port so we have to set the reuse flag.
self.set_reuse_addr()
try:
self.bind(addr)
except:
# We really want to see these exceptions
import traceback
traceback.print_exc()
raise
self.listen(5)
self.log('bound and listening')
def log(self, msg, *args):
log(self._label, msg, *args)
def handle_accept(self):
sock, addr = self.accept()
self.log('in handle_accept()')
# When we're done with everything, close the storage. Do not write
# the ack character until the storage is finished closing.
if self._count <= 0:
self.log('closing the storage')
self._server.close()
if not self._keep:
for storage in self._server.storages.values():
cleanup(storage)
self.log('exiting')
# Close all the other sockets so that we don't have to wait
# for os._exit() to get to it before starting the next
# server process.
for s in self._sockets:
s.close()
# Now explicitly close the socket returned from accept(),
# since it didn't go through the wrapper.
sock.close()
os._exit(0)
self.log('continuing')
sock.send('X')
self._count -= 1
def register_socket(self, sock):
# Register a socket to be closed when server shutsdown.
self._sockets.append(sock)
class Suicide(threading.Thread):
def __init__(self, addr):
threading.Thread.__init__(self)
self._adminaddr = addr
def run(self):
# If this process doesn't exit in 330 seconds, commit suicide.
# The client threads in the ConcurrentUpdate tests will run for
# as long as 300 seconds. Set this timeout to 330 to minimize
# chance that the server gives up before the clients.
time.sleep(999)
log(str(os.getpid()), "suicide thread invoking shutdown")
# If the server hasn't shut down yet, the client may not be
# able to connect to it. If so, try to kill the process to
# force it to shutdown.
if hasattr(os, "kill"):
os.kill(pid, signal.SIGTERM)
time.sleep(5)
os.kill(pid, signal.SIGKILL)
else:
from ZEO.tests.forker import shutdown_zeo_server
# Nott: If the -k option was given to zeoserver, then the
# process will go away but the temp files won't get
# cleaned up.
shutdown_zeo_server(self._adminaddr)
def main():
global pid
pid = os.getpid()
label = str(pid)
log(label, "starting")
# We don't do much sanity checking of the arguments, since if we get it
# wrong, it's a bug in the test suite.
keep = 0
configfile = None
suicide = True
# Parse the arguments and let getopt.error percolate
opts, args = getopt.getopt(sys.argv[1:], 'dkSC:v:')
for opt, arg in opts:
if opt == '-k':
keep = 1
if opt == '-d':
ZEO.zrpc.connection.debug_zrpc = True
elif opt == '-C':
configfile = arg
elif opt == '-S':
suicide = False
elif opt == '-v':
ZEO.zrpc.connection.Connection.current_protocol = arg
zo = ZEO.runzeo.ZEOOptions()
zo.realize(["-C", configfile])
addr = zo.address
if zo.auth_protocol == "plaintext":
__import__('ZEO.tests.auth_plaintext')
if isinstance(addr, tuple):
test_addr = addr[0], addr[1]+1
else:
test_addr = addr + '-test'
log(label, 'creating the storage server')
storage = zo.storages[0].open()
mon_addr = None
if zo.monitor_address:
mon_addr = zo.monitor_address
server = ZEO.runzeo.create_server({"1": storage}, zo)
try:
log(label, 'creating the test server, keep: %s', keep)
t = ZEOTestServer(test_addr, server, keep)
except socket.error, e:
if e[0] != errno.EADDRINUSE:
raise
log(label, 'addr in use, closing and exiting')
storage.close()
cleanup(storage)
sys.exit(2)
t.register_socket(server.dispatcher)
if suicide:
# Create daemon suicide thread
d = Suicide(test_addr)
d.setDaemon(1)
d.start()
# Loop for socket events
log(label, 'entering asyncore loop')
server.start_thread()
asyncore.loop()
if __name__ == '__main__':
import warnings
warnings.simplefilter('ignore')
main()
src/ZEO/util.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Utilities for setting up the server environment."""
import os
def parentdir(p, n=1):
"""Return the ancestor of p from n levels up."""
d = p
while n:
d = os.path.dirname(d)
if not d or d == '.':
d = os.getcwd()
n -= 1
return d
class Environment:
"""Determine location of the Data.fs & ZEO_SERVER.pid files.
Pass the argv[0] used to start ZEO to the constructor.
Use the zeo_pid and fs attributes to get the filenames.
"""
def __init__(self, argv0):
v = os.environ.get("INSTANCE_HOME")
if v is None:
# looking for a Zope/var directory assuming that this code
# is installed in Zope/lib/python/ZEO
p = parentdir(argv0, 4)
if os.path.isdir(os.path.join(p, "var")):
v = p
else:
v = os.getcwd()
self.home = v
self.var = os.path.join(v, "var")
if not os.path.isdir(self.var):
self.var = self.home
pid = os.environ.get("ZEO_SERVER_PID")
if pid is None:
pid = os.path.join(self.var, "ZEO_SERVER.pid")
self.zeo_pid = pid
self.fs = os.path.join(self.var, "Data.fs")
src/ZEO/zeoctl.py deleted 100644 → 0
View file @ 56c7dd30
#!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2005 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Wrapper script for zdctl.py that causes it to use the ZEO schema."""
import os
import ZEO
import zdaemon.zdctl
# Main program
def main(args=None):
options = zdaemon.zdctl.ZDCtlOptions()
options.schemadir = os.path.dirname(ZEO.__file__)
options.schemafile = "zeoctl.xml"
zdaemon.zdctl.main(args, options)
if __name__ == "__main__":
main()
src/ZEO/zeoctl.xml deleted 100644 → 0
View file @ 56c7dd30
<schema>
<description>
This schema describes the configuration of the ZEO storage server
controller. It differs from the schema for the storage server
only in that the "runner" section is required.
</description>
<!-- Use the storage types defined by ZODB. -->
<import package="ZODB"/>
<!-- Use the ZEO server information structure. -->
<import package="ZEO"/>
<import package="ZConfig.components.logger"/>
<!-- runner control -->
<import package="zdaemon"/>
<section type="zeo" name="*" required="yes" attribute="zeo" />
<section type="runner" name="*" required="yes" attribute="runner" />
<multisection name="+" type="ZODB.storage"
attribute="storages"
required="yes" />
<section name="*" type="eventlog" attribute="eventlog" required="no" />
</schema>
src/ZEO/zeopasswd.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Update a user's authentication tokens for a ZEO server.
usage: python zeopasswd.py [options] username [password]
Specify either a configuration file:
-C/--configuration -- ZConfig configuration file
or the individual options:
-f/--filename -- authentication database filename
-p/--protocol -- authentication protocol name
-r/--realm -- authentication database realm
Additional options:
-d/--delete -- delete user instead of updating password
"""
import getopt
import getpass
import sys
import os
import ZConfig
import ZEO
def usage(msg):
print __doc__
print msg
sys.exit(2)
def options(args):
"""Password-specific options loaded from regular ZEO config file."""
try:
opts, args = getopt.getopt(args, "dr:p:f:C:", ["configure=",
"protocol=",
"filename=",
"realm"])
except getopt.error, msg:
usage(msg)
config = None
delete = 0
auth_protocol = None
auth_db = ""
auth_realm = None
for k, v in opts:
if k == '-C' or k == '--configure':
schemafile = os.path.join(os.path.dirname(ZEO.__file__),
"schema.xml")
schema = ZConfig.loadSchema(schemafile)
config, nil = ZConfig.loadConfig(schema, v)
if k == '-d' or k == '--delete':
delete = 1
if k == '-p' or k == '--protocol':
auth_protocol = v
if k == '-f' or k == '--filename':
auth_db = v
if k == '-r' or k == '--realm':
auth_realm = v
if config is not None:
if auth_protocol or auth_db:
usage("Error: Conflicting options; use either -C *or* -p and -f")
auth_protocol = config.zeo.authentication_protocol
auth_db = config.zeo.authentication_database
auth_realm = config.zeo.authentication_realm
elif not (auth_protocol and auth_db):
usage("Error: Must specifiy configuration file or protocol and database")
password = None
if delete:
if not args:
usage("Error: Must specify a username to delete")
elif len(args) > 1:
usage("Error: Too many arguments")
username = args[0]
else:
if not args:
usage("Error: Must specify a username")
elif len(args) > 2:
usage("Error: Too many arguments")
elif len(args) == 1:
username = args[0]
else:
username, password = args
return auth_protocol, auth_db, auth_realm, delete, username, password
def main(args=None, dbclass=None):
if args is None:
args = sys.argv[1:]
p, auth_db, auth_realm, delete, username, password = options(args)
if p is None:
usage("Error: configuration does not specify auth protocol")
if p == "digest":
from ZEO.auth.auth_digest import DigestDatabase as Database
elif p == "srp":
from ZEO.auth.auth_srp import SRPDatabase as Database
elif dbclass:
# dbclass is used for testing tests.auth_plaintext, see testAuth.py
Database = dbclass
else:
raise ValueError("Unknown database type %r" % p)
if auth_db is None:
usage("Error: configuration does not specify auth database")
db = Database(auth_db, auth_realm)
if delete:
db.del_user(username)
else:
if password is None:
password = getpass.getpass("Enter password: ")
db.add_user(username, password)
db.save()
src/ZEO/zrpc/__init__.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# zrpc is a package with the following modules
# client -- manages connection creation to remote server
# connection -- object dispatcher
# log -- logging helper
# error -- exceptions raised by zrpc
# marshal -- internal, handles basic protocol issues
# server -- manages incoming connections from remote clients
# smac -- sized message async connections
# trigger -- medusa's trigger
# zrpc is not an advertised subpackage of ZEO; its interfaces are internal
src/ZEO/zrpc/_hmac.py deleted 100644 → 0
View file @ 56c7dd30
# This file is a slightly modified copy of Python 2.3's Lib/hmac.py.
# This file is under the Python Software Foundation (PSF) license.
"""HMAC (Keyed-Hashing for Message Authentication) Python module.
Implements the HMAC algorithm as described by RFC 2104.
"""
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
"""
return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
# The size of the digests returned by HMAC depends on the underlying
# hashing module used.
digest_size = None
class HMAC:
"""RFC2104 HMAC class.
This supports the API for Cryptographic Hash Functions (PEP 247).
"""
def __init__(self, key, msg = None, digestmod = None):
"""Create a new HMAC object.
key: key for the keyed hash object.
msg: Initial input for the hash, if provided.
digestmod: A module supporting PEP 247. Defaults to the md5 module.
"""
if digestmod is None:
import md5
digestmod = md5
self.digestmod = digestmod
self.outer = digestmod.new()
self.inner = digestmod.new()
# Python 2.1 and 2.2 differ about the correct spelling
try:
self.digest_size = digestmod.digestsize
except AttributeError:
self.digest_size = digestmod.digest_size
blocksize = 64
ipad = "\x36" * blocksize
opad = "\x5C" * blocksize
if len(key) > blocksize:
key = digestmod.new(key).digest()
key = key + chr(0) * (blocksize - len(key))
self.outer.update(_strxor(key, opad))
self.inner.update(_strxor(key, ipad))
if msg is not None:
self.update(msg)
## def clear(self):
## raise NotImplementedError("clear() method not available in HMAC.")
def update(self, msg):
"""Update this hashing object with the string msg.
"""
self.inner.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
An update to this copy won't affect the original object.
"""
other = HMAC("")
other.digestmod = self.digestmod
other.inner = self.inner.copy()
other.outer = self.outer.copy()
return other
def digest(self):
"""Return the hash value of this hashing object.
This returns a string containing 8-bit data. The object is
not altered in any way by this function; you can continue
updating the object after calling this function.
"""
h = self.outer.copy()
h.update(self.inner.digest())
return h.digest()
def hexdigest(self):
"""Like digest(), but returns a string of hexadecimal digits instead.
"""
return "".join([hex(ord(x))[2:].zfill(2)
for x in tuple(self.digest())])
def new(key, msg = None, digestmod = None):
"""Create a new hashing object and return it.
key: The starting key for the hash.
msg: if available, will immediately be hashed into the object's starting
state.
You can now feed arbitrary strings into the object using its update()
method, and can ask for the hash value at any time by calling its digest()
method.
"""
return HMAC(key, msg, digestmod)
src/ZEO/zrpc/client.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import errno
import logging
import select
import socket
import sys
import threading
import time
import ZEO.zrpc.trigger
from ZEO.zrpc.connection import ManagedClientConnection
from ZEO.zrpc.log import log
from ZEO.zrpc.error import DisconnectedError
from ZODB.POSException import ReadOnlyError
from ZODB.loglevels import BLATHER
def client_timeout():
return 30.0
def client_loop(map):
read = asyncore.read
write = asyncore.write
_exception = asyncore._exception
while map:
try:
# The next two lines intentionally don't use
# iterators. Other threads can close dispatchers, causeing
# the socket map to shrink.
r = e = map.keys()
w = [fd for (fd, obj) in map.items() if obj.writable()]
try:
r, w, e = select.select(r, w, e, client_timeout())
except select.error, err:
if err[0] != errno.EINTR:
if err[0] == errno.EBADF:
# If a connection is closed while we are
# calling select on it, we can get a bad
# file-descriptor error. We'll check for this
# case by looking for entries in r and w that
# are not in the socket map.
if [fd for fd in r if fd not in map]:
continue
if [fd for fd in w if fd not in map]:
continue
raise
else:
continue
if not map:
break
if not (r or w or e):
# The line intentionally doesn't use iterators. Other
# threads can close dispatchers, causeing the socket
# map to shrink.
for obj in map.values():
if isinstance(obj, ManagedClientConnection):
# Send a heartbeat message as a reply to a
# non-existent message id.
try:
obj.send_reply(-1, None)
except DisconnectedError:
pass
continue
for fd in r:
obj = map.get(fd)
if obj is None:
continue
read(obj)
for fd in w:
obj = map.get(fd)
if obj is None:
continue
write(obj)
for fd in e:
obj = map.get(fd)
if obj is None:
continue
_exception(obj)
except:
if map:
try:
logging.getLogger(__name__+'.client_loop').critical(
'A ZEO client loop failed.',
exc_info=sys.exc_info())
except:
pass
for fd, obj in map.items():
if not hasattr(obj, 'mgr'):
continue
try:
obj.mgr.client.close()
except:
map.pop(fd, None)
try:
logging.getLogger(__name__+'.client_loop'
).critical(
"Couldn't close a dispatcher.",
exc_info=sys.exc_info())
except:
pass
class ConnectionManager(object):
"""Keeps a connection up over time"""
sync_wait = 30
def __init__(self, addrs, client, tmin=1, tmax=180):
self.client = client
self._start_asyncore_loop()
self.addrlist = self._parse_addrs(addrs)
self.tmin = min(tmin, tmax)
self.tmax = tmax
self.cond = threading.Condition(threading.Lock())
self.connection = None # Protected by self.cond
self.closed = 0
# If thread is not None, then there is a helper thread
# attempting to connect.
self.thread = None # Protected by self.cond
def new_addrs(self, addrs):
self.addrlist = self._parse_addrs(addrs)
def _start_asyncore_loop(self):
self.map = {}
self.trigger = ZEO.zrpc.trigger.trigger(self.map)
self.loop_thread = threading.Thread(
name="%s zeo client networking thread" % self.client.__name__,
target=client_loop, args=(self.map,))
self.loop_thread.setDaemon(True)
self.loop_thread.start()
def __repr__(self):
return "<%s for %s>" % (self.__class__.__name__, self.addrlist)
def _parse_addrs(self, addrs):
# Return a list of (addr_type, addr) pairs.
# For backwards compatibility (and simplicity?) the
# constructor accepts a single address in the addrs argument --
# a string for a Unix domain socket or a 2-tuple with a
# hostname and port. It can also accept a list of such addresses.
addr_type = self._guess_type(addrs)
if addr_type is not None:
return [(addr_type, addrs)]
else:
addrlist = []
for addr in addrs:
addr_type = self._guess_type(addr)
if addr_type is None:
raise ValueError("unknown address in list: %s" % repr(addr))
addrlist.append((addr_type, addr))
return addrlist
def _guess_type(self, addr):
if isinstance(addr, str):
return socket.AF_UNIX
if (len(addr) == 2
and isinstance(addr[0], str)
and isinstance(addr[1], int)):
return socket.AF_INET # also denotes IPv6
# not anything I know about
return None
def close(self):
"""Prevent ConnectionManager from opening new connections"""
self.closed = 1
self.cond.acquire()
try:
t = self.thread
self.thread = None
finally:
self.cond.release()
if t is not None:
log("CM.close(): stopping and joining thread")
t.stop()
t.join(30)
if t.isAlive():
log("CM.close(): self.thread.join() timed out",
level=logging.WARNING)
for fd, obj in self.map.items():
if obj is not self.trigger:
try:
obj.close()
except:
logging.getLogger(__name__+'.'+self.__class__.__name__
).critical(
"Couldn't close a dispatcher.",
exc_info=sys.exc_info())
self.map.clear()
self.trigger.pull_trigger()
try:
self.loop_thread.join(9)
except RuntimeError:
pass # we are the thread :)
self.trigger.close()
def attempt_connect(self):
"""Attempt a connection to the server without blocking too long.
There isn't a crisp definition for too long. When a
ClientStorage is created, it attempts to connect to the
server. If the server isn't immediately available, it can
operate from the cache. This method will start the background
connection thread and wait a little while to see if it
finishes quickly.
"""
# Will a single attempt take too long?
# Answer: it depends -- normally, you'll connect or get a
# connection refused error very quickly. Packet-eating
# firewalls and other mishaps may cause the connect to take a
# long time to time out though. It's also possible that you
# connect quickly to a slow server, and the attempt includes
# at least one roundtrip to the server (the register() call).
# But that's as fast as you can expect it to be.
self.connect()
self.cond.acquire()
try:
t = self.thread
conn = self.connection
finally:
self.cond.release()
if t is not None and conn is None:
event = t.one_attempt
event.wait()
self.cond.acquire()
try:
conn = self.connection
finally:
self.cond.release()
return conn is not None
def connect(self, sync=0):
self.cond.acquire()
try:
if self.connection is not None:
return
t = self.thread
if t is None:
log("CM.connect(): starting ConnectThread")
self.thread = t = ConnectThread(self, self.client)
t.setDaemon(1)
t.start()
if sync:
while self.connection is None and t.isAlive():
self.cond.wait(self.sync_wait)
if self.connection is None:
log("CM.connect(sync=1): still waiting...")
assert self.connection is not None
finally:
self.cond.release()
def connect_done(self, conn, preferred):
# Called by ConnectWrapper.notify_client() after notifying the client
log("CM.connect_done(preferred=%s)" % preferred)
self.cond.acquire()
try:
self.connection = conn
if preferred:
self.thread = None
self.cond.notifyAll() # Wake up connect(sync=1)
finally:
self.cond.release()
def close_conn(self, conn):
# Called by the connection when it is closed
self.cond.acquire()
try:
if conn is not self.connection:
# Closing a non-current connection
log("CM.close_conn() non-current", level=BLATHER)
return
log("CM.close_conn()")
self.connection = None
finally:
self.cond.release()
self.client.notifyDisconnected()
if not self.closed:
self.connect()
def is_connected(self):
self.cond.acquire()
try:
return self.connection is not None
finally:
self.cond.release()
# When trying to do a connect on a non-blocking socket, some outcomes
# are expected. Set _CONNECT_IN_PROGRESS to the errno value(s) expected
# when an initial connect can't complete immediately. Set _CONNECT_OK
# to the errno value(s) expected if the connect succeeds *or* if it's
# already connected (our code can attempt redundant connects).
if hasattr(errno, "WSAEWOULDBLOCK"): # Windows
# Caution: The official Winsock docs claim that WSAEALREADY should be
# treated as yet another "in progress" indicator, but we've never
# seen this.
_CONNECT_IN_PROGRESS = (errno.WSAEWOULDBLOCK,)
# Win98: WSAEISCONN; Win2K: WSAEINVAL
_CONNECT_OK = (0, errno.WSAEISCONN, errno.WSAEINVAL)
else: # Unix
_CONNECT_IN_PROGRESS = (errno.EINPROGRESS,)
_CONNECT_OK = (0, errno.EISCONN)
class ConnectThread(threading.Thread):
"""Thread that tries to connect to server given one or more addresses.
The thread is passed a ConnectionManager and the manager's client
as arguments. It calls testConnection() on the client when a
socket connects; that should return 1 or 0 indicating whether this
is a preferred or a fallback connection. It may also raise an
exception, in which case the connection is abandoned.
The thread will continue to run, attempting connections, until a
preferred connection is seen and successfully handed over to the
manager and client.
As soon as testConnection() finds a preferred connection, or after
all sockets have been tried and at least one fallback connection
has been seen, notifyConnected(connection) is called on the client
and connect_done() on the manager. If this was a preferred
connection, the thread then exits; otherwise, it keeps trying
until it gets a preferred connection, and then reconnects the
client using that connection.
"""
__super_init = threading.Thread.__init__
# We don't expect clients to call any methods of this Thread other
# than close() and those defined by the Thread API.
def __init__(self, mgr, client):
self.__super_init(name="Connect(%s)" % mgr.addrlist)
self.mgr = mgr
self.client = client
self.stopped = 0
self.one_attempt = threading.Event()
# A ConnectThread keeps track of whether it has finished a
# call to try_connecting(). This allows the ConnectionManager
# to make an attempt to connect right away, but not block for
# too long if the server isn't immediately available.
def stop(self):
self.stopped = 1
def run(self):
delay = self.mgr.tmin
success = 0
# Don't wait too long the first time.
# TODO: make timeout configurable?
attempt_timeout = 5
while not self.stopped:
success = self.try_connecting(attempt_timeout)
if not self.one_attempt.isSet():
self.one_attempt.set()
attempt_timeout = 75
if success > 0:
break
time.sleep(delay)
if self.mgr.is_connected():
log("CT: still trying to replace fallback connection",
level=logging.INFO)
delay = min(delay*2, self.mgr.tmax)
log("CT: exiting thread: %s" % self.getName())
def try_connecting(self, timeout):
"""Try connecting to all self.mgr.addrlist addresses.
Return 1 if a preferred connection was found; 0 if no
connection was found; and -1 if a fallback connection was
found.
If no connection is found within timeout seconds, return 0.
"""
log("CT: attempting to connect on %d sockets" % len(self.mgr.addrlist))
deadline = time.time() + timeout
wrappers = self._create_wrappers()
for wrap in wrappers.keys():
if wrap.state == "notified":
return 1
try:
if time.time() > deadline:
return 0
r = self._connect_wrappers(wrappers, deadline)
if r is not None:
return r
if time.time() > deadline:
return 0
r = self._fallback_wrappers(wrappers, deadline)
if r is not None:
return r
# Alas, no luck.
assert not wrappers
finally:
for wrap in wrappers.keys():
wrap.close()
del wrappers
return 0
def _expand_addrlist(self):
for domain, addr in self.mgr.addrlist:
# AF_INET really means either IPv4 or IPv6, possibly
# indirected by DNS. By design, DNS lookup is deferred
# until connections get established, so that DNS
# reconfiguration can affect failover
if domain == socket.AF_INET:
host, port = addr
for (family, socktype, proto, cannoname, sockaddr
) in socket.getaddrinfo(host or 'localhost', port):
# for IPv6, drop flowinfo, and restrict addresses
# to [host]:port
yield family, sockaddr[:2]
else:
yield domain, addr
def _create_wrappers(self):
# Create socket wrappers
wrappers = {} # keys are active wrappers
for domain, addr in self._expand_addrlist():
wrap = ConnectWrapper(domain, addr, self.mgr, self.client)
wrap.connect_procedure()
if wrap.state == "notified":
for w in wrappers.keys():
w.close()
return {wrap: wrap}
if wrap.state != "closed":
wrappers[wrap] = wrap
return wrappers
def _connect_wrappers(self, wrappers, deadline):
# Next wait until they all actually connect (or fail)
# The deadline is necessary, because we'd wait forever if a
# sockets never connects or fails.
while wrappers:
if self.stopped:
for wrap in wrappers.keys():
wrap.close()
return 0
# Select connecting wrappers
connecting = [wrap
for wrap in wrappers.keys()
if wrap.state == "connecting"]
if not connecting:
break
if time.time() > deadline:
break
try:
r, w, x = select.select([], connecting, connecting, 1.0)
log("CT: select() %d, %d, %d" % tuple(map(len, (r,w,x))))
except select.error, msg:
log("CT: select failed; msg=%s" % str(msg),
level=logging.WARNING)
continue
# Exceptable wrappers are in trouble; close these suckers
for wrap in x:
log("CT: closing troubled socket %s" % str(wrap.addr))
del wrappers[wrap]
wrap.close()
# Writable sockets are connected
for wrap in w:
wrap.connect_procedure()
if wrap.state == "notified":
del wrappers[wrap] # Don't close this one
for wrap in wrappers.keys():
wrap.close()
return 1
if wrap.state == "closed":
del wrappers[wrap]
def _fallback_wrappers(self, wrappers, deadline):
# If we've got wrappers left at this point, they're fallback
# connections. Try notifying them until one succeeds.
for wrap in wrappers.keys():
assert wrap.state == "tested" and wrap.preferred == 0
if self.mgr.is_connected():
wrap.close()
else:
wrap.notify_client()
if wrap.state == "notified":
del wrappers[wrap] # Don't close this one
for wrap in wrappers.keys():
wrap.close()
return -1
assert wrap.state == "closed"
del wrappers[wrap]
# TODO: should check deadline
class ConnectWrapper:
"""An object that handles the connection procedure for one socket.
This is a little state machine with states:
closed
opened
connecting
connected
tested
notified
"""
def __init__(self, domain, addr, mgr, client):
"""Store arguments and create non-blocking socket."""
self.domain = domain
self.addr = addr
self.mgr = mgr
self.client = client
# These attributes are part of the interface
self.state = "closed"
self.sock = None
self.conn = None
self.preferred = 0
log("CW: attempt to connect to %s" % repr(addr))
try:
self.sock = socket.socket(domain, socket.SOCK_STREAM)
except socket.error, err:
log("CW: can't create socket, domain=%s: %s" % (domain, err),
level=logging.ERROR)
self.close()
return
self.sock.setblocking(0)
self.state = "opened"
def connect_procedure(self):
"""Call sock.connect_ex(addr) and interpret result."""
if self.state in ("opened", "connecting"):
try:
err = self.sock.connect_ex(self.addr)
except socket.error, msg:
log("CW: connect_ex(%r) failed: %s" % (self.addr, msg),
level=logging.ERROR)
self.close()
return
log("CW: connect_ex(%s) returned %s" %
(self.addr, errno.errorcode.get(err) or str(err)))
if err in _CONNECT_IN_PROGRESS:
self.state = "connecting"
return
if err not in _CONNECT_OK:
log("CW: error connecting to %s: %s" %
(self.addr, errno.errorcode.get(err) or str(err)),
level=logging.WARNING)
self.close()
return
self.state = "connected"
if self.state == "connected":
self.test_connection()
def test_connection(self):
"""Establish and test a connection at the zrpc level.
Call the client's testConnection(), giving the client a chance
to do app-level check of the connection.
"""
self.conn = ManagedClientConnection(self.sock, self.addr, self.mgr)
self.sock = None # The socket is now owned by the connection
try:
self.preferred = self.client.testConnection(self.conn)
self.state = "tested"
except ReadOnlyError:
log("CW: ReadOnlyError in testConnection (%s)" % repr(self.addr))
self.close()
return
except:
log("CW: error in testConnection (%s)" % repr(self.addr),
level=logging.ERROR, exc_info=True)
self.close()
return
if self.preferred:
self.notify_client()
def notify_client(self):
"""Call the client's notifyConnected().
If this succeeds, call the manager's connect_done().
If the client is already connected, we assume it's a fallback
connection, and the new connection must be a preferred
connection. The client will close the old connection.
"""
try:
self.client.notifyConnected(self.conn)
except:
log("CW: error in notifyConnected (%s)" % repr(self.addr),
level=logging.ERROR, exc_info=True)
self.close()
return
self.state = "notified"
self.mgr.connect_done(self.conn, self.preferred)
def close(self):
"""Close the socket and reset everything."""
self.state = "closed"
self.mgr = self.client = None
self.preferred = 0
if self.conn is not None:
# Closing the ZRPC connection will eventually close the
# socket, somewhere in asyncore. Guido asks: Why do we care?
self.conn.close()
self.conn = None
if self.sock is not None:
self.sock.close()
self.sock = None
def fileno(self):
return self.sock.fileno()
src/ZEO/zrpc/connection.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import errno
import sys
import threading
import logging
import ZEO.zrpc.marshal
import ZEO.zrpc.trigger
from ZEO.zrpc import smac
from ZEO.zrpc.error import ZRPCError, DisconnectedError
from ZEO.zrpc.log import short_repr, log
from ZODB.loglevels import BLATHER, TRACE
import ZODB.POSException
REPLY = ".reply" # message name used for replies
exception_type_type = type(Exception)
debug_zrpc = False
class Delay:
"""Used to delay response to client for synchronous calls.
When a synchronous call is made and the original handler returns
without handling the call, it returns a Delay object that prevents
the mainloop from sending a response.
"""
msgid = conn = sent = None
def set_sender(self, msgid, conn):
self.msgid = msgid
self.conn = conn
def reply(self, obj):
self.sent = 'reply'
self.conn.send_reply(self.msgid, obj)
def error(self, exc_info):
self.sent = 'error'
log("Error raised in delayed method", logging.ERROR, exc_info=True)
self.conn.return_error(self.msgid, *exc_info[:2])
def __repr__(self):
return "%s[%s, %r, %r, %r]" % (
self.__class__.__name__, id(self), self.msgid, self.conn, self.sent)
class Result(Delay):
def __init__(self, *args):
self.args = args
def set_sender(self, msgid, conn):
reply, callback = self.args
conn.send_reply(msgid, reply, False)
callback()
class MTDelay(Delay):
def __init__(self):
self.ready = threading.Event()
def set_sender(self, *args):
Delay.set_sender(self, *args)
self.ready.set()
def reply(self, obj):
self.ready.wait()
self.conn.call_from_thread(self.conn.send_reply, self.msgid, obj)
def error(self, exc_info):
self.ready.wait()
self.conn.call_from_thread(Delay.error, self, exc_info)
# PROTOCOL NEGOTIATION
#
# The code implementing protocol version 2.0.0 (which is deployed
# in the field and cannot be changed) *only* talks to peers that
# send a handshake indicating protocol version 2.0.0. In that
# version, both the client and the server immediately send out
# their protocol handshake when a connection is established,
# without waiting for their peer, and disconnect when a different
# handshake is receive.
#
# The new protocol uses this to enable new clients to talk to
# 2.0.0 servers. In the new protocol:
#
# The server sends its protocol handshake to the client at once.
#
# The client waits until it receives the server's protocol handshake
# before sending its own handshake. The client sends the lower of its
# own protocol version and the server protocol version, allowing it to
# talk to servers using later protocol versions (2.0.2 and higher) as
# well: the effective protocol used will be the lower of the client
# and server protocol. However, this changed in ZODB 3.3.1 (and
# should have changed in ZODB 3.3) because an older server doesn't
# support MVCC methods required by 3.3 clients.
#
# [Ugly details: In order to treat the first received message (protocol
# handshake) differently than all later messages, both client and server
# start by patching their message_input() method to refer to their
# recv_handshake() method instead. In addition, the client has to arrange
# to queue (delay) outgoing messages until it receives the server's
# handshake, so that the first message the client sends to the server is
# the client's handshake. This multiply-special treatment of the first
# message is delicate, and several asyncore and thread subtleties were
# handled unsafely before ZODB 3.2.6.
# ]
#
# The ZEO modules ClientStorage and ServerStub have backwards
# compatibility code for dealing with the previous version of the
# protocol. The client accepts the old version of some messages,
# and will not send new messages when talking to an old server.
#
# As long as the client hasn't sent its handshake, it can't send
# anything else; output messages are queued during this time.
# (Output can happen because the connection testing machinery can
# start sending requests before the handshake is received.)
#
# UPGRADING FROM ZEO 2.0.0 TO NEWER VERSIONS:
#
# Because a new client can talk to an old server, but not vice
# versa, all clients should be upgraded before upgrading any
# servers. Protocol upgrades beyond 2.0.1 will not have this
# restriction, because clients using protocol 2.0.1 or later can
# talk to both older and newer servers.
#
# No compatibility with protocol version 1 is provided.
# Connection is abstract (it must be derived from). ManagedServerConnection
# and ManagedClientConnection are the concrete subclasses. They need to
# supply a handshake() method appropriate for their role in protocol
# negotiation.
class Connection(smac.SizedMessageAsyncConnection, object):
"""Dispatcher for RPC on object on both sides of socket.
The connection supports synchronous calls, which expect a return,
and asynchronous calls, which do not.
It uses the Marshaller class to handle encoding and decoding of
method calls and arguments. Marshaller uses pickle to encode
arbitrary Python objects. The code here doesn't ever see the wire
format.
A Connection is designed for use in a multithreaded application,
where a synchronous call must block until a response is ready.
A socket connection between a client and a server allows either
side to invoke methods on the other side. The processes on each
end of the socket use a Connection object to manage communication.
The Connection deals with decoded RPC messages. They are
represented as four-tuples containing: msgid, flags, method name,
and a tuple of method arguments.
The msgid starts at zero and is incremented by one each time a
method call message is sent. Each side of the connection has a
separate msgid state.
When one side of the connection (the client) calls a method, it
sends a message with a new msgid. The other side (the server),
replies with a message that has the same msgid, the string
".reply" (the global variable REPLY) as the method name, and the
actual return value in the args position. Note that each side of
the Connection can initiate a call, in which case it will be the
client for that particular call.
The protocol also supports asynchronous calls. The client does
not wait for a return value for an asynchronous call.
If a method call raises an Exception, the exception is propagated
back to the client via the REPLY message. The client side will
raise any exception it receives instead of returning the value to
the caller.
"""
__super_init = smac.SizedMessageAsyncConnection.__init__
__super_close = smac.SizedMessageAsyncConnection.close
__super_setSessionKey = smac.SizedMessageAsyncConnection.setSessionKey
# Protocol history:
#
# Z200 -- Original ZEO 2.0 protocol
#
# Z201 -- Added invalidateTransaction() to client.
# Renamed several client methods.
# Added several sever methods:
# lastTransaction()
# getAuthProtocol() and scheme-specific authentication methods
# getExtensionMethods().
# getInvalidations().
#
# Z303 -- named after the ZODB release 3.3
# Added methods for MVCC:
# loadBefore()
# A Z303 client cannot talk to a Z201 server, because the latter
# doesn't support MVCC. A Z201 client can talk to a Z303 server,
# but because (at least) the type of the root object changed
# from ZODB.PersistentMapping to persistent.mapping, the older
# client can't actually make progress if a Z303 client created,
# or ever modified, the root.
#
# Z308 -- named after the ZODB release 3.8
# Added blob-support server methods:
# sendBlob
# storeBlobStart
# storeBlobChunk
# storeBlobEnd
# storeBlobShared
# Added blob-support client methods:
# receiveBlobStart
# receiveBlobChunk
# receiveBlobStop
#
# Z309 -- named after the ZODB release 3.9
# New server methods:
# restorea, iterator_start, iterator_next,
# iterator_record_start, iterator_record_next,
# iterator_gc
#
# Z310 -- named after the ZODB release 3.10
# New server methods:
# undoa
# Doesn't support undo for older clients.
# Undone oid info returned by vote.
#
# Z3101 -- checkCurrentSerialInTransaction
# Protocol variables:
# Our preferred protocol.
current_protocol = "Z3101"
# If we're a client, an exhaustive list of the server protocols we
# can accept.
servers_we_can_talk_to = ["Z308", "Z309", "Z310", current_protocol]
# If we're a server, an exhaustive list of the client protocols we
# can accept.
clients_we_can_talk_to = [
"Z200", "Z201", "Z303", "Z308", "Z309", "Z310", current_protocol]
# This is pretty excruciating. Details:
#
# 3.3 server 3.2 client
# server sends Z303 to client
# client computes min(Z303, Z201) == Z201 as the protocol to use
# client sends Z201 to server
# OK, because Z201 is in the server's clients_we_can_talk_to
#
# 3.2 server 3.3 client
# server sends Z201 to client
# client computes min(Z303, Z201) == Z201 as the protocol to use
# Z201 isn't in the client's servers_we_can_talk_to, so client
# raises exception
#
# 3.3 server 3.3 client
# server sends Z303 to client
# client computes min(Z303, Z303) == Z303 as the protocol to use
# Z303 is in the client's servers_we_can_talk_to, so client
# sends Z303 to server
# OK, because Z303 is in the server's clients_we_can_talk_to
# Exception types that should not be logged:
unlogged_exception_types = ()
# Client constructor passes 'C' for tag, server constructor 'S'. This
# is used in log messages, and to determine whether we can speak with
# our peer.
def __init__(self, sock, addr, obj, tag, map=None):
self.obj = None
self.decode = ZEO.zrpc.marshal.decode
self.encode = ZEO.zrpc.marshal.encode
self.fast_encode = ZEO.zrpc.marshal.fast_encode
self.closed = False
self.peer_protocol_version = None # set in recv_handshake()
assert tag in "CS"
self.tag = tag
self.logger = logging.getLogger('ZEO.zrpc.Connection(%c)' % tag)
if isinstance(addr, tuple):
self.log_label = "(%s:%d) " % addr
else:
self.log_label = "(%s) " % addr
# Supply our own socket map, so that we don't get registered with
# the asyncore socket map just yet. The initial protocol messages
# are treated very specially, and we dare not get invoked by asyncore
# before that special-case setup is complete. Some of that setup
# occurs near the end of this constructor, and the rest is done by
# a concrete subclass's handshake() method. Unfortunately, because
# we ultimately derive from asyncore.dispatcher, it's not possible
# to invoke the superclass constructor without asyncore stuffing
# us into _some_ socket map.
ourmap = {}
self.__super_init(sock, addr, map=ourmap)
# The singleton dict is used in synchronous mode when a method
# needs to call into asyncore to try to force some I/O to occur.
# The singleton dict is a socket map containing only this object.
self._singleton = {self._fileno: self}
# waiting_for_reply is used internally to indicate whether
# a call is in progress. setting a session key is deferred
# until after the call returns.
self.waiting_for_reply = False
self.delay_sesskey = None
self.register_object(obj)
# The first message we see is a protocol handshake. message_input()
# is temporarily replaced by recv_handshake() to treat that message
# specially. revc_handshake() does "del self.message_input", which
# uncovers the normal message_input() method thereafter.
self.message_input = self.recv_handshake
# Server and client need to do different things for protocol
# negotiation, and handshake() is implemented differently in each.
self.handshake()
# Now it's safe to register with asyncore's socket map; it was not
# safe before message_input was replaced, or before handshake() was
# invoked.
# Obscure: in Python 2.4, the base asyncore.dispatcher class grew
# a ._map attribute, which is used instead of asyncore's global
# socket map when ._map isn't None. Because we passed `ourmap` to
# the base class constructor above, in 2.4 asyncore believes we want
# to use `ourmap` instead of the global socket map -- but we don't.
# So we have to replace our ._map with the global socket map, and
# update the global socket map with `ourmap`. Replacing our ._map
# isn't necessary before Python 2.4, but doesn't hurt then (it just
# gives us an unused attribute in 2.3); updating the global socket
# map is necessary regardless of Python version.
if map is None:
map = asyncore.socket_map
self._map = map
map.update(ourmap)
def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.addr)
__str__ = __repr__ # Defeat asyncore's dreaded __getattr__
def log(self, message, level=BLATHER, exc_info=False):
self.logger.log(level, self.log_label + message, exc_info=exc_info)
def close(self):
self.mgr.close_conn(self)
if self.closed:
return
self._singleton.clear()
self.closed = True
self.__super_close()
self.trigger.pull_trigger()
def register_object(self, obj):
"""Register obj as the true object to invoke methods on."""
self.obj = obj
# Subclass must implement. handshake() is called by the constructor,
# near its end, but before self is added to asyncore's socket map.
# When a connection is created the first message sent is a 4-byte
# protocol version. This allows the protocol to evolve over time, and
# lets servers handle clients using multiple versions of the protocol.
# In general, the server's handshake() just needs to send the server's
# preferred protocol; the client's also needs to queue (delay) outgoing
# messages until it sees the handshake from the server.
def handshake(self):
raise NotImplementedError
# Replaces message_input() for the first message received. Records the
# protocol sent by the peer in `peer_protocol_version`, restores the
# normal message_input() method, and raises an exception if the peer's
# protocol is unacceptable. That's all the server needs to do. The
# client needs to do additional work in response to the server's
# handshake, and extends this method.
def recv_handshake(self, proto):
# Extended by ManagedClientConnection.
del self.message_input # uncover normal-case message_input()
self.peer_protocol_version = proto
if self.tag == 'C':
good_protos = self.servers_we_can_talk_to
else:
assert self.tag == 'S'
good_protos = self.clients_we_can_talk_to
if proto in good_protos:
self.log("received handshake %r" % proto, level=logging.INFO)
else:
self.log("bad handshake %s" % short_repr(proto),
level=logging.ERROR)
raise ZRPCError("bad handshake %r" % proto)
def message_input(self, message):
"""Decode an incoming message and dispatch it"""
# If something goes wrong during decoding, the marshaller
# will raise an exception. The exception will ultimately
# result in asycnore calling handle_error(), which will
# close the connection.
msgid, async, name, args = self.decode(message)
if debug_zrpc:
self.log("recv msg: %s, %s, %s, %s" % (msgid, async, name,
short_repr(args)),
level=TRACE)
if name == 'loadEx':
# Special case and inline the heck out of load case:
try:
ret = self.obj.loadEx(*args)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, msg:
if not isinstance(msg, self.unlogged_exception_types):
self.log("%s() raised exception: %s" % (name, msg),
logging.ERROR, exc_info=True)
self.return_error(msgid, *sys.exc_info()[:2])
else:
try:
self.message_output(self.fast_encode(msgid, 0, REPLY, ret))
self.poll()
except:
# Fall back to normal version for better error handling
self.send_reply(msgid, ret)
elif name == REPLY:
assert not async
self.handle_reply(msgid, args)
else:
self.handle_request(msgid, async, name, args)
def handle_request(self, msgid, async, name, args):
obj = self.obj
if name.startswith('_') or not hasattr(obj, name):
if obj is None:
if debug_zrpc:
self.log("no object calling %s%s"
% (name, short_repr(args)),
level=logging.DEBUG)
return
msg = "Invalid method name: %s on %s" % (name, repr(obj))
raise ZRPCError(msg)
if debug_zrpc:
self.log("calling %s%s" % (name, short_repr(args)),
level=logging.DEBUG)
meth = getattr(obj, name)
try:
self.waiting_for_reply = True
try:
ret = meth(*args)
finally:
self.waiting_for_reply = False
except (SystemExit, KeyboardInterrupt):
raise
except Exception, msg:
if not isinstance(msg, self.unlogged_exception_types):
self.log("%s() raised exception: %s" % (name, msg),
logging.ERROR, exc_info=True)
error = sys.exc_info()[:2]
if async:
self.log("Asynchronous call raised exception: %s" % self,
level=logging.ERROR, exc_info=True)
else:
self.return_error(msgid, *error)
return
if async:
if ret is not None:
raise ZRPCError("async method %s returned value %s" %
(name, short_repr(ret)))
else:
if debug_zrpc:
self.log("%s returns %s" % (name, short_repr(ret)),
logging.DEBUG)
if isinstance(ret, Delay):
ret.set_sender(msgid, self)
else:
self.send_reply(msgid, ret, not self.delay_sesskey)
if self.delay_sesskey:
self.__super_setSessionKey(self.delay_sesskey)
self.delay_sesskey = None
def return_error(self, msgid, err_type, err_value):
# Note that, ideally, this should be defined soley for
# servers, but a test arranges to get it called on
# a client. Too much trouble to fix it now. :/
if not isinstance(err_value, Exception):
err_value = err_type, err_value
# encode() can pass on a wide variety of exceptions from cPickle.
# While a bare `except` is generally poor practice, in this case
# it's acceptable -- we really do want to catch every exception
# cPickle may raise.
try:
msg = self.encode(msgid, 0, REPLY, (err_type, err_value))
except: # see above
try:
r = short_repr(err_value)
except:
r = "<unreprable>"
err = ZRPCError("Couldn't pickle error %.100s" % r)
msg = self.encode(msgid, 0, REPLY, (ZRPCError, err))
self.message_output(msg)
self.poll()
def handle_error(self):
if sys.exc_info()[0] == SystemExit:
raise sys.exc_info()
self.log("Error caught in asyncore",
level=logging.ERROR, exc_info=True)
self.close()
def setSessionKey(self, key):
if self.waiting_for_reply:
self.delay_sesskey = key
else:
self.__super_setSessionKey(key)
def send_call(self, method, args, async=False):
# send a message and return its msgid
if async:
msgid = 0
else:
msgid = self._new_msgid()
if debug_zrpc:
self.log("send msg: %d, %d, %s, ..." % (msgid, async, method),
level=TRACE)
buf = self.encode(msgid, async, method, args)
self.message_output(buf)
return msgid
def callAsync(self, method, *args):
if self.closed:
raise DisconnectedError()
self.send_call(method, args, 1)
self.poll()
def callAsyncNoPoll(self, method, *args):
# Like CallAsync but doesn't poll. This exists so that we can
# send invalidations atomically to all clients without
# allowing any client to sneak in a load request.
if self.closed:
raise DisconnectedError()
self.send_call(method, args, 1)
def callAsyncNoSend(self, method, *args):
# Like CallAsync but doesn't poll. This exists so that we can
# send invalidations atomically to all clients without
# allowing any client to sneak in a load request.
if self.closed:
raise DisconnectedError()
self.send_call(method, args, 1)
self.call_from_thread()
def callAsyncIterator(self, iterator):
"""Queue a sequence of calls using an iterator
The calls will not be interleaved with other calls from the same
client.
"""
self.message_output(self.encode(0, 1, method, args)
for method, args in iterator)
def handle_reply(self, msgid, ret):
assert msgid == -1 and ret is None
def poll(self):
"""Invoke asyncore mainloop to get pending message out."""
if debug_zrpc:
self.log("poll()", level=TRACE)
self.trigger.pull_trigger()
# import cProfile, time
class ManagedServerConnection(Connection):
"""Server-side Connection subclass."""
# Exception types that should not be logged:
unlogged_exception_types = (ZODB.POSException.POSKeyError, )
def __init__(self, sock, addr, obj, mgr):
self.mgr = mgr
map = {}
Connection.__init__(self, sock, addr, obj, 'S', map=map)
self.decode = ZEO.zrpc.marshal.server_decode
self.trigger = ZEO.zrpc.trigger.trigger(map)
self.call_from_thread = self.trigger.pull_trigger
t = threading.Thread(target=server_loop, args=(map,))
t.setDaemon(True)
t.start()
# self.profile = cProfile.Profile()
# def message_input(self, message):
# self.profile.enable()
# try:
# Connection.message_input(self, message)
# finally:
# self.profile.disable()
def handshake(self):
# Send the server's preferred protocol to the client.
self.message_output(self.current_protocol)
def recv_handshake(self, proto):
Connection.recv_handshake(self, proto)
self.obj.notifyConnected(self)
def close(self):
self.obj.notifyDisconnected()
Connection.close(self)
# self.profile.dump_stats(str(time.time())+'.stats')
def send_reply(self, msgid, ret, immediately=True):
# encode() can pass on a wide variety of exceptions from cPickle.
# While a bare `except` is generally poor practice, in this case
# it's acceptable -- we really do want to catch every exception
# cPickle may raise.
try:
msg = self.encode(msgid, 0, REPLY, ret)
except: # see above
try:
r = short_repr(ret)
except:
r = "<unreprable>"
err = ZRPCError("Couldn't pickle return %.100s" % r)
msg = self.encode(msgid, 0, REPLY, (ZRPCError, err))
self.message_output(msg)
if immediately:
self.poll()
poll = smac.SizedMessageAsyncConnection.handle_write
def server_loop(map):
while len(map) > 1:
try:
asyncore.poll(30.0, map)
except Exception, v:
if v.args[0] != errno.EBADF:
raise
for o in map.values():
o.close()
class ManagedClientConnection(Connection):
"""Client-side Connection subclass."""
__super_init = Connection.__init__
base_message_output = Connection.message_output
def __init__(self, sock, addr, mgr):
self.mgr = mgr
# We can't use the base smac's message_output directly because the
# client needs to queue outgoing messages until it's seen the
# initial protocol handshake from the server. So we have our own
# message_ouput() method, and support for initial queueing. This is
# a delicate design, requiring an output mutex to be wholly
# thread-safe.
# Caution: we must set this up before calling the base class
# constructor, because the latter registers us with asyncore;
# we need to guarantee that we'll queue outgoing messages before
# asyncore learns about us.
self.output_lock = threading.Lock()
self.queue_output = True
self.queued_messages = []
# msgid_lock guards access to msgid
self.msgid = 0
self.msgid_lock = threading.Lock()
# replies_cond is used to block when a synchronous call is
# waiting for a response
self.replies_cond = threading.Condition()
self.replies = {}
self.__super_init(sock, addr, None, tag='C', map=mgr.map)
self.trigger = mgr.trigger
self.call_from_thread = self.trigger.pull_trigger
self.call_from_thread()
def close(self):
Connection.close(self)
self.replies_cond.acquire()
self.replies_cond.notifyAll()
self.replies_cond.release()
# Our message_ouput() queues messages until recv_handshake() gets the
# protocol handshake from the server.
def message_output(self, message):
self.output_lock.acquire()
try:
if self.queue_output:
self.queued_messages.append(message)
else:
assert not self.queued_messages
self.base_message_output(message)
finally:
self.output_lock.release()
def handshake(self):
# The client waits to see the server's handshake. Outgoing messages
# are queued for the duration. The client will send its own
# handshake after the server's handshake is seen, in recv_handshake()
# below. It will then send any messages queued while waiting.
assert self.queue_output # the constructor already set this
def recv_handshake(self, proto):
# The protocol to use is the older of our and the server's preferred
# protocols.
proto = min(proto, self.current_protocol)
# Restore the normal message_input method, and raise an exception
# if the protocol version is too old.
Connection.recv_handshake(self, proto)
# Tell the server the protocol in use, then send any messages that
# were queued while waiting to hear the server's protocol, and stop
# queueing messages.
self.output_lock.acquire()
try:
self.base_message_output(proto)
for message in self.queued_messages:
self.base_message_output(message)
self.queued_messages = []
self.queue_output = False
finally:
self.output_lock.release()
def _new_msgid(self):
self.msgid_lock.acquire()
try:
msgid = self.msgid
self.msgid = self.msgid + 1
return msgid
finally:
self.msgid_lock.release()
def call(self, method, *args):
if self.closed:
raise DisconnectedError()
msgid = self.send_call(method, args)
r_args = self.wait(msgid)
if (isinstance(r_args, tuple) and len(r_args) > 1
and type(r_args[0]) == exception_type_type
and issubclass(r_args[0], Exception)):
inst = r_args[1]
raise inst # error raised by server
else:
return r_args
def wait(self, msgid):
"""Invoke asyncore mainloop and wait for reply."""
if debug_zrpc:
self.log("wait(%d)" % msgid, level=TRACE)
self.trigger.pull_trigger()
self.replies_cond.acquire()
try:
while 1:
if self.closed:
raise DisconnectedError()
reply = self.replies.get(msgid, self)
if reply is not self:
del self.replies[msgid]
if debug_zrpc:
self.log("wait(%d): reply=%s" %
(msgid, short_repr(reply)), level=TRACE)
return reply
self.replies_cond.wait()
finally:
self.replies_cond.release()
# For testing purposes, it is useful to begin a synchronous call
# but not block waiting for its response.
def _deferred_call(self, method, *args):
if self.closed:
raise DisconnectedError()
msgid = self.send_call(method, args)
self.trigger.pull_trigger()
return msgid
def _deferred_wait(self, msgid):
r_args = self.wait(msgid)
if (isinstance(r_args, tuple)
and type(r_args[0]) == exception_type_type
and issubclass(r_args[0], Exception)):
inst = r_args[1]
raise inst # error raised by server
else:
return r_args
def handle_reply(self, msgid, args):
if debug_zrpc:
self.log("recv reply: %s, %s"
% (msgid, short_repr(args)), level=TRACE)
self.replies_cond.acquire()
try:
self.replies[msgid] = args
self.replies_cond.notifyAll()
finally:
self.replies_cond.release()
def send_reply(self, msgid, ret):
# Whimper. Used to send heartbeat
assert msgid == -1 and ret is None
self.message_output('(J\xff\xff\xff\xffK\x00U\x06.replyNt.')
src/ZEO/zrpc/error.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from ZODB import POSException
from ZEO.Exceptions import ClientDisconnected
class ZRPCError(POSException.StorageError):
pass
class DisconnectedError(ZRPCError, ClientDisconnected):
"""The database storage is disconnected from the storage server.
The error occurred because a problem in the low-level RPC connection,
or because the connection was closed.
"""
# This subclass is raised when zrpc catches the error.
src/ZEO/zrpc/log.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
import threading
import logging
from ZODB.loglevels import BLATHER
LOG_THREAD_ID = 0 # Set this to 1 during heavy debugging
logger = logging.getLogger('ZEO.zrpc')
_label = "%s" % os.getpid()
def new_label():
global _label
_label = str(os.getpid())
def log(message, level=BLATHER, label=None, exc_info=False):
label = label or _label
if LOG_THREAD_ID:
label = label + ':' + threading.currentThread().getName()
logger.log(level, '(%s) %s' % (label, message), exc_info=exc_info)
REPR_LIMIT = 60
def short_repr(obj):
"Return an object repr limited to REPR_LIMIT bytes."
# Some of the objects being repr'd are large strings. A lot of memory
# would be wasted to repr them and then truncate, so they are treated
# specially in this function.
# Also handle short repr of a tuple containing a long string.
# This strategy works well for arguments to StorageServer methods.
# The oid is usually first and will get included in its entirety.
# The pickle is near the beginning, too, and you can often fit the
# module name in the pickle.
if isinstance(obj, str):
if len(obj) > REPR_LIMIT:
r = repr(obj[:REPR_LIMIT])
else:
r = repr(obj)
if len(r) > REPR_LIMIT:
r = r[:REPR_LIMIT-4] + '...' + r[-1]
return r
elif isinstance(obj, (list, tuple)):
elts = []
size = 0
for elt in obj:
r = short_repr(elt)
elts.append(r)
size += len(r)
if size > REPR_LIMIT:
break
if isinstance(obj, tuple):
r = "(%s)" % (", ".join(elts))
else:
r = "[%s]" % (", ".join(elts))
else:
r = repr(obj)
if len(r) > REPR_LIMIT:
return r[:REPR_LIMIT] + '...'
else:
return r
src/ZEO/zrpc/marshal.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from cPickle import Unpickler, Pickler
from cStringIO import StringIO
import logging
from ZEO.zrpc.error import ZRPCError
from ZEO.zrpc.log import log, short_repr
def encode(*args): # args: (msgid, flags, name, args)
# (We used to have a global pickler, but that's not thread-safe. :-( )
# It's not thread safe if, in the couse of pickling, we call the
# Python interpeter, which releases the GIL.
# Note that args may contain very large binary pickles already; for
# this reason, it's important to use proto 1 (or higher) pickles here
# too. For a long time, this used proto 0 pickles, and that can
# bloat our pickle to 4x the size (due to high-bit and control bytes
# being represented by \xij escapes in proto 0).
# Undocumented: cPickle.Pickler accepts a lone protocol argument;
# pickle.py does not.
pickler = Pickler(1)
pickler.fast = 1
return pickler.dump(args, 1)
@apply
def fast_encode():
# Only use in cases where you *know* the data contains only basic
# Python objects
pickler = Pickler(1)
pickler.fast = 1
dump = pickler.dump
def fast_encode(*args):
return dump(args, 1)
return fast_encode
def decode(msg):
"""Decodes msg and returns its parts"""
unpickler = Unpickler(StringIO(msg))
unpickler.find_global = find_global
try:
return unpickler.load() # msgid, flags, name, args
except:
log("can't decode message: %s" % short_repr(msg),
level=logging.ERROR)
raise
def server_decode(msg):
"""Decodes msg and returns its parts"""
unpickler = Unpickler(StringIO(msg))
unpickler.find_global = server_find_global
try:
return unpickler.load() # msgid, flags, name, args
except:
log("can't decode message: %s" % short_repr(msg),
level=logging.ERROR)
raise
_globals = globals()
_silly = ('__doc__',)
exception_type_type = type(Exception)
def find_global(module, name):
"""Helper for message unpickler"""
try:
m = __import__(module, _globals, _globals, _silly)
except ImportError, msg:
raise ZRPCError("import error %s: %s" % (module, msg))
try:
r = getattr(m, name)
except AttributeError:
raise ZRPCError("module %s has no global %s" % (module, name))
safe = getattr(r, '__no_side_effects__', 0)
if safe:
return r
# TODO: is there a better way to do this?
if type(r) == exception_type_type and issubclass(r, Exception):
return r
raise ZRPCError("Unsafe global: %s.%s" % (module, name))
def server_find_global(module, name):
"""Helper for message unpickler"""
try:
if module != 'ZopeUndo.Prefix':
raise ImportError
m = __import__(module, _globals, _globals, _silly)
except ImportError, msg:
raise ZRPCError("import error %s: %s" % (module, msg))
try:
r = getattr(m, name)
except AttributeError:
raise ZRPCError("module %s has no global %s" % (module, name))
return r
src/ZEO/zrpc/server.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import socket
# _has_dualstack: True if the dual-stack sockets are supported
try:
# Check whether IPv6 sockets can be created
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
except (socket.error, AttributeError):
_has_dualstack = False
else:
# Check whether enabling dualstack (disabling v6only) works
try:
s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False)
except (socket.error, AttributeError):
_has_dualstack = False
else:
_has_dualstack = True
s.close()
del s
from ZEO.zrpc.connection import Connection
from ZEO.zrpc.log import log
import ZEO.zrpc.log
import logging
# Export the main asyncore loop
loop = asyncore.loop
class Dispatcher(asyncore.dispatcher):
"""A server that accepts incoming RPC connections"""
__super_init = asyncore.dispatcher.__init__
def __init__(self, addr, factory=Connection, map=None):
self.__super_init(map=map)
self.addr = addr
self.factory = factory
self._open_socket()
def _open_socket(self):
if type(self.addr) == tuple:
if self.addr[0] == '' and _has_dualstack:
# Wildcard listen on all interfaces, both IPv4 and
# IPv6 if possible
self.create_socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.setsockopt(
socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False)
elif ':' in self.addr[0]:
self.create_socket(socket.AF_INET6, socket.SOCK_STREAM)
if _has_dualstack:
# On Linux, IPV6_V6ONLY is off by default.
# If the user explicitly asked for IPv6, don't bind to IPv4
self.socket.setsockopt(
socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, True)
else:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
log("listening on %s" % str(self.addr), logging.INFO)
self.bind(self.addr)
self.listen(5)
def writable(self):
return 0
def readable(self):
return 1
def handle_accept(self):
try:
sock, addr = self.accept()
except socket.error, msg:
log("accepted failed: %s" % msg)
return
# We could short-circuit the attempt below in some edge cases
# and avoid a log message by checking for addr being None.
# Unfortunately, our test for the code below,
# quick_close_doesnt_kill_server, causes addr to be None and
# we'd have to write a test for the non-None case, which is
# *even* harder to provoke. :/ So we'll leave things as they
# are for now.
# It might be better to check whether the socket has been
# closed, but I don't see a way to do that. :(
# Drop flow-info from IPv6 addresses
if addr: # Sometimes None on Mac. See above.
addr = addr[:2]
try:
c = self.factory(sock, addr)
except:
if sock.fileno() in asyncore.socket_map:
del asyncore.socket_map[sock.fileno()]
ZEO.zrpc.log.logger.exception("Error in handle_accept")
else:
log("connect from %s: %s" % (repr(addr), c))
src/ZEO/zrpc/smac.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Sized Message Async Connections.
This class extends the basic asyncore layer with a record-marking
layer. The message_output() method accepts an arbitrary sized string
as its argument. It sends over the wire the length of the string
encoded using struct.pack('>I') and the string itself. The receiver
passes the original string to message_input().
This layer also supports an optional message authentication code
(MAC). If a session key is present, it uses HMAC-SHA-1 to generate a
20-byte MAC. If a MAC is present, the high-order bit of the length
is set to 1 and the MAC immediately follows the length.
"""
import asyncore
import errno
try:
import hmac
except ImportError:
import _hmac as hmac
import socket
import struct
import threading
from ZEO.zrpc.log import log
from ZEO.zrpc.error import DisconnectedError
import ZEO.hash
# Use the dictionary to make sure we get the minimum number of errno
# entries. We expect that EWOULDBLOCK == EAGAIN on most systems --
# or that only one is actually used.
tmp_dict = {errno.EWOULDBLOCK: 0,
errno.EAGAIN: 0,
errno.EINTR: 0,
}
expected_socket_read_errors = tuple(tmp_dict.keys())
tmp_dict = {errno.EAGAIN: 0,
errno.EWOULDBLOCK: 0,
errno.ENOBUFS: 0,
errno.EINTR: 0,
}
expected_socket_write_errors = tuple(tmp_dict.keys())
del tmp_dict
# We chose 60000 as the socket limit by looking at the largest strings
# that we could pass to send() without blocking.
SEND_SIZE = 60000
MAC_BIT = 0x80000000L
_close_marker = object()
class SizedMessageAsyncConnection(asyncore.dispatcher):
__super_init = asyncore.dispatcher.__init__
__super_close = asyncore.dispatcher.close
__closed = True # Marker indicating that we're closed
socket = None # to outwit Sam's getattr
def __init__(self, sock, addr, map=None):
self.addr = addr
# __input_lock protects __inp, __input_len, __state, __msg_size
self.__input_lock = threading.Lock()
self.__inp = None # None, a single String, or a list
self.__input_len = 0
# Instance variables __state, __msg_size and __has_mac work together:
# when __state == 0:
# __msg_size == 4, and the next thing read is a message size;
# __has_mac is set according to the MAC_BIT in the header
# when __state == 1:
# __msg_size is variable, and the next thing read is a message.
# __has_mac indicates if we're in MAC mode or not (and
# therefore, if we need to check the mac header)
# The next thing read is always of length __msg_size.
# The state alternates between 0 and 1.
self.__state = 0
self.__has_mac = 0
self.__msg_size = 4
self.__output_messages = []
self.__output = []
self.__closed = False
# Each side of the connection sends and receives messages. A
# MAC is generated for each message and depends on each
# previous MAC; the state of the MAC generator depends on the
# history of operations it has performed. So the MACs must be
# generated in the same order they are verified.
# Each side is guaranteed to receive messages in the order
# they are sent, but there is no ordering constraint between
# message sends and receives. If the two sides are A and B
# and message An indicates the nth message sent by A, then
# A1 A2 B1 B2 and A1 B1 B2 A2 are both legitimate total
# orderings of the messages.
# As a result, there must be seperate MAC generators for each
# side of the connection. If not, the generator state would
# be different after A1 A2 B1 B2 than it would be after
# A1 B1 B2 A2; if the generator state was different, the MAC
# could not be verified.
self.__hmac_send = None
self.__hmac_recv = None
self.__super_init(sock, map)
# asyncore overwrites addr with the getpeername result
# restore our value
self.addr = addr
def setSessionKey(self, sesskey):
log("set session key %r" % sesskey)
# Low-level construction is now delayed until data are sent.
# This is to allow use of iterators that generate messages
# only when we're ready to do I/O so that we can effeciently
# transmit large files. Because we delay messages, we also
# have to delay setting the session key to retain proper
# ordering.
# The low-level output queue supports strings, a special close
# marker, and iterators. It doesn't support callbacks. We
# can create a allback by providing an iterator that doesn't
# yield anything.
# The hack fucntion below is a callback in iterator's
# clothing. :) It never yields anything, but is a generator
# and thus iterator, because it contains a yield statement.
def hack():
self.__hmac_send = hmac.HMAC(sesskey, digestmod=ZEO.hash)
self.__hmac_recv = hmac.HMAC(sesskey, digestmod=ZEO.hash)
if False:
yield ''
self.message_output(hack())
def get_addr(self):
return self.addr
# TODO: avoid expensive getattr calls? Can't remember exactly what
# this comment was supposed to mean, but it has something to do
# with the way asyncore uses getattr and uses if sock:
def __nonzero__(self):
return 1
def handle_read(self):
self.__input_lock.acquire()
try:
# Use a single __inp buffer and integer indexes to make this fast.
try:
d = self.recv(8192)
except socket.error, err:
if err[0] in expected_socket_read_errors:
return
raise
if not d:
return
input_len = self.__input_len + len(d)
msg_size = self.__msg_size
state = self.__state
has_mac = self.__has_mac
inp = self.__inp
if msg_size > input_len:
if inp is None:
self.__inp = d
elif type(self.__inp) is str:
self.__inp = [self.__inp, d]
else:
self.__inp.append(d)
self.__input_len = input_len
return # keep waiting for more input
# load all previous input and d into single string inp
if isinstance(inp, str):
inp = inp + d
elif inp is None:
inp = d
else:
inp.append(d)
inp = "".join(inp)
offset = 0
while (offset + msg_size) <= input_len:
msg = inp[offset:offset + msg_size]
offset = offset + msg_size
if not state:
msg_size = struct.unpack(">I", msg)[0]
has_mac = msg_size & MAC_BIT
if has_mac:
msg_size ^= MAC_BIT
msg_size += 20
elif self.__hmac_send:
raise ValueError("Received message without MAC")
state = 1
else:
msg_size = 4
state = 0
# Obscure: We call message_input() with __input_lock
# held!!! And message_input() may end up calling
# message_output(), which has its own lock. But
# message_output() cannot call message_input(), so
# the locking order is always consistent, which
# prevents deadlock. Also, message_input() may
# take a long time, because it can cause an
# incoming call to be handled. During all this
# time, the __input_lock is held. That's a good
# thing, because it serializes incoming calls.
if has_mac:
mac = msg[:20]
msg = msg[20:]
if self.__hmac_recv:
self.__hmac_recv.update(msg)
_mac = self.__hmac_recv.digest()
if mac != _mac:
raise ValueError("MAC failed: %r != %r"
% (_mac, mac))
else:
log("Received MAC but no session key set")
elif self.__hmac_send:
raise ValueError("Received message without MAC")
self.message_input(msg)
self.__state = state
self.__has_mac = has_mac
self.__msg_size = msg_size
self.__inp = inp[offset:]
self.__input_len = input_len - offset
finally:
self.__input_lock.release()
def readable(self):
return True
def writable(self):
return bool(self.__output_messages or self.__output)
def should_close(self):
self.__output_messages.append(_close_marker)
def handle_write(self):
output = self.__output
messages = self.__output_messages
while output or messages:
# Process queued messages until we have enough output
size = sum((len(s) for s in output))
while (size <= SEND_SIZE) and messages:
message = messages[0]
if message.__class__ is str:
size += self.__message_output(messages.pop(0), output)
elif message is _close_marker:
del messages[:]
del output[:]
return self.close()
else:
try:
message = message.next()
except StopIteration:
messages.pop(0)
else:
size += self.__message_output(message, output)
v = "".join(output)
del output[:]
try:
n = self.send(v)
except socket.error, err:
# Fix for https://bugs.launchpad.net/zodb/+bug/182833
# ensure the above mentioned "output" invariant
output.insert(0, v)
if err[0] in expected_socket_write_errors:
break # we couldn't write anything
raise
if n < len(v):
output.append(v[n:])
break # we can't write any more
def handle_close(self):
self.close()
def message_output(self, message):
if self.__closed:
raise DisconnectedError(
"This action is temporarily unavailable.<p>")
self.__output_messages.append(message)
def __message_output(self, message, output):
# do two separate appends to avoid copying the message string
size = 4
if self.__hmac_send:
output.append(struct.pack(">I", len(message) | MAC_BIT))
self.__hmac_send.update(message)
output.append(self.__hmac_send.digest())
size += 20
else:
output.append(struct.pack(">I", len(message)))
if len(message) <= SEND_SIZE:
output.append(message)
else:
for i in range(0, len(message), SEND_SIZE):
output.append(message[i:i+SEND_SIZE])
return size + len(message)
def close(self):
if not self.__closed:
self.__closed = True
self.__super_close()
src/ZEO/zrpc/trigger.py deleted 100644 → 0
View file @ 56c7dd30
##############################################################################
#
# Copyright (c) 2001-2005 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from __future__ import with_statement
import asyncore
import os
import socket
import thread
import errno
from ZODB.utils import positive_id
# Original comments follow; they're hard to follow in the context of
# ZEO's use of triggers. TODO: rewrite from a ZEO perspective.
# Wake up a call to select() running in the main thread.
#
# This is useful in a context where you are using Medusa's I/O
# subsystem to deliver data, but the data is generated by another
# thread. Normally, if Medusa is in the middle of a call to
# select(), new output data generated by another thread will have
# to sit until the call to select() either times out or returns.
# If the trigger is 'pulled' by another thread, it should immediately
# generate a READ event on the trigger object, which will force the
# select() invocation to return.
#
# A common use for this facility: letting Medusa manage I/O for a
# large number of connections; but routing each request through a
# thread chosen from a fixed-size thread pool. When a thread is
# acquired, a transaction is performed, but output data is
# accumulated into buffers that will be emptied more efficiently
# by Medusa. [picture a server that can process database queries
# rapidly, but doesn't want to tie up threads waiting to send data
# to low-bandwidth connections]
#
# The other major feature provided by this class is the ability to
# move work back into the main thread: if you call pull_trigger()
# with a thunk argument, when select() wakes up and receives the
# event it will call your thunk from within that thread. The main
# purpose of this is to remove the need to wrap thread locks around
# Medusa's data structures, which normally do not need them. [To see
# why this is true, imagine this scenario: A thread tries to push some
# new data onto a channel's outgoing data queue at the same time that
# the main thread is trying to remove some]
class _triggerbase(object):
"""OS-independent base class for OS-dependent trigger class."""
kind = None # subclass must set to "pipe" or "loopback"; used by repr
def __init__(self):
self._closed = False
# `lock` protects the `thunks` list from being traversed and
# appended to simultaneously.
self.lock = thread.allocate_lock()
# List of no-argument callbacks to invoke when the trigger is
# pulled. These run in the thread running the asyncore mainloop,
# regardless of which thread pulls the trigger.
self.thunks = []
def readable(self):
return 1
def writable(self):
return 0
def handle_connect(self):
pass
def handle_close(self):
self.close()
# Override the asyncore close() method, because it doesn't know about
# (so can't close) all the gimmicks we have open. Subclass must
# supply a _close() method to do platform-specific closing work. _close()
# will be called iff we're not already closed.
def close(self):
if not self._closed:
self._closed = True
self.del_channel()
self._close() # subclass does OS-specific stuff
def _close(self): # see close() above; subclass must supply
raise NotImplementedError
def pull_trigger(self, *thunk):
if thunk:
with self.lock:
self.thunks.append(thunk)
try:
self._physical_pull()
except Exception:
if not self._closed:
raise
# Subclass must supply _physical_pull, which does whatever the OS
# needs to do to provoke the "write" end of the trigger.
def _physical_pull(self):
raise NotImplementedError
def handle_read(self):
try:
self.recv(8192)
except socket.error:
return
while 1:
with self.lock:
if self.thunks:
thunk = self.thunks.pop(0)
else:
return
try:
thunk[0](*thunk[1:])
except:
nil, t, v, tbinfo = asyncore.compact_traceback()
print ('exception in trigger thunk:'
' (%s:%s %s)' % (t, v, tbinfo))
def __repr__(self):
return '<select-trigger (%s) at %x>' % (self.kind, positive_id(self))
if os.name == 'posix':
class trigger(_triggerbase, asyncore.file_dispatcher):
kind = "pipe"
def __init__(self, map=None):
_triggerbase.__init__(self)
r, self.trigger = os.pipe()
asyncore.file_dispatcher.__init__(self, r, map)
if self.fd != r:
# Starting in Python 2.6, the descriptor passed to
# file_dispatcher gets duped and assigned to
# self.fd. This breals the instantiation semantics and
# is a bug imo. I dount it will get fixed, but maybe
# it will. Who knows. For that reason, we test for the
# fd changing rather than just checking the Python version.
os.close(r)
def _close(self):
os.close(self.trigger)
asyncore.file_dispatcher.close(self)
def _physical_pull(self):
os.write(self.trigger, 'x')
else:
# Windows version; uses just sockets, because a pipe isn't select'able
# on Windows.
class BindError(Exception):
pass
class trigger(_triggerbase, asyncore.dispatcher):
kind = "loopback"
def __init__(self, map=None):
_triggerbase.__init__(self)
# Get a pair of connected sockets. The trigger is the 'w'
# end of the pair, which is connected to 'r'. 'r' is put
# in the asyncore socket map. "pulling the trigger" then
# means writing something on w, which will wake up r.
w = socket.socket()
# Disable buffering -- pulling the trigger sends 1 byte,
# and we want that sent immediately, to wake up asyncore's
# select() ASAP.
w.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
count = 0
while 1:
count += 1
# Bind to a local port; for efficiency, let the OS pick
# a free port for us.
# Unfortunately, stress tests showed that we may not
# be able to connect to that port ("Address already in
# use") despite that the OS picked it. This appears
# to be a race bug in the Windows socket implementation.
# So we loop until a connect() succeeds (almost always
# on the first try). See the long thread at
# http://mail.zope.org/pipermail/zope/2005-July/160433.html
# for hideous details.
a = socket.socket()
a.bind(("127.0.0.1", 0))
connect_address = a.getsockname() # assigned (host, port) pair
a.listen(1)
try:
w.connect(connect_address)
break # success
except socket.error, detail:
if detail[0] != errno.WSAEADDRINUSE:
# "Address already in use" is the only error
# I've seen on two WinXP Pro SP2 boxes, under
# Pythons 2.3.5 and 2.4.1.
raise
# (10048, 'Address already in use')
# assert count <= 2 # never triggered in Tim's tests
if count >= 10: # I've never seen it go above 2
a.close()
w.close()
raise BindError("Cannot bind trigger!")
# Close `a` and try again. Note: I originally put a short
# sleep() here, but it didn't appear to help or hurt.
a.close()
r, addr = a.accept() # r becomes asyncore's (self.)socket
a.close()
self.trigger = w
asyncore.dispatcher.__init__(self, r, map)
def _close(self):
# self.socket is r, and self.trigger is w, from __init__
self.socket.close()
self.trigger.close()
def _physical_pull(self):
self.trigger.send('x')
src/ZODB/tests/testConfig.py
View file @ e3082da7
...@@ -96,45 +96,6 @@ class ZODBConfigTest(ConfigTestBase): ...@@ -96,45 +96,6 @@ class ZODBConfigTest(ConfigTestBase):
self._test(cfg) self._test(cfg)
class ZEOConfigTest(ConfigTestBase):
def test_zeo_config(self):
# We're looking for a port that doesn't exist so a
# connection attempt will fail. Instead of elaborate
# logic to loop over a port calculation, we'll just pick a
# simple "random", likely to not-exist port number and add
# an elaborate comment explaining this instead. Go ahead,
# grep for 9.
from ZEO.ClientStorage import ClientDisconnected
import ZConfig
from ZODB.config import getDbSchema
from StringIO import StringIO
cfg = """
<zodb>
<zeoclient>
server localhost:56897
wait false
</zeoclient>
</zodb>
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg))
self.assertEqual(config.database[0].config.storage.config.blob_dir,
None)
self.assertRaises(ClientDisconnected, self._test, cfg)
cfg = """
<zodb>
<zeoclient>
blob-dir blobs
server localhost:56897
wait false
</zeoclient>
</zodb>
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg))
self.assertEqual(config.database[0].config.storage.config.blob_dir,
'blobs')
self.assertRaises(ClientDisconnected, self._test, cfg)
def database_xrefs_config(): def database_xrefs_config():
r""" r"""
>>> db = ZODB.config.databaseFromString( >>> db = ZODB.config.databaseFromString(
...@@ -227,7 +188,6 @@ def test_suite(): ...@@ -227,7 +188,6 @@ def test_suite():
suite.addTest(doctest.DocTestSuite( suite.addTest(doctest.DocTestSuite(
setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown)) setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown))
suite.addTest(unittest.makeSuite(ZODBConfigTest)) suite.addTest(unittest.makeSuite(ZODBConfigTest))
suite.addTest(unittest.makeSuite(ZEOConfigTest))
return suite return suite
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7