Skip to content

Z
ZODB
Commit 9e154d03 authored by Jeremy Hylton's avatar Jeremy Hylton
Browse files
Options

Undo merge of ZEO-ZRPC-Dev branch into the trunk (for now).

parent 48bcb3a7
Expand all Show whitespace changes
Inline Side-by-side
Showing with 1235 additions and 2098 deletions
src/ZEO/ClientCache.py
View file @ 9e154d03
...@@ -144,20 +144,18 @@ file 0 and file 1. ...@@ -144,20 +144,18 @@ file 0 and file 1.
""" """
__version__ = "$Revision: 1.19 $"[11:-2] __version__ = "$Revision: 1.20 $"[11:-2]
import os, tempfile import os, tempfile
from struct import pack, unpack from struct import pack, unpack
from thread import allocate_lock from thread import allocate_lock
import sys
import zLOG import zLOG
def log(msg, level=zLOG.INFO):
zLOG.LOG("ZEC", level, msg)
magic='ZEC0' magic='ZEC0'
def LOG(msg, level=zLOG.BLATHER):
zLOG.LOG("ZEC", level, msg)
class ClientCache: class ClientCache:
def __init__(self, storage='', size=20000000, client=None, var=None): def __init__(self, storage='', size=20000000, client=None, var=None):
...@@ -213,14 +211,16 @@ class ClientCache: ...@@ -213,14 +211,16 @@ class ClientCache:
f[0].write(magic) f[0].write(magic)
current=0 current=0
log("cache opened. current = %s" % current)
self._limit=size/2 self._limit=size/2
self._current=current self._current=current
def close(self):
try:
self._f[self._current].close()
except (os.error, ValueError):
pass
def open(self): def open(self):
# XXX open is overloaded to perform two tasks for
# optimization reasons
self._acquire() self._acquire()
try: try:
self._index=index={} self._index=index={}
...@@ -235,19 +235,6 @@ class ClientCache: ...@@ -235,19 +235,6 @@ class ClientCache:
return serial.items() return serial.items()
finally: self._release() finally: self._release()
def close(self):
for f in self._f:
if f is not None:
f.close()
def verify(self, verifyFunc):
"""Call the verifyFunc on every object in the cache.
verifyFunc(oid, serialno, version)
"""
for oid, (s, vs) in self.open():
verifyFunc(oid, s, vs)
def invalidate(self, oid, version): def invalidate(self, oid, version):
self._acquire() self._acquire()
try: try:
...@@ -386,6 +373,8 @@ class ClientCache: ...@@ -386,6 +373,8 @@ class ClientCache:
self._f[current]=open(self._p[current],'w+b') self._f[current]=open(self._p[current],'w+b')
else: else:
# Temporary cache file: # Temporary cache file:
if self._f[current] is not None:
self._f[current].close()
self._f[current] = tempfile.TemporaryFile(suffix='.zec') self._f[current] = tempfile.TemporaryFile(suffix='.zec')
self._f[current].write(magic) self._f[current].write(magic)
self._pos=pos=4 self._pos=pos=4
...@@ -394,57 +383,55 @@ class ClientCache: ...@@ -394,57 +383,55 @@ class ClientCache:
def store(self, oid, p, s, version, pv, sv): def store(self, oid, p, s, version, pv, sv):
self._acquire() self._acquire()
try: try: self._store(oid, p, s, version, pv, sv)
self._store(oid, p, s, version, pv, sv) finally: self._release()
finally:
self._release()
def _store(self, oid, p, s, version, pv, sv): def _store(self, oid, p, s, version, pv, sv):
if not s: if not s:
p = '' p=''
s = '\0\0\0\0\0\0\0\0' s='\0\0\0\0\0\0\0\0'
tlen = 31 + len(p) tlen=31+len(p)
if version: if version:
tlen = tlen + len(version) + 12 + len(pv) tlen=tlen+len(version)+12+len(pv)
vlen = len(version) vlen=len(version)
else: else:
vlen = 0 vlen=0
stlen = pack(">I", tlen) pos=self._pos
# accumulate various data to write into a list current=self._current
l = [oid, 'v', stlen, pack(">HI", vlen, len(p)), s] f=self._f[current]
if p: f.seek(pos)
l.append(p) stlen=pack(">I",tlen)
write=f.write
write(oid+'v'+stlen+pack(">HI", vlen, len(p))+s)
if p: write(p)
if version: if version:
l.extend([version, write(version)
pack(">I", len(pv)), write(pack(">I", len(pv)))
pv, sv]) write(pv)
l.append(stlen) write(sv)
f = self._f[self._current]
f.seek(self._pos) write(stlen)
f.write("".join(l))
if self._current:
self._index[oid] = - self._pos
else:
self._index[oid] = self._pos
self._pos += tlen if current: self._index[oid]=-pos
else: self._index[oid]=pos
self._pos=pos+tlen
def read_index(index, serial, f, current): def read_index(index, serial, f, current):
LOG("read_index(%s)" % f.name)
seek=f.seek seek=f.seek
read=f.read read=f.read
pos=4 pos=4
seek(0,2)
size=f.tell()
while 1: while 1:
f.seek(pos) seek(pos)
h=read(27) h=read(27)
if len(h)==27 and h[8] in 'vni': if len(h)==27 and h[8] in 'vni':
tlen, vlen, dlen = unpack(">iHi", h[9:19]) tlen, vlen, dlen = unpack(">iHi", h[9:19])
else: tlen=-1 else:
break
if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen:
break break
...@@ -479,3 +466,15 @@ def read_index(index, serial, f, current): ...@@ -479,3 +466,15 @@ def read_index(index, serial, f, current):
except: pass except: pass
return pos return pos
def main(files):
for file in files:
print file
index = {}
serial = {}
read_index(index, serial, open(file), 0)
print index.keys()
if __name__ == "__main__":
import sys
main(sys.argv[1:])
src/ZEO/ClientStorage.py
View file @ 9e154d03
This diff is collapsed.
src/ZEO/ClientStub.py deleted 100644 → 0
View file @ 48bcb3a7
"""Stub for interface exported by ClientStorage"""
class ClientStorage:
def __init__(self, rpc):
self.rpc = rpc
def beginVerify(self):
self.rpc.callAsync('begin')
# XXX what's the difference between these two?
def invalidate(self, args):
self.rpc.callAsync('invalidate', args)
def Invalidate(self, args):
self.rpc.callAsync('Invalidate', args)
def endVerify(self):
self.rpc.callAsync('end')
def serialno(self, arg):
self.rpc.callAsync('serialno', arg)
def info(self, arg):
self.rpc.callAsync('info', arg)
src/ZEO/Exceptions.py deleted 100644 → 0
View file @ 48bcb3a7
"""Exceptions for ZEO."""
class Disconnected(Exception):
"""Exception raised when a ZEO client is disconnected from the
ZEO server."""
src/ZEO/ServerStub.py deleted 100644 → 0
View file @ 48bcb3a7
"""Stub for interface exposed by StorageServer"""
class StorageServer:
def __init__(self, rpc):
self.rpc = rpc
def register(self, storage_name, read_only):
self.rpc.call('register', storage_name, read_only)
def get_info(self):
return self.rpc.call('get_info')
def get_size_info(self):
return self.rpc.call('get_size_info')
def beginZeoVerify(self):
self.rpc.callAsync('beginZeoVerify')
def zeoVerify(self, oid, s, sv):
self.rpc.callAsync('zeoVerify', oid, s, sv)
def endZeoVerify(self):
self.rpc.callAsync('endZeoVerify')
def new_oids(self, n=None):
if n is None:
return self.rpc.call('new_oids')
else:
return self.rpc.call('new_oids', n)
def pack(self, t, wait=None):
if wait is None:
self.rpc.call('pack', t)
else:
self.rpc.call('pack', t, wait)
def zeoLoad(self, oid):
return self.rpc.call('zeoLoad', oid)
def storea(self, oid, serial, data, version, id):
self.rpc.callAsync('storea', oid, serial, data, version, id)
def tpc_begin(self, id, user, descr, ext):
return self.rpc.call('tpc_begin', id, user, descr, ext)
def vote(self, trans_id):
return self.rpc.call('vote', trans_id)
def tpc_finish(self, id):
return self.rpc.call('tpc_finish', id)
def tpc_abort(self, id):
self.rpc.callAsync('tpc_abort', id)
def abortVersion(self, src, id):
return self.rpc.call('abortVersion', src, id)
def commitVersion(self, src, dest, id):
return self.rpc.call('commitVersion', src, dest, id)
def history(self, oid, version, length=None):
if length is not None:
return self.rpc.call('history', oid, version)
else:
return self.rpc.call('history', oid, version, length)
def load(self, oid, version):
return self.rpc.call('load', oid, version)
def loadSerial(self, oid, serial):
return self.rpc.call('loadSerial', oid, serial)
def modifiedInVersion(self, oid):
return self.rpc.call('modifiedInVersion', oid)
def new_oid(self, last=None):
if last is None:
return self.rpc.call('new_oid')
else:
return self.rpc.call('new_oid', last)
def store(self, oid, serial, data, version, trans):
return self.rpc.call('store', oid, serial, data, version, trans)
def transactionalUndo(self, trans_id, trans):
return self.rpc.call('transactionalUndo', trans_id, trans)
def undo(self, trans_id):
return self.rpc.call('undo', trans_id)
def undoLog(self, first, last):
# XXX filter not allowed across RPC
return self.rpc.call('undoLog', first, last)
def undoInfo(self, first, last, spec):
return self.rpc.call('undoInfo', first, last, spec)
def versionEmpty(self, vers):
return self.rpc.call('versionEmpty', vers)
def versions(self, max=None):
if max is None:
return self.rpc.call('versions')
else:
return self.rpc.call('versions', max)
src/ZEO/StorageServer.py
View file @ 9e154d03
This diff is collapsed.
src/ZEO/TransactionBuffer.py deleted 100644 → 0
View file @ 48bcb3a7
"""A TransactionBuffer store transaction updates until commit or abort.
A transaction may generate enough data that it is not practical to
always hold pending updates in memory. Instead, a TransactionBuffer
is used to store the data until a commit or abort.
"""
# XXX Figure out what a sensible storage format is
# XXX A faster implementation might store trans data in memory until
# it reaches a certain size.
import tempfile
import cPickle
class TransactionBuffer:
def __init__(self):
self.file = tempfile.TemporaryFile()
self.count = 0
self.size = 0
# It's safe to use a fast pickler because the only objects
# stored are builtin types -- strings or None.
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
def store(self, oid, version, data):
"""Store oid, version, data for later retrieval"""
self.pickler.dump((oid, version, data))
self.count += 1
# Estimate per-record cache size
self.size = self.size + len(data) + (27 + 12)
if version:
self.size = self.size + len(version) + 4
def invalidate(self, oid, version):
self.pickler.dump((oid, version, None))
self.count += 1
def clear(self):
"""Mark the buffer as empty"""
self.file.seek(0)
self.count = 0
self.size = 0
# XXX unchecked constraints:
# 1. can't call store() after begin_iterate()
# 2. must call clear() after iteration finishes
def begin_iterate(self):
"""Move the file pointer in advance of iteration"""
self.file.flush()
self.file.seek(0)
self.unpickler = cPickle.Unpickler(self.file)
def next(self):
"""Return next tuple of data or None if EOF"""
if self.count == 0:
del self.unpickler
return None
oid_ver_data = self.unpickler.load()
self.count -= 1
return oid_ver_data
def get_size(self):
"""Return size of data stored in buffer (just a hint)."""
return self.size
src/ZEO/smac.py
View file @ 9e154d03
...@@ -85,14 +85,11 @@ ...@@ -85,14 +85,11 @@
"""Sized message async connections """Sized message async connections
""" """
__version__ = "$Revision: 1.12 $"[11:-2] __version__ = "$Revision: 1.13 $"[11:-2]
import asyncore, struct
from Exceptions import Disconnected
from zLOG import LOG, TRACE, ERROR, INFO, BLATHER
from types import StringType
import asyncore, string, struct, zLOG, sys, Acquisition
import socket, errno import socket, errno
from zLOG import LOG, TRACE, ERROR, INFO
# Use the dictionary to make sure we get the minimum number of errno # Use the dictionary to make sure we get the minimum number of errno
# entries. We expect that EWOULDBLOCK == EAGAIN on most systems -- # entries. We expect that EWOULDBLOCK == EAGAIN on most systems --
...@@ -112,101 +109,81 @@ tmp_dict = {errno.EAGAIN: 0, ...@@ -112,101 +109,81 @@ tmp_dict = {errno.EAGAIN: 0,
expected_socket_write_errors = tuple(tmp_dict.keys()) expected_socket_write_errors = tuple(tmp_dict.keys())
del tmp_dict del tmp_dict
class SizedMessageAsyncConnection(asyncore.dispatcher): class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
__super_init = asyncore.dispatcher.__init__
__super_close = asyncore.dispatcher.close
__closed = 1 # Marker indicating that we're closed
socket = None # to outwit Sam's getattr __append=None # Marker indicating that we're closed
READ_SIZE = 8096 socket=None # to outwit Sam's getattr
def __init__(self, sock, addr, map=None, debug=None): def __init__(self, sock, addr, map=None, debug=None):
self.__super_init(sock, map) SizedMessageAsyncConnection.inheritedAttribute(
self.addr = addr '__init__')(self, sock, map)
self.addr=addr
if debug is not None: if debug is not None:
self._debug = debug self._debug=debug
elif not hasattr(self, '_debug'): elif not hasattr(self, '_debug'):
self._debug = __debug__ and 'smac' self._debug=__debug__ and 'smac'
self.__state = None self.__state=None
self.__inp = None # None, a single String, or a list self.__inp=None
self.__input_len = 0 self.__inpl=0
self.__msg_size = 4 self.__l=4
self.__output = [] self.__output=output=[]
self.__closed = None self.__append=output.append
self.__pop=output.pop
# XXX avoid expensive getattr calls?
def __nonzero__(self): def handle_read(self,
return 1 join=string.join, StringType=type(''), _type=type,
_None=None):
def handle_read(self):
# Use a single __inp buffer and integer indexes to make this
# fast.
try: try:
d=self.recv(8096) d=self.recv(8096)
except socket.error, err: except socket.error, err:
if err[0] in expected_socket_read_errors: if err[0] in expected_socket_read_errors:
return return
raise raise
if not d: if not d: return
return
input_len = self.__input_len + len(d)
msg_size = self.__msg_size
state = self.__state
inp = self.__inp inp=self.__inp
if msg_size > input_len: if inp is _None:
if inp is None: inp=d
self.__inp = d elif _type(inp) is StringType:
elif type(self.__inp) is StringType: inp=[inp,d]
self.__inp = [self.__inp, d]
else:
self.__inp.append(d)
self.__input_len = input_len
return # keep waiting for more input
# load all previous input and d into single string inp
if isinstance(inp, StringType):
inp = inp + d
elif inp is None:
inp = d
else: else:
inp.append(d) inp.append(d)
inp = "".join(inp)
offset = 0 inpl=self.__inpl+len(d)
while (offset + msg_size) <= input_len: l=self.__l
msg = inp[offset:offset + msg_size]
offset = offset + msg_size while 1:
if state is None:
if l <= inpl:
# Woo hoo, we have enough data
if _type(inp) is not StringType: inp=join(inp,'')
d=inp[:l]
inp=inp[l:]
inpl=inpl-l
if self.__state is _None:
# waiting for message # waiting for message
msg_size = struct.unpack(">i", msg)[0] l=struct.unpack(">i",d)[0]
state = 1 self.__state=1
else: else:
msg_size = 4 l=4
state = None self.__state=_None
self.message_input(msg) self.message_input(d)
else:
self.__state = state break # not enough data
self.__msg_size = msg_size
self.__inp = inp[offset:]
self.__input_len = input_len - offset
def readable(self): self.__l=l
return 1 self.__inp=inp
self.__inpl=inpl
def writable(self): def readable(self): return 1
if len(self.__output) == 0: def writable(self): return not not self.__output
return 0
else:
return 1
def handle_write(self): def handle_write(self):
output = self.__output output=self.__output
while output: while output:
v = output[0] v=output[0]
try: try:
n=self.send(v) n=self.send(v)
except socket.error, err: except socket.error, err:
...@@ -214,33 +191,42 @@ class SizedMessageAsyncConnection(asyncore.dispatcher): ...@@ -214,33 +191,42 @@ class SizedMessageAsyncConnection(asyncore.dispatcher):
break # we couldn't write anything break # we couldn't write anything
raise raise
if n < len(v): if n < len(v):
output[0] = v[n:] output[0]=v[n:]
break # we can't write any more break # we can't write any more
else: else:
del output[0] del output[0]
#break # waaa
def handle_close(self): def handle_close(self):
self.close() self.close()
def message_output(self, message): def message_output(self, message,
if __debug__: pack=struct.pack, len=len):
if self._debug: if self._debug:
if len(message) > 40: if len(message) > 40: m=message[:40]+' ...'
m = message[:40]+' ...' else: m=message
else:
m = message
LOG(self._debug, TRACE, 'message_output %s' % `m`) LOG(self._debug, TRACE, 'message_output %s' % `m`)
if self.__closed is not None: append=self.__append
raise Disconnected, ( if append is None:
"This action is temporarily unavailable." raise Disconnected("This action is temporarily unavailable.<p>")
"<p>"
) append(pack(">i",len(message))+message)
# do two separate appends to avoid copying the message string
self.__output.append(struct.pack(">i", len(message))) def log_info(self, message, type='info'):
self.__output.append(message) if type=='error': type=ERROR
else: type=INFO
LOG('ZEO', type, message)
log=log_info
def close(self): def close(self):
if self.__closed is None: if self.__append is not None:
self.__closed = 1 self.__append=None
self.__super_close() SizedMessageAsyncConnection.inheritedAttribute('close')(self)
class Disconnected(Exception):
"""The client has become disconnected from the server
"""
src/ZEO/start.py
View file @ 9e154d03
...@@ -86,13 +86,10 @@ ...@@ -86,13 +86,10 @@
"""Start the server storage. """Start the server storage.
""" """
__version__ = "$Revision: 1.27 $"[11:-2] __version__ = "$Revision: 1.28 $"[11:-2]
import sys, os, getopt, string import sys, os, getopt, string
import StorageServer
import asyncore
def directory(p, n=1): def directory(p, n=1):
d=p d=p
while n: while n:
...@@ -118,11 +115,9 @@ def get_storage(m, n, cache={}): ...@@ -118,11 +115,9 @@ def get_storage(m, n, cache={}):
def main(argv): def main(argv):
me=argv[0] me=argv[0]
sys.path[:]==filter(None, sys.path)
sys.path.insert(0, directory(me, 2)) sys.path.insert(0, directory(me, 2))
# XXX hack for profiling support
global unix, storages, zeo_pid, asyncore
args=[] args=[]
last='' last=''
for a in argv[1:]: for a in argv[1:]:
...@@ -135,13 +130,25 @@ def main(argv): ...@@ -135,13 +130,25 @@ def main(argv):
args.append(a) args.append(a)
last=a last=a
INSTANCE_HOME=os.environ.get('INSTANCE_HOME', directory(me, 4)) if os.environ.has_key('INSTANCE_HOME'):
INSTANCE_HOME=os.environ['INSTANCE_HOME']
elif os.path.isdir(os.path.join(directory(me, 4),'var')):
INSTANCE_HOME=directory(me, 4)
else:
INSTANCE_HOME=os.getcwd()
if os.path.isdir(os.path.join(INSTANCE_HOME, 'var')):
var=os.path.join(INSTANCE_HOME, 'var')
else:
var=INSTANCE_HOME
zeo_pid=os.environ.get('ZEO_SERVER_PID', zeo_pid=os.environ.get('ZEO_SERVER_PID',
os.path.join(INSTANCE_HOME, 'var', 'ZEO_SERVER.pid') os.path.join(var, 'ZEO_SERVER.pid')
) )
fs=os.path.join(INSTANCE_HOME, 'var', 'Data.fs') opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:')
fs=os.path.join(var, 'Data.fs')
usage="""%s [options] [filename] usage="""%s [options] [filename]
...@@ -149,14 +156,17 @@ def main(argv): ...@@ -149,14 +156,17 @@ def main(argv):
-D -- Run in debug mode -D -- Run in debug mode
-d -- Generate detailed debug logging without running
in the foreground.
-U -- Unix-domain socket file to listen on -U -- Unix-domain socket file to listen on
-u username or uid number -u username or uid number
The username to run the ZEO server as. You may want to run The username to run the ZEO server as. You may want to run
the ZEO server as 'nobody' or some other user with limited the ZEO server as 'nobody' or some other user with limited
resouces. The only works under Unix, and if ZServer is resouces. The only works under Unix, and if the storage
started by root. server is started by root.
-p port -- port to listen on -p port -- port to listen on
...@@ -179,42 +189,23 @@ def main(argv): ...@@ -179,42 +189,23 @@ def main(argv):
attr_name -- This is the name to which the storage object attr_name -- This is the name to which the storage object
is assigned in the module. is assigned in the module.
-P file -- Run under profile and dump output to file. Implies the
-s flag.
if no file name is specified, then %s is used. if no file name is specified, then %s is used.
""" % (me, fs) """ % (me, fs)
try:
opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:P:')
except getopt.error, msg:
print usage
print msg
sys.exit(1)
port=None port=None
debug=0 debug=detailed=0
host='' host=''
unix=None unix=None
Z=1 Z=1
UID='nobody' UID='nobody'
prof = None
for o, v in opts: for o, v in opts:
if o=='-p': port=string.atoi(v) if o=='-p': port=string.atoi(v)
elif o=='-h': host=v elif o=='-h': host=v
elif o=='-U': unix=v elif o=='-U': unix=v
elif o=='-u': UID=v elif o=='-u': UID=v
elif o=='-D': debug=1 elif o=='-D': debug=1
elif o=='-d': detailed=1
elif o=='-s': Z=0 elif o=='-s': Z=0
elif o=='-P': prof = v
if prof:
Z = 0
try:
from ZServer.medusa import asyncore
sys.modules['asyncore']=asyncore
except: pass
if port is None and unix is None: if port is None and unix is None:
print usage print usage
...@@ -228,9 +219,10 @@ def main(argv): ...@@ -228,9 +219,10 @@ def main(argv):
sys.exit(1) sys.exit(1)
fs=args[0] fs=args[0]
__builtins__.__debug__=debug
if debug: os.environ['Z_DEBUG_MODE']='1' if debug: os.environ['Z_DEBUG_MODE']='1'
if detailed: os.environ['STUPID_LOG_SEVERITY']='-99999'
from zLOG import LOG, INFO, ERROR from zLOG import LOG, INFO, ERROR
# Try to set uid to "-u" -provided uid. # Try to set uid to "-u" -provided uid.
...@@ -271,6 +263,10 @@ def main(argv): ...@@ -271,6 +263,10 @@ def main(argv):
import zdaemon import zdaemon
zdaemon.run(sys.argv, '') zdaemon.run(sys.argv, '')
try:
import ZEO.StorageServer, asyncore
storages={} storages={}
for o, v in opts: for o, v in opts:
if o=='-S': if o=='-S':
...@@ -297,9 +293,10 @@ def main(argv): ...@@ -297,9 +293,10 @@ def main(argv):
signal.signal(signal.SIGINT, signal.signal(signal.SIGINT,
lambda sig, frame, s=storages: shutdown(s, 0) lambda sig, frame, s=storages: shutdown(s, 0)
) )
signal.signal(signal.SIGHUP, rotate_logs_handler) try: signal.signal(signal.SIGHUP, rotate_logs_handler)
except: pass
finally: pass except: pass
items=storages.items() items=storages.items()
items.sort() items.sort()
...@@ -308,25 +305,40 @@ def main(argv): ...@@ -308,25 +305,40 @@ def main(argv):
if not unix: unix=host, port if not unix: unix=host, port
if prof: ZEO.StorageServer.StorageServer(unix, storages)
cmds = \
"StorageServer.StorageServer(unix, storages);" \ try: ppid, pid = os.getppid(), os.getpid()
'open(zeo_pid,"w").write("%s %s" % (os.getppid(), os.getpid()));' \ except: pass # getpid not supported
"asyncore.loop()" else: open(zeo_pid,'w').write("%s %s" % (ppid, pid))
import profile
profile.run(cmds, prof) except:
else: # Log startup exception and tell zdaemon not to restart us.
StorageServer.StorageServer(unix, storages) info=sys.exc_info()
open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) try:
import zLOG
zLOG.LOG("z2", zLOG.PANIC, "Startup exception",
error=info)
except:
pass
import traceback
apply(traceback.print_exception, info)
sys.exit(0)
asyncore.loop() asyncore.loop()
def rotate_logs(): def rotate_logs():
import zLOG import zLOG
if hasattr(zLOG.log_write, 'reinitialize'): if hasattr(zLOG.log_write, 'reinitialize'):
zLOG.log_write.reinitialize() zLOG.log_write.reinitialize()
else: else:
# Hm, lets at least try to take care of the stupid logger: # Hm, lets at least try to take care of the stupid logger:
zLOG._stupid_dest=None if hasattr(zLOG, '_set_stupid_dest'):
zLOG._set_stupid_dest(None)
else:
zLOG._stupid_dest = None
def rotate_logs_handler(signum, frame): def rotate_logs_handler(signum, frame):
rotate_logs() rotate_logs()
...@@ -347,7 +359,7 @@ def shutdown(storages, die=1): ...@@ -347,7 +359,7 @@ def shutdown(storages, die=1):
for storage in storages.values(): for storage in storages.values():
try: storage.close() try: storage.close()
finally: pass except: pass
try: try:
from zLOG import LOG, INFO from zLOG import LOG, INFO
......
src/ZEO/tests/forker.py
View file @ 9e154d03
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 1.1 (ZPL). A copy of the ZPL should accompany this
# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL
# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST
# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
"""Library for forking storage server and connecting client storage""" """Library for forking storage server and connecting client storage"""
import asyncore import asyncore
import os import os
import profile
import random import random
import socket import socket
import sys import sys
import traceback
import types import types
import ZEO.ClientStorage, ZEO.StorageServer import ZEO.ClientStorage, ZEO.StorageServer
# Change value of PROFILE to enable server-side profiling
PROFILE = 0 PROFILE = 0
if PROFILE:
import hotshot
def get_port(): def get_port():
"""Return a port that is not in use. """Return a port that is not in use.
...@@ -78,11 +66,9 @@ else: ...@@ -78,11 +66,9 @@ else:
buf = self.recv(4) buf = self.recv(4)
if buf: if buf:
assert buf == "done" assert buf == "done"
server.close_server()
asyncore.socket_map.clear() asyncore.socket_map.clear()
def handle_close(self): def handle_close(self):
server.close_server()
asyncore.socket_map.clear() asyncore.socket_map.clear()
class ZEOClientExit: class ZEOClientExit:
...@@ -91,27 +77,20 @@ else: ...@@ -91,27 +77,20 @@ else:
self.pipe = pipe self.pipe = pipe
def close(self): def close(self):
try:
os.write(self.pipe, "done") os.write(self.pipe, "done")
os.close(self.pipe) os.close(self.pipe)
except os.error:
pass
def start_zeo_server(storage, addr): def start_zeo_server(storage, addr):
rd, wr = os.pipe() rd, wr = os.pipe()
pid = os.fork() pid = os.fork()
if pid == 0: if pid == 0:
try:
if PROFILE: if PROFILE:
p = hotshot.Profile("stats.s.%d" % os.getpid()) p = profile.Profile()
p.runctx("run_server(storage, addr, rd, wr)", p.runctx("run_server(storage, addr, rd, wr)", globals(),
globals(), locals()) locals())
p.close() p.dump_stats("stats.s.%d" % os.getpid())
else: else:
run_server(storage, addr, rd, wr) run_server(storage, addr, rd, wr)
except:
print "Exception in ZEO server process"
traceback.print_exc()
os._exit(0) os._exit(0)
else: else:
os.close(rd) os.close(rd)
...@@ -119,11 +98,11 @@ else: ...@@ -119,11 +98,11 @@ else:
def run_server(storage, addr, rd, wr): def run_server(storage, addr, rd, wr):
# in the child, run the storage server # in the child, run the storage server
global server
os.close(wr) os.close(wr)
ZEOServerExit(rd) ZEOServerExit(rd)
server = ZEO.StorageServer.StorageServer(addr, {'1':storage}) serv = ZEO.StorageServer.StorageServer(addr, {'1':storage})
asyncore.loop() asyncore.loop()
os.close(rd)
storage.close() storage.close()
if isinstance(addr, types.StringType): if isinstance(addr, types.StringType):
os.unlink(addr) os.unlink(addr)
...@@ -149,7 +128,6 @@ else: ...@@ -149,7 +128,6 @@ else:
s = ZEO.ClientStorage.ClientStorage(addr, storage_id, s = ZEO.ClientStorage.ClientStorage(addr, storage_id,
debug=1, client=cache, debug=1, client=cache,
cache_size=cache_size, cache_size=cache_size,
min_disconnect_poll=0.5, min_disconnect_poll=0.5)
wait_for_server_on_startup=1)
return s, exit, pid return s, exit, pid
src/ZEO/tests/testTransactionBuffer.py deleted 100644 → 0
View file @ 48bcb3a7
import random
import unittest
from ZEO.TransactionBuffer import TransactionBuffer
def random_string(size):
"""Return a random string of size size."""
l = [chr(random.randrange(256)) for i in range(size)]
return "".join(l)
def new_store_data():
"""Return arbitrary data to use as argument to store() method."""
return random_string(8), '', random_string(random.randrange(1000))
def new_invalidate_data():
"""Return arbitrary data to use as argument to invalidate() method."""
return random_string(8), ''
class TransBufTests(unittest.TestCase):
def checkTypicalUsage(self):
tbuf = TransactionBuffer()
tbuf.store(*new_store_data())
tbuf.invalidate(*new_invalidate_data())
tbuf.begin_iterate()
while 1:
o = tbuf.next()
if o is None:
break
tbuf.clear()
def doUpdates(self, tbuf):
data = []
for i in range(10):
d = new_store_data()
tbuf.store(*d)
data.append(d)
d = new_invalidate_data()
tbuf.invalidate(*d)
data.append(d)
tbuf.begin_iterate()
for i in range(len(data)):
x = tbuf.next()
if x[2] is None:
# the tbuf add a dummy None to invalidates
x = x[:2]
self.assertEqual(x, data[i])
def checkOrderPreserved(self):
tbuf = TransactionBuffer()
self.doUpdates(tbuf)
def checkReusable(self):
tbuf = TransactionBuffer()
self.doUpdates(tbuf)
tbuf.clear()
self.doUpdates(tbuf)
tbuf.clear()
self.doUpdates(tbuf)
def test_suite():
return unittest.makeSuite(TransBufTests, 'check')
src/ZEO/zrpc/smac.py
View file @ 9e154d03
...@@ -85,14 +85,11 @@ ...@@ -85,14 +85,11 @@
"""Sized message async connections """Sized message async connections
""" """
__version__ = "$Revision: 1.12 $"[11:-2] __version__ = "$Revision: 1.13 $"[11:-2]
import asyncore, struct
from Exceptions import Disconnected
from zLOG import LOG, TRACE, ERROR, INFO, BLATHER
from types import StringType
import asyncore, string, struct, zLOG, sys, Acquisition
import socket, errno import socket, errno
from zLOG import LOG, TRACE, ERROR, INFO
# Use the dictionary to make sure we get the minimum number of errno # Use the dictionary to make sure we get the minimum number of errno
# entries. We expect that EWOULDBLOCK == EAGAIN on most systems -- # entries. We expect that EWOULDBLOCK == EAGAIN on most systems --
...@@ -112,101 +109,81 @@ tmp_dict = {errno.EAGAIN: 0, ...@@ -112,101 +109,81 @@ tmp_dict = {errno.EAGAIN: 0,
expected_socket_write_errors = tuple(tmp_dict.keys()) expected_socket_write_errors = tuple(tmp_dict.keys())
del tmp_dict del tmp_dict
class SizedMessageAsyncConnection(asyncore.dispatcher): class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
__super_init = asyncore.dispatcher.__init__
__super_close = asyncore.dispatcher.close
__closed = 1 # Marker indicating that we're closed
socket = None # to outwit Sam's getattr __append=None # Marker indicating that we're closed
READ_SIZE = 8096 socket=None # to outwit Sam's getattr
def __init__(self, sock, addr, map=None, debug=None): def __init__(self, sock, addr, map=None, debug=None):
self.__super_init(sock, map) SizedMessageAsyncConnection.inheritedAttribute(
self.addr = addr '__init__')(self, sock, map)
self.addr=addr
if debug is not None: if debug is not None:
self._debug = debug self._debug=debug
elif not hasattr(self, '_debug'): elif not hasattr(self, '_debug'):
self._debug = __debug__ and 'smac' self._debug=__debug__ and 'smac'
self.__state = None self.__state=None
self.__inp = None # None, a single String, or a list self.__inp=None
self.__input_len = 0 self.__inpl=0
self.__msg_size = 4 self.__l=4
self.__output = [] self.__output=output=[]
self.__closed = None self.__append=output.append
self.__pop=output.pop
# XXX avoid expensive getattr calls?
def __nonzero__(self): def handle_read(self,
return 1 join=string.join, StringType=type(''), _type=type,
_None=None):
def handle_read(self):
# Use a single __inp buffer and integer indexes to make this
# fast.
try: try:
d=self.recv(8096) d=self.recv(8096)
except socket.error, err: except socket.error, err:
if err[0] in expected_socket_read_errors: if err[0] in expected_socket_read_errors:
return return
raise raise
if not d: if not d: return
return
input_len = self.__input_len + len(d)
msg_size = self.__msg_size
state = self.__state
inp = self.__inp inp=self.__inp
if msg_size > input_len: if inp is _None:
if inp is None: inp=d
self.__inp = d elif _type(inp) is StringType:
elif type(self.__inp) is StringType: inp=[inp,d]
self.__inp = [self.__inp, d]
else:
self.__inp.append(d)
self.__input_len = input_len
return # keep waiting for more input
# load all previous input and d into single string inp
if isinstance(inp, StringType):
inp = inp + d
elif inp is None:
inp = d
else: else:
inp.append(d) inp.append(d)
inp = "".join(inp)
offset = 0 inpl=self.__inpl+len(d)
while (offset + msg_size) <= input_len: l=self.__l
msg = inp[offset:offset + msg_size]
offset = offset + msg_size while 1:
if state is None:
if l <= inpl:
# Woo hoo, we have enough data
if _type(inp) is not StringType: inp=join(inp,'')
d=inp[:l]
inp=inp[l:]
inpl=inpl-l
if self.__state is _None:
# waiting for message # waiting for message
msg_size = struct.unpack(">i", msg)[0] l=struct.unpack(">i",d)[0]
state = 1 self.__state=1
else: else:
msg_size = 4 l=4
state = None self.__state=_None
self.message_input(msg) self.message_input(d)
else:
self.__state = state break # not enough data
self.__msg_size = msg_size
self.__inp = inp[offset:]
self.__input_len = input_len - offset
def readable(self): self.__l=l
return 1 self.__inp=inp
self.__inpl=inpl
def writable(self): def readable(self): return 1
if len(self.__output) == 0: def writable(self): return not not self.__output
return 0
else:
return 1
def handle_write(self): def handle_write(self):
output = self.__output output=self.__output
while output: while output:
v = output[0] v=output[0]
try: try:
n=self.send(v) n=self.send(v)
except socket.error, err: except socket.error, err:
...@@ -214,33 +191,42 @@ class SizedMessageAsyncConnection(asyncore.dispatcher): ...@@ -214,33 +191,42 @@ class SizedMessageAsyncConnection(asyncore.dispatcher):
break # we couldn't write anything break # we couldn't write anything
raise raise
if n < len(v): if n < len(v):
output[0] = v[n:] output[0]=v[n:]
break # we can't write any more break # we can't write any more
else: else:
del output[0] del output[0]
#break # waaa
def handle_close(self): def handle_close(self):
self.close() self.close()
def message_output(self, message): def message_output(self, message,
if __debug__: pack=struct.pack, len=len):
if self._debug: if self._debug:
if len(message) > 40: if len(message) > 40: m=message[:40]+' ...'
m = message[:40]+' ...' else: m=message
else:
m = message
LOG(self._debug, TRACE, 'message_output %s' % `m`) LOG(self._debug, TRACE, 'message_output %s' % `m`)
if self.__closed is not None: append=self.__append
raise Disconnected, ( if append is None:
"This action is temporarily unavailable." raise Disconnected("This action is temporarily unavailable.<p>")
"<p>"
) append(pack(">i",len(message))+message)
# do two separate appends to avoid copying the message string
self.__output.append(struct.pack(">i", len(message))) def log_info(self, message, type='info'):
self.__output.append(message) if type=='error': type=ERROR
else: type=INFO
LOG('ZEO', type, message)
log=log_info
def close(self): def close(self):
if self.__closed is None: if self.__append is not None:
self.__closed = 1 self.__append=None
self.__super_close() SizedMessageAsyncConnection.inheritedAttribute('close')(self)
class Disconnected(Exception):
"""The client has become disconnected from the server
"""
src/ZEO/zrpc2.py deleted 100644 → 0
View file @ 48bcb3a7
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7