Skip to content

Z
ZEO
Commit d8bc34cc authored by Stephan Richter's avatar Stephan Richter
Browse files
Options

Some work in progress on porting. It is going slowly.

parent 09db1b2e
Hide whitespace changes
Inline Side-by-side
Showing with 814 additions and 503 deletions
.gitignore 0 → 100644
View file @ d8bc34cc
*.pyc
__pycache__
src/*.egg-info
benchmark/*.egg-info
.installed.cfg
.tox
bin
develop-eggs
docs
parts
MANIFEST.in 0 → 100644
View file @ d8bc34cc
include *.rst
include *.txt
include *.py
include buildout.cfg
include tox.ini
recursive-include src *
global-exclude *.pyc
bootstrap.py
View file @ d8bc34cc
......@@ -18,49 +18,148 @@ The script accepts buildout command-line options, so you can
use the -c option to specify an alternate configuration file.
"""
import os, shutil, sys, tempfile, urllib2
import os, shutil, sys, tempfile
from optparse import OptionParser
tmpeggs = tempfile.mkdtemp()
ez = {}
exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
).read() in ez
ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
usage = '''\
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
import pkg_resources
Bootstraps a buildout-based project.
is_jython = sys.platform.startswith('java')
Simply run this script in a directory containing a buildout.cfg, using the
Python that you want bin/buildout to use.
if is_jython:
import subprocess
Note that by using --setup-source and --download-base to point to
local resources, you can keep this script from going over the network.
'''
cmd = 'from setuptools.command.easy_install import main; main()'
if sys.platform == 'win32':
cmd = '"%s"' % cmd # work around spawn lamosity on windows
parser = OptionParser(usage=usage)
parser.add_option("-v", "--version", help="use a specific zc.buildout version")
ws = pkg_resources.working_set
parser.add_option("-t", "--accept-buildout-test-releases",
dest='accept_buildout_test_releases',
action="store_true", default=False,
help=("Normally, if you do not specify a --version, the "
"bootstrap script and buildout gets the newest "
"*final* versions of zc.buildout and its recipes and "
"extensions for you. If you use this flag, "
"bootstrap and buildout will get the newest releases "
"even if they are alphas or betas."))
parser.add_option("-c", "--config-file",
help=("Specify the path to the buildout configuration "
"file to be used."))
parser.add_option("-f", "--find-links",
help=("Specify a URL to search for buildout releases"))
if is_jython:
assert subprocess.Popen(
[sys.executable] + ['-c', cmd, '-mqNxd', tmpeggs, 'zc.buildout'],
env = dict(os.environ,
PYTHONPATH=
ws.find(pkg_resources.Requirement.parse('setuptools')).location
),
).wait() == 0
else:
assert os.spawnle(
os.P_WAIT, sys.executable, sys.executable,
'-c', cmd, '-mqNxd', tmpeggs, 'zc.buildout',
dict(os.environ,
PYTHONPATH=
ws.find(pkg_resources.Requirement.parse('setuptools')).location
),
) == 0
options, args = parser.parse_args()
######################################################################
# load/install distribute
to_reload = False
try:
import pkg_resources, setuptools
if not hasattr(pkg_resources, '_distribute'):
to_reload = True
raise ImportError
except ImportError:
ez = {}
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
exec(urlopen('http://python-distribute.org/distribute_setup.py').read(), ez)
setup_args = dict(to_dir=tmpeggs, download_delay=0, no_fake=True)
ez['use_setuptools'](**setup_args)
if to_reload:
reload(pkg_resources)
import pkg_resources
# This does not (always?) update the default working set. We will
# do it.
for path in sys.path:
if path not in pkg_resources.working_set.entries:
pkg_resources.working_set.add_entry(path)
######################################################################
# Install buildout
ws = pkg_resources.working_set
cmd = [sys.executable, '-c',
'from setuptools.command.easy_install import main; main()',
'-mZqNxd', tmpeggs]
find_links = os.environ.get(
'bootstrap-testing-find-links',
options.find_links or
('http://downloads.buildout.org/'
if options.accept_buildout_test_releases else None)
)
if find_links:
cmd.extend(['-f', find_links])
distribute_path = ws.find(
pkg_resources.Requirement.parse('distribute')).location
requirement = 'zc.buildout'
version = options.version
if version is None and not options.accept_buildout_test_releases:
# Figure out the most recent final version of zc.buildout.
import setuptools.package_index
_final_parts = '*final-', '*final'
def _final_version(parsed_version):
for part in parsed_version:
if (part[:1] == '*') and (part not in _final_parts):
return False
return True
index = setuptools.package_index.PackageIndex(
search_path=[distribute_path])
if find_links:
index.add_find_links((find_links,))
req = pkg_resources.Requirement.parse(requirement)
if index.obtain(req) is not None:
best = []
bestv = None
for dist in index[req.project_name]:
distv = dist.parsed_version
if _final_version(distv):
if bestv is None or distv > bestv:
best = [dist]
bestv = distv
elif distv == bestv:
best.append(dist)
if best:
best.sort()
version = best[-1].version
if version:
requirement = '=='.join((requirement, version))
cmd.append(requirement)
import subprocess
if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=distribute_path)) != 0:
raise Exception(
"Failed to execute command:\n%s",
repr(cmd)[1:-1])
######################################################################
# Import and run buildout
ws.add_entry(tmpeggs)
ws.require('zc.buildout')
ws.require(requirement)
import zc.buildout.buildout
zc.buildout.buildout.main(sys.argv[1:] + ['bootstrap'])
if not [a for a in args if '=' not in a]:
args.append('bootstrap')
# if -c was provided, we push it back into args for buildout' main function
if options.config_file is not None:
args[0:0] = ['-c', options.config_file]
zc.buildout.buildout.main(args)
shutil.rmtree(tmpeggs)
buildout.cfg
View file @ d8bc34cc
[buildout]
develop = .
../ZODB
#find-links =
# ${buildout:directory}/ZODB-4.0.0dev.tar.gz
parts =
test
scripts
versions = versions
[versions]
zc.recipe.testrunner = 1.3.0
zdaemon = 4.0.0a1
[test]
recipe = zc.recipe.testrunner
......
setup.py
View file @ d8bc34cc
......@@ -11,9 +11,8 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
VERSION = "4.0.0dev"
"""Setup
"""
from setuptools import setup, find_packages
import os
import sys
......@@ -28,6 +27,9 @@ License :: OSI Approved :: Zope Public License
Programming Language :: Python
Programming Language :: Python :: 2.6
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Programming Language :: Python :: Implementation :: CPython
Topic :: Database
Topic :: Software Development :: Libraries :: Python Modules
Operating System :: Microsoft :: Windows
......@@ -77,23 +79,22 @@ long_description = (
open('CHANGES.txt').read()
)
setup(name="ZEO",
version='4.0.0dev',
description = long_description.split('\n', 2)[1],
long_description = long_description,
version=VERSION,
maintainer="Zope Foundation and Contributors",
maintainer_email="zodb-dev@zope.org",
packages = find_packages('src'),
package_dir = {'': 'src'},
license = "ZPL 2.1",
platforms = ["any"],
# description = doclines[0],
classifiers = filter(None, classifiers.split("\n")),
# long_description = long_description,
test_suite="__main__.alltests", # to support "setup.py test"
tests_require = tests_require,
extras_require = dict(test=tests_require),
install_requires = [
'ZODB',
'six',
'transaction',
'persistent',
'zc.lockfile',
......
src/ZEO/ClientStorage.py
View file @ d8bc34cc
......@@ -18,18 +18,7 @@ Public contents of this module:
ClientStorage -- the main class, implementing the Storage API
"""
from persistent.TimeStamp import TimeStamp
from ZEO.auth import get_module
from ZEO.cache import ClientCache
from ZEO.Exceptions import ClientStorageError, ClientDisconnected, AuthError
from ZEO import ServerStub
from ZEO.TransactionBuffer import TransactionBuffer
from ZEO.zrpc.client import ConnectionManager
from ZODB import POSException
from ZODB import utils
import BTrees.IOBTree
import cPickle
import logging
import os
import re
......@@ -37,7 +26,6 @@ import socket
import stat
import sys
import tempfile
import thread
import threading
import time
import weakref
......@@ -48,6 +36,17 @@ import ZODB.BaseStorage
import ZODB.interfaces
import ZODB.event
import zope.interface
import six
from persistent.TimeStamp import TimeStamp
from ZEO._compat import Pickler, Unpickler, get_ident
from ZEO.auth import get_module
from ZEO.cache import ClientCache
from ZEO.Exceptions import ClientStorageError, ClientDisconnected, AuthError
from ZEO import ServerStub
from ZEO.TransactionBuffer import TransactionBuffer
from ZEO.zrpc.client import ConnectionManager
from ZODB import POSException
from ZODB import utils
logger = logging.getLogger(__name__)
......@@ -689,7 +688,7 @@ class ClientStorage(object):
host = addr[0]
try:
canonical, aliases, addrs = socket.gethostbyaddr(host)
except socket.error, err:
except socket.error as err:
logger.debug("%s Error resolving host: %s (%s)",
self.__name__, host, err)
canonical = host
......@@ -1221,7 +1220,7 @@ class ClientStorage(object):
if self._cache is None:
return
for oid, _ in self._seriald.iteritems():
for oid, _ in six.iteritems(self._seriald):
self._cache.invalidate(oid, tid)
for oid, data in self._tbuf:
......@@ -1330,7 +1329,7 @@ class ClientStorage(object):
# setup tempfile to hold zeoVerify results and interim
# invalidation results
self._tfile = tempfile.TemporaryFile(suffix=".inv")
self._pickler = cPickle.Pickler(self._tfile, 1)
self._pickler = Pickler(self._tfile, 1)
self._pickler.fast = 1 # Don't use the memo
if self._connection.peer_protocol_version < 'Z309':
......@@ -1449,7 +1448,7 @@ class ClientStorage(object):
self._pickler.dump((None, None))
self._pickler = None
self._tfile.seek(0)
unpickler = cPickle.Unpickler(self._tfile)
unpickler = Unpickler(self._tfile)
min_tid = self._cache.getLastTid()
while 1:
tid, oids = unpickler.load()
......@@ -1682,12 +1681,12 @@ def _check_blob_cache_size(blob_dir, target):
except zc.lockfile.LockError:
# Someone is already cleaning up, so don't bother
logger.debug("%s Another thread is checking the blob cache size.",
thread.get_ident())
get_ident())
open(attempt_path, 'w').close() # Mark that we tried
return
logger.debug("%s Checking blob cache size. (target: %s)",
thread.get_ident(), target)
get_ident(), target)
try:
while 1:
......@@ -1714,7 +1713,7 @@ def _check_blob_cache_size(blob_dir, target):
files_by_atime[t] = []
files_by_atime[t].append(os.path.join(dirname, file_name))
logger.debug("%s blob cache size: %s", thread.get_ident(), size)
logger.debug("%s blob cache size: %s", get_ident(), size)
if size <= target:
if os.path.isfile(attempt_path):
......@@ -1723,7 +1722,7 @@ def _check_blob_cache_size(blob_dir, target):
except OSError:
pass # Sigh, windows
continue
logger.debug("%s -->", thread.get_ident())
logger.debug("%s -->", get_ident())
break
while size > target and files_by_atime:
......@@ -1735,7 +1734,7 @@ def _check_blob_cache_size(blob_dir, target):
lock = zc.lockfile.LockFile(lockfilename)
except zc.lockfile.LockError:
logger.debug("%s Skipping locked %s",
thread.get_ident(),
get_ident(),
os.path.basename(file_name))
continue # In use, skip
......@@ -1743,7 +1742,7 @@ def _check_blob_cache_size(blob_dir, target):
fsize = os.stat(file_name).st_size
try:
ZODB.blob.remove_committed(file_name)
except OSError, v:
except OSError as v:
pass # probably open on windows
else:
size -= fsize
......@@ -1754,7 +1753,7 @@ def _check_blob_cache_size(blob_dir, target):
break
logger.debug("%s reduced blob cache size: %s",
thread.get_ident(), size)
get_ident(), size)
finally:
check_lock.close()
......
src/ZEO/StorageServer.py
View file @ d8bc34cc
......@@ -19,22 +19,7 @@ file storage or Berkeley storage.
TODO: Need some basic access control-- a declaration of the methods
exported for invocation by the server.
"""
from __future__ import with_statement
from ZEO.Exceptions import AuthError
from ZEO.monitor import StorageStats, StatsServer
from ZEO.zrpc.connection import ManagedServerConnection, Delay, MTDelay, Result
from ZEO.zrpc.server import Dispatcher
from ZODB.ConflictResolution import ResolvedSerial
from ZODB.loglevels import BLATHER
from ZODB.POSException import StorageError, StorageTransactionError
from ZODB.POSException import TransactionError, ReadOnlyError, ConflictError
from ZODB.serialize import referencesf
from ZODB.utils import oid_repr, p64, u64, z64
import asyncore
import cPickle
import itertools
import logging
import os
......@@ -50,7 +35,19 @@ import ZODB.event
import ZODB.serialize
import ZODB.TimeStamp
import zope.interface
import six
from ZEO._compat import Pickler, Unpickler, PY3, BytesIO
from ZEO.Exceptions import AuthError
from ZEO.monitor import StorageStats, StatsServer
from ZEO.zrpc.connection import ManagedServerConnection, Delay, MTDelay, Result
from ZEO.zrpc.server import Dispatcher
from ZODB.ConflictResolution import ResolvedSerial
from ZODB.loglevels import BLATHER
from ZODB.POSException import StorageError, StorageTransactionError
from ZODB.POSException import TransactionError, ReadOnlyError, ConflictError
from ZODB.serialize import referencesf
from ZODB.utils import oid_repr, p64, u64, z64
logger = logging.getLogger('ZEO.StorageServer')
......@@ -92,7 +89,7 @@ class ZEOStorage:
# The authentication protocol may define extra methods.
self._extensions = {}
for func in self.extensions:
self._extensions[func.func_name] = None
self._extensions[func.__name__] = None
self._iterators = {}
self._iterator_ids = itertools.count()
# Stores the last item that was handed out for a
......@@ -605,7 +602,7 @@ class ZEOStorage:
self.storage.deleteObject(oid, serial, self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
except Exception as err:
self._op_error(oid, err, 'delete')
return err is None
......@@ -617,7 +614,7 @@ class ZEOStorage:
oid, serial, self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
except Exception as err:
self._op_error(oid, err, 'checkCurrentSerialInTransaction')
return err is None
......@@ -633,13 +630,14 @@ class ZEOStorage:
oid, serial, data, blobfile, '', self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
self._op_error(oid, err, 'store')
except Exception as error:
self._op_error(oid, error, 'store')
err = error
else:
if serial != "\0\0\0\0\0\0\0\0":
self.invalidated.append(oid)
if isinstance(newserial, str):
if isinstance(newserial, bytes):
newserial = [(oid, newserial)]
for oid, s in newserial or ():
......@@ -660,7 +658,7 @@ class ZEOStorage:
self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
except Exception as err:
self._op_error(oid, err, 'restore')
return err is None
......@@ -671,7 +669,7 @@ class ZEOStorage:
tid, oids = self.storage.undo(trans_id, self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
except Exception as err:
self._op_error(z64, err, 'undo')
else:
self.invalidated.extend(oids)
......@@ -682,7 +680,10 @@ class ZEOStorage:
def _marshal_error(self, error):
# Try to pickle the exception. If it can't be pickled,
# the RPC response would fail, so use something that can be pickled.
pickler = cPickle.Pickler()
if PY3:
pickler = Pickler(BytesIO(), 1)
else:
pickler = Pickler()
pickler.fast = 1
try:
pickler.dump(error, 1)
......@@ -695,14 +696,14 @@ class ZEOStorage:
# IStorageIteration support
def iterator_start(self, start, stop):
iid = self._iterator_ids.next()
iid = next(self._iterator_ids)
self._iterators[iid] = iter(self.storage.iterator(start, stop))
return iid
def iterator_next(self, iid):
iterator = self._iterators[iid]
try:
info = iterator.next()
info = next(iterator)
except StopIteration:
del self._iterators[iid]
item = None
......@@ -720,7 +721,7 @@ class ZEOStorage:
return item
def iterator_record_start(self, txn_iid, tid):
record_iid = self._iterator_ids.next()
record_iid = next(self._iterator_ids)
txn_info = self._txn_iterators_last[txn_iid]
if txn_info.tid != tid:
raise Exception(
......@@ -732,7 +733,7 @@ class ZEOStorage:
def iterator_record_next(self, iid):
iterator = self._iterators[iid]
try:
info = iterator.next()
info = next(iterator)
except StopIteration:
del self._iterators[iid]
item = None
......@@ -1183,7 +1184,7 @@ class StorageServer:
except:
pass
for name, storage in self.storages.iteritems():
for name, storage in six.iteritems(self.storages):
logger.info("closing storage %r", name)
storage.close()
......@@ -1566,7 +1567,7 @@ class CommitLog:
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".comit-log")
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler = Pickler(self.file, 1)
self.pickler.fast = 1
self.stores = 0
......@@ -1595,7 +1596,7 @@ class CommitLog:
def __iter__(self):
self.file.seek(0)
unpickler = cPickle.Unpickler(self.file)
unpickler = Unpickler(self.file)
for i in range(self.stores):
yield unpickler.load()
......
src/ZEO/TransactionBuffer.py
View file @ d8bc34cc
......@@ -23,9 +23,9 @@ is used to store the data until a commit or abort.
from threading import Lock
import os
import cPickle
import tempfile
import ZODB.blob
from ZEO._compat import Pickler, Unpickler
class TransactionBuffer:
......@@ -64,7 +64,7 @@ class TransactionBuffer:
self.blobs = []
# It's safe to use a fast pickler because the only objects
# stored are builtin types -- strings or None.
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler = Pickler(self.file, 1)
self.pickler.fast = 1
def close(self):
......@@ -137,7 +137,7 @@ class TBIterator(object):
def __init__(self, f, count):
self.file = f
self.count = count
self.unpickler = cPickle.Unpickler(f)
self.unpickler = Unpickler(f)
def __iter__(self):
return self
......
src/ZEO/_compat.py 0 → 100644
View file @ d8bc34cc
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Python versions compatiblity
"""
import sys
PY3 = sys.version_info[0] >= 3
# Pickle support
from ZODB._compat import Pickler, Unpickler, dump, dumps, loads
# String and Bytes IO
from ZODB._compat import BytesIO
if PY3:
import _thread as thread
from threading import get_ident
else:
import thread
from thread import get_ident
src/ZEO/auth/__init__.py
View file @ d8bc34cc
......@@ -19,12 +19,12 @@ def get_module(name):
from auth_sha import StorageClass, SHAClient, Database
return StorageClass, SHAClient, Database
elif name == 'digest':
from auth_digest import StorageClass, DigestClient, DigestDatabase
from .auth_digest import StorageClass, DigestClient, DigestDatabase
return StorageClass, DigestClient, DigestDatabase
else:
return _auth_modules.get(name)
def register_module(name, storage_class, client, db):
if _auth_modules.has_key(name):
if name in _auth_modules:
raise TypeError("%s is already registred" % name)
_auth_modules[name] = storage_class, client, db
src/ZEO/auth/auth_digest.py
View file @ d8bc34cc
......@@ -47,7 +47,7 @@ from ZEO.hash import sha1
def get_random_bytes(n=8):
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
f = open("/dev/urandom", 'rb')
s = f.read(n)
f.close()
else:
......@@ -56,7 +56,7 @@ def get_random_bytes(n=8):
return s
def hexdigest(s):
return sha1(s).hexdigest()
return sha1(s.encode()).hexdigest()
class DigestDatabase(Database):
def __init__(self, filename, realm=None):
......
src/ZEO/auth/base.py
View file @ d8bc34cc
......@@ -16,6 +16,8 @@
Database -- abstract base class for password database
Client -- abstract base class for authentication client
"""
from __future__ import print_function
from __future__ import print_function
import os
from ZEO.hash import sha1
......@@ -73,10 +75,10 @@ class Database:
if not fd:
fd = open(filename, 'w')
if self.realm:
print >> fd, "realm", self.realm
print("realm", self.realm, file=fd)
for username in sort(self._users.keys()):
print >> fd, "%s: %s" % (username, self._users[username])
for username in sorted(self._users.keys()):
print("%s: %s" % (username, self._users[username]), file=fd)
def load(self):
filename = self.filename
......@@ -108,24 +110,24 @@ class Database:
Callers must check for LookupError, which is raised in
the case of a non-existent user specified."""
if not self._users.has_key(username):
if username not in self._users:
raise LookupError("No such user: %s" % username)
return self._users[username]
def hash(self, s):
return sha1(s).hexdigest()
return sha1(s.encode()).hexdigest()
def add_user(self, username, password):
if self._users.has_key(username):
if username in self._users:
raise LookupError("User %s already exists" % username)
self._store_password(username, password)
def del_user(self, username):
if not self._users.has_key(username):
if username not in self._users:
raise LookupError("No such user: %s" % username)
del self._users[username]
def change_password(self, username, password):
if not self._users.has_key(username):
if username not in self._users:
raise LookupError("No such user: %s" % username)
self._store_password(username, password)
src/ZEO/auth/hmac.py
View file @ d8bc34cc
......@@ -2,6 +2,8 @@
Implements the HMAC algorithm as described by RFC 2104.
"""
from six.moves import map
from six.moves import zip
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
......
src/ZEO/cache.py
View file @ d8bc34cc
......@@ -21,7 +21,7 @@ store(), and invalidate(). It manages in-memory data structures that allow
it to map this richer API onto the simple key-based API of the lower-level
FileCache.
"""
from __future__ import print_function
from struct import pack, unpack
import BTrees.LLBTree
......@@ -35,6 +35,7 @@ import time
import ZODB.fsIndex
import zc.lockfile
from ZODB.utils import p64, u64, z64
import six
logger = logging.getLogger("ZEO.cache")
......@@ -78,7 +79,7 @@ logger = logging.getLogger("ZEO.cache")
# file's magic number - ZEC3 - indicating zeo cache version 4. The
# next eight bytes are the last transaction id.
magic = "ZEC3"
magic = b"ZEC3"
ZEC_HEADER_SIZE = 12
# Maximum block size. Note that while we are doing a store, we may
......@@ -91,15 +92,15 @@ max_block_size = (1<<31) - 1
# After the header, the file contains a contiguous sequence of blocks. All
# blocks begin with a one-byte status indicator:
#
# 'a'
# b'a'
# Allocated. The block holds an object; the next 4 bytes are >I
# format total block size.
#
# 'f'
# b'f'
# Free. The block is free; the next 4 bytes are >I format total
# block size.
#
# '1', '2', '3', '4'
# b'1', b'2', b'3', b'4'
# The block is free, and consists of 1, 2, 3 or 4 bytes total.
#
# "Total" includes the status byte, and size bytes. There are no
......@@ -108,7 +109,7 @@ max_block_size = (1<<31) - 1
# Allocated blocks have more structure:
#
# 1 byte allocation status ('a').
# 1 byte allocation status (b'a').
# 4 bytes block size, >I format.
# 8 byte oid
# 8 byte start_tid
......@@ -281,11 +282,11 @@ class ClientCache(object):
last = ofs = ZEC_HEADER_SIZE
first_free_offset = 0
current = self.current
status = ' '
status = b' '
while ofs < fsize:
seek(ofs)
status = read(1)
if status == 'a':
if status == b'a':
size, oid, start_tid, end_tid, lver = unpack(
">I8s8s8sH", read(30))
if ofs+size <= maxsize:
......@@ -301,18 +302,18 @@ class ClientCache(object):
# free block
if first_free_offset == 0:
first_free_offset = ofs
if status == 'f':
if status == b'f':
size, = unpack(">I", read(4))
if size > max_block_size:
# Oops, we either have an old cache, or a we
# crashed while storing. Split this block into two.
assert size <= max_block_size*2
seek(ofs+max_block_size)
write('f'+pack(">I", size-max_block_size))
write(b'f'+pack(">I", size-max_block_size))
seek(ofs)
write('f'+pack(">I", max_block_size))
write(b'f'+pack(">I", max_block_size))
sync(f)
elif status in '1234':
elif status in b'1234':
size = int(status)
else:
raise ValueError("unknown status byte value %s in client "
......@@ -329,9 +330,9 @@ class ClientCache(object):
size = maxsize-last
seek(last)
if size > 4:
write('f'+pack(">I", size))
write(b'f'+pack(">I", size))
else:
write("012345"[size])
write("012345"[size].encode())
sync(f)
ofs = maxsize
break
......@@ -340,19 +341,19 @@ class ClientCache(object):
assert ofs==fsize
# Make sure the OS really saves enough bytes for the file.
seek(self.maxsize - 1)
write('x')
write(b'x')
# add as many free blocks as are needed to fill the space
seek(ofs)
nfree = maxsize - ZEC_HEADER_SIZE
for i in range(0, nfree, max_block_size):
block_size = min(max_block_size, nfree-i)
write('f' + pack(">I", block_size))
write(b'f' + pack(">I", block_size))
seek(block_size-5, 1)
sync(self.f)
# There is always data to read and
assert last and status in ' f1234'
assert last and (status in b' f1234')
first_free_offset = last
else:
assert ofs==maxsize
......@@ -435,7 +436,7 @@ class ClientCache(object):
while nbytes > 0:
seek(ofs)
status = read(1)
if status == 'a':
if status == b'a':
size, oid, start_tid, end_tid = unpack(">I8s8s8s", read(28))
self._n_evicts += 1
self._n_evicted_bytes += size
......@@ -445,10 +446,10 @@ class ClientCache(object):
self._del_noncurrent(oid, start_tid)
self._len -= 1
else:
if status == 'f':
if status == b'f':
size = unpack(">I", read(4))[0]
else:
assert status in '1234'
assert status in b'1234'
size = int(status)
ofs += size
nbytes -= size
......@@ -469,7 +470,7 @@ class ClientCache(object):
raise ValueError("new last tid (%s) must be greater than "
"previous one (%s)"
% (u64(tid), u64(self.tid)))
assert isinstance(tid, str) and len(tid) == 8, tid
assert isinstance(tid, bytes) and len(tid) == 8, tid
self.tid = tid
self.f.seek(len(magic))
self.f.write(tid)
......@@ -498,7 +499,7 @@ class ClientCache(object):
self.f.seek(ofs)
read = self.f.read
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid)
assert status == b'a', (ofs, self.f.tell(), oid)
size, saved_oid, tid, end_tid, lver, ldata = unpack(
">I8s8s8sHI", read(34))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
......@@ -528,7 +529,7 @@ class ClientCache(object):
# Remove fromn old loc:
del self.current[oid]
self.f.seek(ofs)
self.f.write('f'+pack(">I", size))
self.f.write(b'f'+pack(">I", size))
# Write to new location:
self._store(oid, tid, None, data, size)
......@@ -558,7 +559,7 @@ class ClientCache(object):
self.f.seek(ofs)
read = self.f.read
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid, before_tid)
assert status == b'a', (ofs, self.f.tell(), oid, before_tid)
size, saved_oid, saved_tid, end_tid, lver, ldata = unpack(
">I8s8s8sHI", read(34))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
......@@ -598,7 +599,7 @@ class ClientCache(object):
seek(ofs)
read = self.f.read
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid)
assert status == b'a', (ofs, self.f.tell(), oid)
size, saved_oid, saved_tid, end_tid = unpack(
">I8s8s8s", read(28))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
......@@ -646,11 +647,11 @@ class ClientCache(object):
# free block following the end of the data record. That isn't
# expensive -- it's all a contiguous write.
if excess == 0:
extra = ''
extra = b''
elif excess < 5:
extra = "01234"[excess]
extra = "01234"[excess].encode()
else:
extra = 'f' + pack(">I", excess)
extra = b'f' + pack(">I", excess)
ofs = self.currentofs
seek = self.f.seek
......@@ -660,7 +661,7 @@ class ClientCache(object):
# Before writing data, we'll write a free block for the space freed.
# We'll come back with a last atomic write to rewrite the start of the
# allocated-block header.
write('f'+pack(">I", nfreebytes))
write(b'f'+pack(">I", nfreebytes))
# Now write the rest of the allocation block header and object data.
write(pack(">8s8s8sHI", oid, start_tid, end_tid or z64, 0, len(data)))
......@@ -671,7 +672,7 @@ class ClientCache(object):
# Now, we'll go back and rewrite the beginning of the
# allocated block header.
seek(ofs)
write('a'+pack(">I", size))
write(b'a'+pack(">I", size))
if end_tid:
self._set_noncurrent(oid, start_tid, ofs)
......@@ -706,14 +707,14 @@ class ClientCache(object):
self.f.seek(ofs)
read = self.f.read
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid)
assert status == b'a', (ofs, self.f.tell(), oid)
size, saved_oid, saved_tid, end_tid = unpack(">I8s8s8s", read(28))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
assert end_tid == z64, (ofs, self.f.tell(), oid)
del self.current[oid]
if tid is None:
self.f.seek(ofs)
self.f.write('f'+pack(">I", size))
self.f.write(b'f'+pack(">I", size))
# 0x1E = invalidate (hit, discarding current or non-current)
self._trace(0x1E, oid, tid)
self._len -= 1
......@@ -735,12 +736,12 @@ class ClientCache(object):
# depends on whether the caller may change the cache.
seek = self.f.seek
read = self.f.read
for oid, ofs in self.current.iteritems():
for oid, ofs in six.iteritems(self.current):
self._lock.acquire()
try:
seek(ofs)
status = read(1)
assert status == 'a', (ofs, self.f.tell(), oid)
assert status == b'a', (ofs, self.f.tell(), oid)
size, saved_oid, tid, end_tid = unpack(">I8s8s8s", read(28))
assert saved_oid == oid, (ofs, self.f.tell(), oid, saved_oid)
assert end_tid == z64, (ofs, self.f.tell(), oid)
......@@ -752,18 +753,18 @@ class ClientCache(object):
def dump(self):
from ZODB.utils import oid_repr
print "cache size", len(self)
print("cache size", len(self))
L = list(self.contents())
L.sort()
for oid, tid in L:
print oid_repr(oid), oid_repr(tid)
print "dll contents"
print(oid_repr(oid), oid_repr(tid))
print("dll contents")
L = list(self)
L.sort(lambda x, y: cmp(x.key, y.key))
for x in L:
end_tid = x.end_tid or z64
print oid_repr(x.key[0]), oid_repr(x.key[1]), oid_repr(end_tid)
print
print(oid_repr(x.key[0]), oid_repr(x.key[1]), oid_repr(end_tid))
print()
# If `path` isn't None (== we're using a persistent cache file), and
# envar ZEO_CACHE_TRACE is set to a non-empty value, try to open
......@@ -781,7 +782,7 @@ class ClientCache(object):
tfn = path + ".trace"
try:
_tracefile = open(tfn, "ab")
except IOError, msg:
except IOError as msg:
logger.warning("cannot write tracefile %r (%s)", tfn, msg)
else:
logger.info("opened tracefile %r", tfn)
......@@ -790,7 +791,7 @@ class ClientCache(object):
return
now = time.time
def _trace(code, oid="", tid=z64, end_tid=z64, dlen=0):
def _trace(code, oid=b"", tid=z64, end_tid=z64, dlen=0):
# The code argument is two hex digits; bits 0 and 7 must be zero.
# The first hex digit shows the operation, the second the outcome.
# This method has been carefully tuned to be as fast as possible.
......@@ -803,10 +804,10 @@ class ClientCache(object):
try:
_tracefile.write(
pack(">iiH8s8s",
now(), encoded, len(oid), tid, end_tid) + oid,
int(now()), encoded, len(oid), tid, end_tid) + oid,
)
except:
print `tid`, `end_tid`
print(repr(tid), repr(end_tid))
raise
self._trace = _trace
......
src/ZEO/monitor.py
View file @ d8bc34cc
......@@ -15,6 +15,22 @@
$Id$
"""
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import asyncore
import socket
......@@ -85,19 +101,19 @@ class StorageStats:
self.conflicts_resolved = int(value)
def dump(self, f):
print >> f, "Server started:", self.start
print >> f, "Clients:", self.clients
print >> f, "Clients verifying:", self.verifying_clients
print >> f, "Active transactions:", self.active_txns
print("Server started:", self.start, file=f)
print("Clients:", self.clients, file=f)
print("Clients verifying:", self.verifying_clients, file=f)
print("Active transactions:", self.active_txns, file=f)
if self.lock_time:
howlong = time.time() - self.lock_time
print >> f, "Commit lock held for:", int(howlong)
print >> f, "Commits:", self.commits
print >> f, "Aborts:", self.aborts
print >> f, "Loads:", self.loads
print >> f, "Stores:", self.stores
print >> f, "Conflicts:", self.conflicts
print >> f, "Conflicts resolved:", self.conflicts_resolved
print("Commit lock held for:", int(howlong), file=f)
print("Commits:", self.commits, file=f)
print("Aborts:", self.aborts, file=f)
print("Loads:", self.loads, file=f)
print("Stores:", self.stores, file=f)
print("Conflicts:", self.conflicts, file=f)
print("Conflicts resolved:", self.conflicts_resolved, file=f)
class StatsClient(asyncore.dispatcher):
......@@ -164,14 +180,14 @@ class StatsServer(asyncore.dispatcher):
f.close()
def dump(self, f):
print >> f, "ZEO monitor server version %s" % zeo_version
print >> f, time.ctime()
print >> f
print("ZEO monitor server version %s" % zeo_version, file=f)
print(time.ctime(), file=f)
print(file=f)
L = self.stats.keys()
L.sort()
for k in L:
stats = self.stats[k]
print >> f, "Storage:", k
print("Storage:", k, file=f)
stats.dump(f)
print >> f
print(file=f)
src/ZEO/runzeo.py
View file @ d8bc34cc
......@@ -29,6 +29,8 @@ Options:
Unless -C is specified, -a and -f are required.
"""
from __future__ import print_function
from __future__ import print_function
# The code here is designed to be reused by other, similar servers.
# For the forseeable future, it must work under Python 2.1 as well as
......@@ -256,7 +258,7 @@ class ZEOServer:
def loop_forever(self):
if self.options.testing_exit_immediately:
print "testing exit immediately"
print("testing exit immediately")
else:
self.server.loop()
......@@ -322,7 +324,7 @@ class ZEOServer:
if os.path.exists(pidfile):
os.unlink(pidfile)
f = open(pidfile, 'w')
print >> f, pid
print(pid, file=f)
f.close()
log("created PID file '%s'" % pidfile)
except IOError:
......
src/ZEO/scripts/cache_simul.py
View file @ d8bc34cc
......@@ -30,6 +30,18 @@ Note:
"""
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import bisect
import getopt
......@@ -42,10 +54,11 @@ from ZODB.utils import z64, u64
# we assign ctime locally to facilitate test replacement!
from time import ctime
import six
def usage(msg):
print >> sys.stderr, msg
print >> sys.stderr, __doc__
print(msg, file=sys.stderr)
print(__doc__, file=sys.stderr)
def main(args=None):
if args is None:
......@@ -58,7 +71,7 @@ def main(args=None):
interval_step = 15
try:
opts, args = getopt.getopt(args, "s:i:r:")
except getopt.error, msg:
except getopt.error as msg:
usage(msg)
return 2
......@@ -89,12 +102,12 @@ def main(args=None):
try:
import gzip
except ImportError:
print >> sys.stderr, "can't read gzipped files (no module gzip)"
print("can't read gzipped files (no module gzip)", file=sys.stderr)
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
elif filename == "-":
# Read from stdin.
......@@ -103,8 +116,8 @@ def main(args=None):
# Open regular file.
try:
f = open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
# Create simulation object.
......@@ -245,13 +258,13 @@ class Simulation(object):
extraname = "*** please override ***"
def printheader(self):
print "%s, cache size %s bytes" % (self.__class__.__name__,
addcommas(self.cachelimit))
print("%s, cache size %s bytes" % (self.__class__.__name__,
addcommas(self.cachelimit)))
self.extraheader()
extranames = tuple([s.upper() for s in self.extras])
args = ("START TIME", "DUR.", "LOADS", "HITS",
"INVALS", "WRITES", "HITRATE") + extranames
print self.format % args
print(self.format % args)
def extraheader(self):
pass
......@@ -267,13 +280,13 @@ class Simulation(object):
self.loads, self.hits, self.invals, self.writes,
hitrate(self.loads, self.hits))
args += tuple([getattr(self, name) for name in self.extras])
print self.format % args
print(self.format % args)
def finish(self):
# Make sure that the last line of output ends with "OVERALL". This
# makes it much easier for another program parsing the output to
# find summary statistics.
print '-'*74
print('-'*74)
if self.nreports < 2:
self.report()
else:
......@@ -288,7 +301,7 @@ class Simulation(object):
hitrate(self.total_loads, self.total_hits))
args += tuple([getattr(self, "total_" + name)
for name in self.extras])
print self.format % args
print(self.format % args)
# For use in CircularCacheSimulation.
......@@ -546,7 +559,7 @@ class CircularCacheSimulation(Simulation):
def report(self):
self.check()
free = used = total = 0
for size, e in self.filemap.itervalues():
for size, e in six.itervalues(self.filemap):
total += size
if e:
used += size
......@@ -571,12 +584,12 @@ class CircularCacheSimulation(Simulation):
assert pos == self.cachelimit
def dump(self):
print len(self.filemap)
print(len(self.filemap))
L = list(self.filemap)
L.sort()
for k in L:
v = self.filemap[k]
print k, v[0], repr(v[1])
print(k, v[0], repr(v[1]))
def roundup(size):
......
src/ZEO/scripts/cache_stats.py
View file @ d8bc34cc
from __future__ import print_function
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
......@@ -55,7 +56,6 @@ this file in the 'explain' dictionary. Note that the keys there (and
also the arguments to _trace() in ClientStorage.py) are 'code & 0x7e',
i.e. the low bit is always zero.
"""
import sys
import time
import getopt
......@@ -63,10 +63,11 @@ import struct
# we assign ctime locally to facilitate test replacement!
from time import ctime
import six
def usage(msg):
print >> sys.stderr, msg
print >> sys.stderr, __doc__
print(msg, file=sys.stderr)
print(__doc__, file=sys.stderr)
def main(args=None):
if args is None:
......@@ -81,7 +82,7 @@ def main(args=None):
heuristic = False
try:
opts, args = getopt.getopt(args, "hi:qsSvX")
except getopt.error, msg:
except getopt.error as msg:
usage(msg)
return 2
for o, a in opts:
......@@ -118,12 +119,12 @@ def main(args=None):
try:
import gzip
except ImportError:
print >> sys.stderr, "can't read gzipped files (no module gzip)"
print("can't read gzipped files (no module gzip)", file=sys.stderr)
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
elif filename == '-':
# Read from stdin
......@@ -132,8 +133,8 @@ def main(args=None):
# Open regular file
try:
f = open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
except IOError as msg:
print("can't open %s: %s" % (filename, msg), file=sys.stderr)
return 1
rt0 = time.time()
......@@ -142,7 +143,7 @@ def main(args=None):
records = 0 # number of trace records read
versions = 0 # number of trace records with versions
datarecords = 0 # number of records with dlen set
datasize = 0L # sum of dlen across records with dlen set
datasize = 0 # sum of dlen across records with dlen set
oids = {} # map oid to number of times it was loaded
bysize = {} # map data size to number of loads
bysizew = {} # map data size to number of writes
......@@ -158,8 +159,8 @@ def main(args=None):
FMT_SIZE = struct.calcsize(FMT)
assert FMT_SIZE == 26
# Read file, gathering statistics, and printing each record if verbose.
print ' '*16, "%7s %7s %7s %7s" % ('loads', 'hits', 'inv(h)', 'writes'),
print 'hitrate'
print(' '*16, "%7s %7s %7s %7s" % ('loads', 'hits', 'inv(h)', 'writes'), end=' ')
print('hitrate')
try:
while 1:
r = f_read(FMT_SIZE)
......@@ -169,7 +170,7 @@ def main(args=None):
if ts == 0:
# Must be a misaligned record caused by a crash.
if not quiet:
print "Skipping 8 bytes at offset", f.tell() - FMT_SIZE
print("Skipping 8 bytes at offset", f.tell() - FMT_SIZE)
f.seek(f.tell() - FMT_SIZE + 8)
continue
oid = f_read(oidlen)
......@@ -208,14 +209,14 @@ def main(args=None):
bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
if verbose:
print "%s %02x %s %016x %016x %c%s" % (
print("%s %02x %s %016x %016x %c%s" % (
ctime(ts)[4:-5],
code,
oid_repr(oid),
U64(start_tid),
U64(end_tid),
version,
dlen and (' '+str(dlen)) or "")
dlen and (' '+str(dlen)) or ""))
if code & 0x70 == 0x20:
oids[oid] = oids.get(oid, 0) + 1
total_loads += 1
......@@ -226,10 +227,10 @@ def main(args=None):
thisinterval = ts // interval
h0 = he = ts
if not quiet:
print ctime(ts)[4:-5],
print '='*20, "Restart", '='*20
print(ctime(ts)[4:-5], end=' ')
print('='*20, "Restart", '='*20)
except KeyboardInterrupt:
print "\nInterrupted. Stats so far:\n"
print("\nInterrupted. Stats so far:\n")
end_pos = f.tell()
f.close()
......@@ -239,74 +240,73 @@ def main(args=None):
# Error if nothing was read
if not records:
print >> sys.stderr, "No records processed"
print("No records processed", file=sys.stderr)
return 1
# Print statistics
if dostats:
print
print "Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0)
print "Versions: %s records used a version" % addcommas(versions)
print "First time: %s" % ctime(t0)
print "Last time: %s" % ctime(te)
print "Duration: %s seconds" % addcommas(te-t0)
print "Data recs: %s (%.1f%%), average size %d bytes" % (
print()
print("Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0))
print("Versions: %s records used a version" % addcommas(versions))
print("First time: %s" % ctime(t0))
print("Last time: %s" % ctime(te))
print("Duration: %s seconds" % addcommas(te-t0))
print("Data recs: %s (%.1f%%), average size %d bytes" % (
addcommas(datarecords),
100.0 * datarecords / records,
datasize / datarecords)
print "Hit rate: %.1f%% (load hits / loads)" % hitrate(bycode)
print
codes = bycode.keys()
codes.sort()
print "%13s %4s %s" % ("Count", "Code", "Function (action)")
datasize / datarecords))
print("Hit rate: %.1f%% (load hits / loads)" % hitrate(bycode))
print()
codes = sorted(bycode.keys())
print("%13s %4s %s" % ("Count", "Code", "Function (action)"))
for code in codes:
print "%13s %02x %s" % (
print("%13s %02x %s" % (
addcommas(bycode.get(code, 0)),
code,
explain.get(code) or "*** unknown code ***")
explain.get(code) or "*** unknown code ***"))
# Print histogram.
if print_histogram:
print
print "Histogram of object load frequency"
print()
print("Histogram of object load frequency")
total = len(oids)
print "Unique oids: %s" % addcommas(total)
print "Total loads: %s" % addcommas(total_loads)
print("Unique oids: %s" % addcommas(total))
print("Total loads: %s" % addcommas(total_loads))
s = addcommas(total)
width = max(len(s), len("objects"))
fmt = "%5d %" + str(width) + "s %5.1f%% %5.1f%% %5.1f%%"
hdr = "%5s %" + str(width) + "s %6s %6s %6s"
print hdr % ("loads", "objects", "%obj", "%load", "%cum")
print(hdr % ("loads", "objects", "%obj", "%load", "%cum"))
cum = 0.0
for binsize, count in histogram(oids):
obj_percent = 100.0 * count / total
load_percent = 100.0 * count * binsize / total_loads
cum += load_percent
print fmt % (binsize, addcommas(count),
obj_percent, load_percent, cum)
print(fmt % (binsize, addcommas(count),
obj_percent, load_percent, cum))
# Print size histogram.
if print_size_histogram:
print
print "Histograms of object sizes"
print
print()
print("Histograms of object sizes")
print()
dumpbysize(bysizew, "written", "writes")
dumpbysize(bysize, "loaded", "loads")
def dumpbysize(bysize, how, how2):
print
print "Unique sizes %s: %s" % (how, addcommas(len(bysize)))
print "%10s %6s %6s" % ("size", "objs", how2)
print()
print("Unique sizes %s: %s" % (how, addcommas(len(bysize))))
print("%10s %6s %6s" % ("size", "objs", how2))
sizes = bysize.keys()
sizes.sort()
for size in sizes:
loads = 0
for n in bysize[size].itervalues():
for n in six.itervalues(bysize[size]):
loads += n
print "%10s %6d %6d" % (addcommas(size),
print("%10s %6d %6d" % (addcommas(size),
len(bysize.get(size, "")),
loads)
loads))
def dumpbyinterval(byinterval, h0, he):
loads = hits = invals = writes = 0
......@@ -327,9 +327,9 @@ def dumpbyinterval(byinterval, h0, he):
else:
hr = 'n/a'
print "%s-%s %7s %7s %7s %7s %7s" % (
print("%s-%s %7s %7s %7s %7s %7s" % (
ctime(h0)[4:-8], ctime(he)[14:-8],
loads, hits, invals, writes, hr)
loads, hits, invals, writes, hr))
def hitrate(bycode):
loads = hits = 0
......@@ -346,7 +346,7 @@ def hitrate(bycode):
def histogram(d):
bins = {}
for v in d.itervalues():
for v in six.itervalues(d):
bins[v] = bins.get(v, 0) + 1
L = bins.items()
L.sort()
......
src/ZEO/scripts/parsezeolog.py
View file @ d8bc34cc
......@@ -5,6 +5,11 @@
An example of the log format is:
2002-04-15T13:05:29 BLATHER(-100) ZEO Server storea(3235680, [714], 235339406490168806) ('10.0.26.30', 45514)
"""
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import re
import time
......@@ -65,8 +70,8 @@ class TStats:
d_finish = self.finish - self.begin
else:
d_finish = "*"
print self.fmt % (time.ctime(self.begin), d_vote, d_finish,
self.user, self.url)
print(self.fmt % (time.ctime(self.begin), d_vote, d_finish,
self.user, self.url))
class TransactionParser:
......@@ -97,7 +102,7 @@ class TransactionParser:
try:
return self.txns[tid]
except KeyError:
print "uknown tid", repr(tid)
print("uknown tid", repr(tid))
return None
def tpc_finish(self, time, args):
......@@ -127,9 +132,9 @@ if __name__ == "__main__":
try:
p.parse(line)
except:
print "line", i
print("line", i)
raise
print "Transaction: %d" % len(p.txns)
print TStats.hdr
print("Transaction: %d" % len(p.txns))
print(TStats.hdr)
for txn in p.get_txns():
txn.report()
src/ZEO/scripts/timeout.py
View file @ d8bc34cc
......@@ -9,6 +9,11 @@ transaction timeout feature of the server.
usage: timeout.py address delay [storage-name]
"""
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import sys
import time
......@@ -44,9 +49,9 @@ def main():
port = int(port)
address = (host, port)
print "Connecting to %s..." % repr(address)
print("Connecting to %s..." % repr(address))
storage = ClientStorage(address, name)
print "Connected. Now starting a transaction..."
print("Connected. Now starting a transaction...")
oid = storage.new_oid()
revid = ZERO
......@@ -56,12 +61,12 @@ def main():
t.user = "timeout.py"
storage.tpc_begin(t)
storage.store(oid, revid, pickled_data, '', t)
print "Stored. Now voting..."
print("Stored. Now voting...")
storage.tpc_vote(t)
print "Voted; now sleeping %s..." % delay
print("Voted; now sleeping %s..." % delay)
time.sleep(delay)
print "Done."
print("Done.")
if __name__ == "__main__":
main()
src/ZEO/scripts/zeopack.py
View file @ d8bc34cc
......@@ -7,6 +7,8 @@ import sys
import time
import traceback
import ZEO.ClientStorage
from six.moves import map
from six.moves import zip
usage = """Usage: %prog [options] [servers]
......
src/ZEO/scripts/zeoqueue.py
View file @ d8bc34cc
......@@ -22,6 +22,7 @@ Options:
when you rotate log files so that the next run will parse from the
beginning of the file.
"""
from __future__ import print_function
import os
import re
......@@ -29,7 +30,7 @@ import sys
import time
import errno
import getopt
import cPickle as pickle
from ZEO._compat import load, dump
COMMASPACE = ', '
STATEFILE = 'zeoqueue.pck'
......@@ -165,7 +166,7 @@ class Status:
def process_file(self, fp):
if self.pos:
if VERBOSE:
print 'seeking to file position', self.pos
print('seeking to file position', self.pos)
fp.seek(self.pos)
while True:
line = fp.readline()
......@@ -237,7 +238,7 @@ class Status:
def call_vote(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
print("Oops!")
txn = self.txns[tid] = Txn(tid)
txn.vote = t
txn.voters.append(client)
......@@ -245,7 +246,7 @@ class Status:
def call_tpc_abort(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
print("Oops!")
txn = self.txns[tid] = Txn(tid)
txn.abort = t
txn.voters = []
......@@ -261,7 +262,7 @@ class Status:
def call_tpc_finish(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
print("Oops!")
txn = self.txns[tid] = Txn(tid)
txn.finish = t
txn.voters = []
......@@ -281,17 +282,17 @@ class Status:
self.commit = self.commit_or_abort = txn
def report(self):
print "Blocked transactions:", self.n_blocked
print("Blocked transactions:", self.n_blocked)
if not VERBOSE:
return
if self.t_restart:
print "Server started:", time.ctime(self.t_restart)
print("Server started:", time.ctime(self.t_restart))
if self.commit is not None:
t = self.commit_or_abort.finish
if t is None:
t = self.commit_or_abort.abort
print "Last finished transaction:", time.ctime(t)
print("Last finished transaction:", time.ctime(t))
# the blocked transaction should be the first one that calls vote
L = [(txn.begin, txn) for txn in self.txns.values()]
......@@ -301,18 +302,18 @@ class Status:
if txn.isactive():
began = txn.begin
if txn.voters:
print "Blocked client (first vote):", txn.voters[0]
print "Blocked transaction began at:", time.ctime(began)
print "Hint:", txn.hint
print "Idle time: %d sec" % int(time.time() - began)
print("Blocked client (first vote):", txn.voters[0])
print("Blocked transaction began at:", time.ctime(began))
print("Hint:", txn.hint)
print("Idle time: %d sec" % int(time.time() - began))
break
def usage(code, msg=''):
print >> sys.stderr, __doc__ % globals()
print(__doc__ % globals(), file=sys.stderr)
if msg:
print >> sys.stderr, msg
print(msg, file=sys.stderr)
sys.exit(code)
......@@ -327,7 +328,7 @@ def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'vhf:r0',
['help', 'verbose', 'file=', 'reset'])
except getopt.error, msg:
except getopt.error as msg:
usage(1, msg)
for opt, arg in opts:
......@@ -347,9 +348,9 @@ def main():
try:
os.unlink(file)
if VERBOSE:
print 'removing pickle state file', file
except OSError, e:
if e.errno <> errno.ENOENT:
print('removing pickle state file', file)
except OSError as e:
if e.errno != errno.ENOENT:
raise
return
......@@ -366,18 +367,18 @@ def main():
try:
statefp = open(file, 'rb')
try:
status = pickle.load(statefp)
status = load(statefp)
if VERBOSE:
print 'reading status from file', file
print('reading status from file', file)
finally:
statefp.close()
except IOError, e:
if e.errno <> errno.ENOENT:
except IOError as e:
if e.errno != errno.ENOENT:
raise
if status is None:
status = Status()
if VERBOSE:
print 'using new status'
print('using new status')
if not seek:
status.pos = 0
......@@ -389,7 +390,7 @@ def main():
fp.close()
# Save state
statefp = open(file, 'wb')
pickle.dump(status, statefp, 1)
dump(status, statefp, 1)
statefp.close()
# Print the report and return the number of blocked clients in the exit
# status code.
......
src/ZEO/scripts/zeoreplay.py
View file @ d8bc34cc
......@@ -24,6 +24,16 @@ Unlike parsezeolog.py, this script generates timestamps for each transaction,
and sub-command in the transaction. We can use this to compare timings with
synthesized data.
"""
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import re
import sys
......@@ -40,6 +50,7 @@ from ZODB.FileStorage import FileStorage
#from Standby.config import RS_PORT
from ZODB.Transaction import Transaction
from ZODB.utils import p64
from functools import reduce
datecre = re.compile('(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)')
methcre = re.compile("ZEO Server (\w+)\((.*)\) \('(.*)', (\d+)")
......@@ -50,9 +61,9 @@ class StopParsing(Exception):
def usage(code, msg=''):
print __doc__
print(__doc__)
if msg:
print msg
print(msg)
sys.exit(code)
......@@ -220,12 +231,12 @@ class ZEOParser:
bytes = reduce(operator.add, [size for oid, size in txn._objects])
else:
bytes = 0
print '%s %s %4d %10d %s %s' % (
print('%s %s %4d %10d %s %s' % (
txn._begintime, txn._finishtime - txn._begintime,
len(txn._objects),
bytes,
time.ctime(txn._begintime),
txn._url)
txn._url))
def replay(self):
for txn in self.__txns:
......@@ -238,16 +249,16 @@ class ZEOParser:
slower.append(txn)
else:
faster.append(txn)
print len(slower), 'laggards,', len(faster), 'on-time or faster'
print(len(slower), 'laggards,', len(faster), 'on-time or faster')
# Find some averages
if slower:
sum = reduce(operator.add,
[txn._replaydelta for txn in slower], 0)
print 'average slower txn was:', float(sum) / len(slower)
print('average slower txn was:', float(sum) / len(slower))
if faster:
sum = reduce(operator.add,
[txn._replaydelta for txn in faster], 0)
print 'average faster txn was:', float(sum) / len(faster)
print('average faster txn was:', float(sum) / len(faster))
......@@ -257,7 +268,7 @@ def main():
sys.argv[1:],
'hr:pm:',
['help', 'replay=', 'report', 'maxtxns='])
except getopt.error, e:
except getopt.error as e:
usage(1, e)
if args:
......@@ -298,16 +309,16 @@ def main():
except StopParsing:
break
except:
print 'input file line:', i
print('input file line:', i)
raise
t1 = now()
print 'total parse time:', t1-t0
print('total parse time:', t1-t0)
t2 = now()
if replay:
p.replay()
t3 = now()
print 'total replay time:', t3-t2
print 'total time:', t3-t0
print('total replay time:', t3-t2)
print('total time:', t3-t0)
......
src/ZEO/scripts/zeoserverlog.py
View file @ d8bc34cc
......@@ -143,8 +143,35 @@ Commands:
$Id$
"""
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import datetime, sys, re, os
from six.moves import map
from six.moves import zip
def time(line):
......@@ -197,7 +224,7 @@ def blocked_times(args):
if waiting == 0:
d = sub(t1, time(line))
if d >= thresh:
print t1, sub(t1, t2), cid, d
print(t1, sub(t1, t2), cid, d)
t1 = t2 = cid = blocking = waiting = last_wait = max_wait = 0
last_blocking = blocking
......@@ -222,11 +249,11 @@ def time_calls(f):
elif ' returns ' in line and t1 is not None:
d = sub(t1, time(line))
if d >= thresh:
print t1, d, connidre.search(line).group(1)
print(t1, d, connidre.search(line).group(1))
maxd = max(maxd, d)
t1 = None
print maxd
print(maxd)
def xopen(f):
if f == '-':
......@@ -268,7 +295,7 @@ def time_tpc(f):
t = time(line)
d = sub(t1, t)
if d >= thresh:
print 'a', t1, cid, sub(t1, t2), vs, sub(t2, t)
print('a', t1, cid, sub(t1, t2), vs, sub(t2, t))
del transactions[cid]
elif ' calling tpc_finish(' in line:
if cid in transactions:
......@@ -280,7 +307,7 @@ def time_tpc(f):
t = time(line)
d = sub(t1, t)
if d >= thresh:
print 'c', t1, cid, sub(t1, t2), vs, sub(t2, t3), sub(t3, t)
print('c', t1, cid, sub(t1, t2), vs, sub(t2, t3), sub(t3, t))
del transactions[cid]
......@@ -336,9 +363,9 @@ def time_trans(f):
t = time(line)
d = sub(t1, t)
if d >= thresh:
print t1, cid, "%s/%s" % (stores, old), \
print(t1, cid, "%s/%s" % (stores, old), \
sub(t0, t1), sub(t1, t2), vs, \
sub(t2, t), 'abort'
sub(t2, t), 'abort')
del transactions[cid]
elif ' calling tpc_finish(' in line:
if cid in transactions:
......@@ -350,9 +377,9 @@ def time_trans(f):
t = time(line)
d = sub(t1, t)
if d >= thresh:
print t1, cid, "%s/%s" % (stores, old), \
print(t1, cid, "%s/%s" % (stores, old), \
sub(t0, t1), sub(t1, t2), vs, \
sub(t2, t3), sub(t3, t)
sub(t2, t3), sub(t3, t))
del transactions[cid]
def minute(f, slice=16, detail=1, summary=1):
......@@ -365,8 +392,8 @@ def minute(f, slice=16, detail=1, summary=1):
cols = ["time", "reads", "stores", "commits", "aborts", "txns"]
fmt = "%18s %6s %6s %7s %6s %6s"
print fmt % cols
print fmt % ["-"*len(col) for col in cols]
print(fmt % cols)
print(fmt % ["-"*len(col) for col in cols])
mlast = r = s = c = a = cl = None
rs = []
......@@ -387,7 +414,7 @@ def minute(f, slice=16, detail=1, summary=1):
if m != mlast:
if mlast:
if detail:
print fmt % (mlast, len(cl), r, s, c, a, a+c)
print(fmt % (mlast, len(cl), r, s, c, a, a+c))
cls.append(len(cl))
rs.append(r)
ss.append(s)
......@@ -412,7 +439,7 @@ def minute(f, slice=16, detail=1, summary=1):
if mlast:
if detail:
print fmt % (mlast, len(cl), r, s, c, a, a+c)
print(fmt % (mlast, len(cl), r, s, c, a, a+c))
cls.append(len(cl))
rs.append(r)
ss.append(s)
......@@ -421,16 +448,16 @@ def minute(f, slice=16, detail=1, summary=1):
ts.append(c+a)
if summary:
print
print 'Summary: \t', '\t'.join(('min', '10%', '25%', 'med',
'75%', '90%', 'max', 'mean'))
print "n=%6d\t" % len(cls), '-'*62
print 'Clients: \t', '\t'.join(map(str,stats(cls)))
print 'Reads: \t', '\t'.join(map(str,stats(rs)))
print 'Stores: \t', '\t'.join(map(str,stats(ss)))
print 'Commits: \t', '\t'.join(map(str,stats(cs)))
print 'Aborts: \t', '\t'.join(map(str,stats(aborts)))
print 'Trans: \t', '\t'.join(map(str,stats(ts)))
print()
print('Summary: \t', '\t'.join(('min', '10%', '25%', 'med',
'75%', '90%', 'max', 'mean')))
print("n=%6d\t" % len(cls), '-'*62)
print('Clients: \t', '\t'.join(map(str,stats(cls))))
print('Reads: \t', '\t'.join(map(str,stats(rs))))
print('Stores: \t', '\t'.join(map(str,stats(ss))))
print('Commits: \t', '\t'.join(map(str,stats(cs))))
print('Aborts: \t', '\t'.join(map(str,stats(aborts))))
print('Trans: \t', '\t'.join(map(str,stats(ts))))
def stats(s):
s.sort()
......@@ -497,7 +524,7 @@ def verify(f):
t1, n = nv[cid]
if n:
d = sub(t1, time(line))
print cid, t1, n, d, n and (d*1000.0/n) or '-'
print(cid, t1, n, d, n and (d*1000.0/n) or '-')
def recovery(f):
f, = f
......@@ -520,17 +547,17 @@ def recovery(f):
else:
if trans:
if len(trans) > 1:
print " ... %s similar records skipped ..." % (
len(trans) - 1)
print n, last.strip()
print(" ... %s similar records skipped ..." % (
len(trans) - 1))
print(n, last.strip())
trans=[]
print n, line.strip()
print(n, line.strip())
last = line
if len(trans) > 1:
print " ... %s similar records skipped ..." % (
len(trans) - 1)
print n, last.strip()
print(" ... %s similar records skipped ..." % (
len(trans) - 1))
print(n, last.strip())
......
src/ZEO/scripts/zeoup.py
View file @ d8bc34cc
......@@ -25,6 +25,11 @@ Options:
You must specify either -p and -h or -U.
"""
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import getopt
import logging
......@@ -90,11 +95,11 @@ def check_server(addr, storage, write):
data, serial = cs.load("\0\0\0\0\0\0\0\0", "")
cs.close()
t1 = time.time()
print "Elapsed time: %.2f" % (t1 - t0)
print("Elapsed time: %.2f" % (t1 - t0))
def usage(exit=1):
print __doc__
print " ".join(sys.argv)
print(__doc__)
print(" ".join(sys.argv))
sys.exit(exit)
def main():
......@@ -119,11 +124,11 @@ def main():
write = 0
elif o == '-1':
ZEO_VERSION = 1
except Exception, err:
except Exception as err:
s = str(err)
if s:
s = ": " + s
print err.__class__.__name__ + s
print(err.__class__.__name__ + s)
usage()
if unix is not None:
......@@ -143,9 +148,9 @@ if __name__ == "__main__":
main()
except SystemExit:
raise
except Exception, err:
except Exception as err:
s = str(err)
if s:
s = ": " + s
print err.__class__.__name__ + s
print(err.__class__.__name__ + s)
sys.exit(1)
src/ZEO/tests/CommitLockTests.py
View file @ d8bc34cc
......@@ -140,7 +140,7 @@ class CommitLockTests:
def _get_timestamp(self):
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
return `t`
return repr(t)
class CommitLockVoteTests(CommitLockTests):
......
src/ZEO/tests/ConnectionTests.py
View file @ d8bc34cc
......@@ -1194,7 +1194,7 @@ class MSTThread(threading.Thread):
for c in clients:
# Check that we got serials for all oids
for oid in c.__oids:
testcase.failUnless(c.__serials.has_key(oid))
testcase.failUnless(oid in c.__serials)
# Check that we got serials for no other oids
for oid in c.__serials.keys():
testcase.failUnless(oid in c.__oids)
......@@ -1218,6 +1218,6 @@ else:
return '::1', forker.get_port(self)
_g = globals()
for name, value in _g.items():
for name, value in tuple(_g.items()):
if isinstance(value, type) and issubclass(value, CommonSetupTearDown):
_g[name+"V6"] = type(name+"V6", (V6Setup, value), {})
src/ZEO/tests/InvalidationTests.py
View file @ d8bc34cc
......@@ -25,6 +25,8 @@ from ZEO.tests.TestThread import TestThread
from ZODB.DB import DB
from ZODB.POSException import ReadConflictError, ConflictError
from six.moves import map
from six.moves import zip
# The tests here let several threads have a go at one or more database
# instances simultaneously. Each thread appends a disjoint (from the
......@@ -86,7 +88,7 @@ class StressTask:
self.tm.get().note("add key %s" % key)
try:
self.tm.get().commit()
except ConflictError, msg:
except ConflictError as msg:
self.tm.abort()
else:
if self.sleep:
......@@ -116,7 +118,7 @@ def _runTasks(rounds, *tasks):
commit(run, actions)
run.append(t)
t.doStep()
actions.append(`t.startnum`)
actions.append(repr(t.startnum))
commit(run,actions)
# stderr.write(' '.join(actions)+'\n')
finally:
......@@ -160,7 +162,7 @@ class StressThread(FailableThread):
self.commitdict[self] = 1
if self.sleep:
time.sleep(self.sleep)
except (ReadConflictError, ConflictError), msg:
except (ReadConflictError, ConflictError) as msg:
tm.abort()
else:
self.added_keys.append(key)
......@@ -205,14 +207,14 @@ class LargeUpdatesThread(FailableThread):
nkeys = len(tkeys)
if nkeys < 50:
tkeys = range(self.startnum, 3000, self.step)
tkeys = list(range(self.startnum, 3000, self.step))
nkeys = len(tkeys)
step = max(int(nkeys / 50), 1)
keys = [tkeys[i] for i in range(0, nkeys, step)]
for key in keys:
try:
tree[key] = self.threadnum
except (ReadConflictError, ConflictError), msg:
except (ReadConflictError, ConflictError) as msg:
# print "%d setting key %s" % (self.threadnum, msg)
transaction.abort()
break
......@@ -224,7 +226,7 @@ class LargeUpdatesThread(FailableThread):
self.commitdict[self] = 1
if self.sleep:
time.sleep(self.sleep)
except ConflictError, msg:
except ConflictError as msg:
# print "%d commit %s" % (self.threadnum, msg)
transaction.abort()
continue
......
src/ZEO/tests/IterationTests.py
View file @ d8bc34cc
......@@ -14,6 +14,7 @@
"""ZEO iterator protocol tests."""
import transaction
import six
class IterationTests:
......@@ -40,7 +41,7 @@ class IterationTests:
# At this point, a wrapping iterator might not have called the CS
# iterator yet. We'll consume one item to make sure this happens.
iterator.next()
six.advance_iterator(iterator)
self.assertEquals(1, len(self._storage._iterator_ids))
iid = list(self._storage._iterator_ids)[0]
self.assertEquals([], list(iterator))
......@@ -70,7 +71,7 @@ class IterationTests:
# We need to actually do some iteration to get the iterator created.
# We do a store to make sure the iterator isn't exhausted right away.
self._dostore()
self._storage.iterator().next()
six.advance_iterator(self._storage.iterator())
self.assertEquals(1, len(self._storage._iterator_ids))
iid = list(self._storage._iterator_ids)[0]
......@@ -87,7 +88,7 @@ class IterationTests:
# We need to actually do some iteration to get the iterator created.
# We do a store to make sure the iterator isn't exhausted right away.
self._dostore()
self._storage.iterator().next()
six.advance_iterator(self._storage.iterator())
iid = list(self._storage._iterator_ids)[0]
......@@ -104,7 +105,7 @@ class IterationTests:
# We need to actually do some iteration to get the iterator created.
# We do a store to make sure the iterator isn't exhausted right away.
self._dostore()
self._storage.iterator().next()
six.advance_iterator(self._storage.iterator())
iid = list(self._storage._iterator_ids)[0]
t = transaction.Transaction()
......@@ -120,11 +121,11 @@ class IterationTests:
self._dostore()
iter1 = self._storage.iterator()
iter2 = self._storage.iterator()
txn_info1 = iter1.next()
txn_info2 = iter2.next()
txn_info1 = six.advance_iterator(iter1)
txn_info2 = six.advance_iterator(iter2)
self.assertEquals(txn_info1.tid, txn_info2.tid)
txn_info1 = iter1.next()
txn_info2 = iter2.next()
txn_info1 = six.advance_iterator(iter1)
txn_info2 = six.advance_iterator(iter2)
self.assertEquals(txn_info1.tid, txn_info2.tid)
self.assertRaises(StopIteration, iter1.next)
self.assertRaises(StopIteration, iter2.next)
......
src/ZEO/tests/TestThread.py
View file @ d8bc34cc
......@@ -12,9 +12,9 @@
#
##############################################################################
"""A Thread base class for use with unittest."""
import threading
import sys
import six
class TestThread(threading.Thread):
"""Base class for defining threads that run from unittest.
......@@ -52,6 +52,6 @@ class TestThread(threading.Thread):
def cleanup(self, timeout=15):
self.join(timeout)
if self._exc_info:
raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
six.reraise(self._exc_info[0], self._exc_info[1], self._exc_info[2])
if self.isAlive():
self._testcase.fail("Thread did not finish: %s" % self)
src/ZEO/tests/forker.py
View file @ d8bc34cc
......@@ -12,7 +12,7 @@
#
##############################################################################
"""Library for forking storage server and connecting client storage"""
from __future__ import print_function
import os
import random
import sys
......@@ -21,12 +21,11 @@ import errno
import socket
import subprocess
import logging
import StringIO
import tempfile
import logging
import ZODB.tests.util
import zope.testing.setupstack
from ZEO._compat import BytesIO
logger = logging.getLogger('ZEO.tests.forker')
class ZEOConfig:
......@@ -50,39 +49,39 @@ class ZEOConfig:
self.loglevel = 'INFO'
def dump(self, f):
print >> f, "<zeo>"
print >> f, "address " + self.address
print("<zeo>", file=f)
print("address " + self.address, file=f)
if self.read_only is not None:
print >> f, "read-only", self.read_only and "true" or "false"
print("read-only", self.read_only and "true" or "false", file=f)
if self.invalidation_queue_size is not None:
print >> f, "invalidation-queue-size", self.invalidation_queue_size
print("invalidation-queue-size", self.invalidation_queue_size, file=f)
if self.invalidation_age is not None:
print >> f, "invalidation-age", self.invalidation_age
print("invalidation-age", self.invalidation_age, file=f)
if self.monitor_address is not None:
print >> f, "monitor-address %s:%s" % self.monitor_address
print("monitor-address %s:%s" % self.monitor_address, file=f)
if self.transaction_timeout is not None:
print >> f, "transaction-timeout", self.transaction_timeout
print("transaction-timeout", self.transaction_timeout, file=f)
if self.authentication_protocol is not None:
print >> f, "authentication-protocol", self.authentication_protocol
print("authentication-protocol", self.authentication_protocol, file=f)
if self.authentication_database is not None:
print >> f, "authentication-database", self.authentication_database
print("authentication-database", self.authentication_database, file=f)
if self.authentication_realm is not None:
print >> f, "authentication-realm", self.authentication_realm
print >> f, "</zeo>"
print("authentication-realm", self.authentication_realm, file=f)
print("</zeo>", file=f)
print >> f, """
print("""
<eventlog>
level %s
<logfile>
path %s
</logfile>
</eventlog>
""" % (self.loglevel, self.logpath)
""" % (self.loglevel, self.logpath), file=f)
def __str__(self):
f = StringIO.StringIO()
f = BytesIO()
self.dump(f)
return f.getvalue()
return f.getvalue().decode()
def encode_format(fmt):
......@@ -179,8 +178,8 @@ def start_zeo_server(storage_conf=None, zeo_conf=None, port=None, keep=False,
s.close()
logging.debug('acked: %s' % ack)
break
except socket.error, e:
if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
except socket.error as e:
if e.args[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
raise
s.close()
else:
......@@ -215,18 +214,18 @@ def shutdown_zeo_server(adminaddr):
if i > 0:
break
raise
except socket.error, e:
if (e[0] == errno.ECONNREFUSED
except socket.error as e:
if (e.args[0] == errno.ECONNREFUSED
or
# MAC OS X uses EINVAL when connecting to a port
# that isn't being listened on.
(sys.platform == 'darwin' and e[0] == errno.EINVAL)
(sys.platform == 'darwin' and e.args[0] == errno.EINVAL)
) and i > 0:
break
raise
try:
ack = s.recv(1024)
except socket.error, e:
except socket.error as e:
ack = 'no ack received'
logger.debug('shutdown_zeo_server(): acked: %s' % ack)
s.close()
......@@ -281,8 +280,8 @@ def get_port2(test):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind(('localhost', port+2))
except socket.error, e:
if e[0] != errno.EADDRINUSE:
except socket.error as e:
if e.args[0] != errno.EADDRINUSE:
raise
continue
......
src/ZEO/tests/servertesting.py
View file @ d8bc34cc
from __future__ import print_function
from __future__ import print_function
##############################################################################
#
# Copyright Zope Foundation and Contributors.
......@@ -57,7 +59,7 @@ class Connection:
self.addr = addr or 'test-addr-'+name
def close(self):
print self.name, 'closed'
print(self.name, 'closed')
self.connected = False
def poll(self):
......@@ -65,7 +67,7 @@ class Connection:
raise ZEO.zrpc.error.DisconnectedError()
def callAsync(self, meth, *args):
print self.name, 'callAsync', meth, repr(args)
print(self.name, 'callAsync', meth, repr(args))
callAsyncNoPoll = callAsync
......
src/ZEO/tests/speed.py
View file @ d8bc34cc
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
......@@ -68,7 +75,7 @@ class ZEOExit(asyncore.file_dispatcher):
self.delete_fs()
os._exit(0)
def handle_close(self):
print "Parent process exited unexpectedly"
print("Parent process exited unexpectedly")
self.delete_fs()
os._exit(0)
def delete_fs(self):
......@@ -88,7 +95,7 @@ def work(db, results, nrep, compress, data, detailed, minimize, threadno=None):
transaction.begin()
rt = jar.root()
key = 's%s' % r
if rt.has_key(key):
if key in rt:
p = rt[key]
else:
rt[key] = p =P()
......@@ -109,10 +116,10 @@ def work(db, results, nrep, compress, data, detailed, minimize, threadno=None):
t = time.time() - t
if detailed:
if threadno is None:
print "%s\t%s\t%.4f\t%d" % (j, r, t, conflicts)
print("%s\t%s\t%.4f\t%d" % (j, r, t, conflicts))
else:
print "%s\t%s\t%.4f\t%d\t%d" % (j, r, t, conflicts,
threadno)
print("%s\t%s\t%.4f\t%d\t%d" % (j, r, t, conflicts,
threadno))
results[r].append((t, conflicts))
rt=d=p=v=None # release all references
if minimize:
......@@ -168,7 +175,7 @@ def main(args):
cache_size=4000,
cache_deactivate_after=6000,)
print "Beginning work..."
print("Beginning work...")
results={1:[], 10:[], 100:[], 1000:[]}
if threads > 1:
import threading
......@@ -191,14 +198,14 @@ def main(args):
os.waitpid(pid, 0)
if detailed:
print '-'*24
print "num\tmean\tmin\tmax"
print('-'*24)
print("num\tmean\tmin\tmax")
for r in 1, 10, 100, 1000:
times = []
for time, conf in results[r]:
times.append(time)
t = mean(times)
print "%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times))
print("%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times)))
def mean(l):
tot = 0
......
src/ZEO/tests/stress.py
View file @ d8bc34cc
......@@ -16,6 +16,7 @@
The stress test should run in an infinite loop and should involve
multiple connections.
"""
from __future__ import print_function
# TODO: This code is currently broken.
import transaction
......@@ -91,7 +92,7 @@ def main():
while 1:
pid = start_child(zaddr)
print "started", pid
print("started", pid)
os.waitpid(pid, 0)
exitserver()
......
src/ZEO/tests/testAuth.py
View file @ d8bc34cc
......@@ -32,7 +32,7 @@ class AuthTest(CommonSetupTearDown):
def setUp(self):
fd, self.pwfile = tempfile.mkstemp('pwfile')
os.close(fd)
if self.realm:
self.pwdb = self.dbclass(self.pwfile, self.realm)
else:
......@@ -40,7 +40,7 @@ class AuthTest(CommonSetupTearDown):
self.pwdb.add_user("foo", "bar")
self.pwdb.save()
self._checkZEOpasswd()
self.__super_setUp()
def _checkZEOpasswd(self):
......
src/ZEO/tests/testConfig.py
View file @ d8bc34cc
......@@ -32,8 +32,8 @@ class ZEOConfigTest(ConfigTestBase):
from ZEO.ClientStorage import ClientDisconnected
import ZConfig
from ZODB.config import getDbSchema
from StringIO import StringIO
cfg = """
from ZEO._compat import BytesIO
cfg = b"""
<zodb>
<zeoclient>
server localhost:56897
......@@ -41,12 +41,12 @@ class ZEOConfigTest(ConfigTestBase):
</zeoclient>
</zodb>
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg))
config, handle = ZConfig.loadConfigFile(getDbSchema(), BytesIO(cfg))
self.assertEqual(config.database[0].config.storage.config.blob_dir,
None)
self.assertRaises(ClientDisconnected, self._test, cfg)
cfg = """
cfg = b"""
<zodb>
<zeoclient>
blob-dir blobs
......@@ -55,7 +55,7 @@ class ZEOConfigTest(ConfigTestBase):
</zeoclient>
</zodb>
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg))
config, handle = ZConfig.loadConfigFile(getDbSchema(), BytesIO(cfg))
self.assertEqual(config.database[0].config.storage.config.blob_dir,
'blobs')
self.assertRaises(ClientDisconnected, self._test, cfg)
......
src/ZEO/tests/testZEO.py
View file @ d8bc34cc
......@@ -12,6 +12,8 @@
#
##############################################################################
"""Test suite for ZEO based on ZODB.tests."""
from __future__ import print_function
from __future__ import print_function
from ZEO.ClientStorage import ClientStorage
from ZEO.tests.forker import get_port
......@@ -259,10 +261,10 @@ class GenericTests(
store.store(oid, revid, 'x', '', t)
store.tpc_vote(t)
store.tpc_finish(t)
except Exception, v:
except Exception as v:
import traceback
print 'E'*70
print v
print('E'*70)
print(v)
traceback.print_exception(*sys.exc_info())
finally:
store.close()
......
src/ZEO/tests/testZEO2.py
View file @ d8bc34cc
......@@ -40,7 +40,7 @@ Set up the storage with some initial blob data.
>>> fs = ZODB.FileStorage.FileStorage('t.fs', blob_dir='t.blobs')
>>> db = ZODB.DB(fs)
>>> conn = db.open()
>>> conn.root.b = ZODB.blob.Blob('x')
>>> conn.root.b = ZODB.blob.Blob(b'x')
>>> transaction.commit()
Get the iod and first serial. We'll use the serial later to provide
......@@ -48,7 +48,8 @@ out-of-date data.
>>> oid = conn.root.b._p_oid
>>> serial = conn.root.b._p_serial
>>> conn.root.b.open('w').write('y')
>>> with conn.root.b.open('w') as file:
... _ = file.write(b'y')
>>> transaction.commit()
>>> data = fs.load(oid)[0]
......@@ -63,7 +64,7 @@ And an initial client.
>>> zs1.notifyConnected(conn1)
>>> zs1.register('1', 0)
>>> zs1.tpc_begin('0', '', '', {})
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', '0')
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64, b'x', '0')
>>> _ = zs1.vote('0') # doctest: +ELLIPSIS
1 callAsync serialnos ...
......@@ -76,13 +77,13 @@ will conflict. It will be blocked at the vote call.
>>> zs2.register('1', 0)
>>> zs2.tpc_begin('1', '', '', {})
>>> zs2.storeBlobStart()
>>> zs2.storeBlobChunk('z')
>>> zs2.storeBlobChunk(b'z')
>>> zs2.storeBlobEnd(oid, serial, data, '1')
>>> delay = zs2.vote('1')
>>> class Sender:
... def send_reply(self, id, reply):
... print 'reply', id, reply
... print('reply', id, reply)
>>> delay.set_sender(1, Sender())
>>> logger = logging.getLogger('ZEO')
......@@ -90,7 +91,7 @@ will conflict. It will be blocked at the vote call.
>>> logger.setLevel(logging.INFO)
>>> logger.addHandler(handler)
Now, when we abort the transaction for the first client. the second
Now, when we abort the transaction for the first client. The second
client will be restarted. It will get a conflict error, that is
handled correctly:
......@@ -130,7 +131,7 @@ And an initial client.
Intentionally break zs1:
>>> zs1._store = lambda : None
>>> _ = zs1.vote('0') # doctest: +ELLIPSIS
>>> _ = zs1.vote('0') # doctest: +ELLIPSIS +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
TypeError: <lambda>() takes no arguments (3 given)
......
src/ZEO/tests/test_cache.py
View file @ d8bc34cc
......@@ -12,6 +12,7 @@
#
##############################################################################
"""Basic unit tests for a client cache."""
from __future__ import print_function
from ZODB.utils import p64, repr_to_oid
import doctest
......@@ -55,7 +56,7 @@ def hexprint(file):
hex = hex[:24] + ' ' + hex[24:]
hex = hex.ljust(49)
printable = printable.ljust(16)
print '%08x %s |%s|' % (offset, hex, printable)
print('%08x %s |%s|' % (offset, hex, printable))
offset += 16
......@@ -90,17 +91,17 @@ class CacheTests(ZODB.tests.util.TestCase):
# Check that setting tids out of order gives an error:
# the cache complains only when it's non-empty
self.cache.store(n1, n3, None, 'x')
self.cache.store(n1, n3, None, b'x')
self.assertRaises(ValueError, self.cache.setLastTid, n2)
def testLoad(self):
data1 = "data for n1"
data1 = b"data for n1"
self.assertEqual(self.cache.load(n1), None)
self.cache.store(n1, n3, None, data1)
self.assertEqual(self.cache.load(n1), (data1, n3))
def testInvalidate(self):
data1 = "data for n1"
data1 = b"data for n1"
self.cache.store(n1, n3, None, data1)
self.cache.invalidate(n2, n2)
self.cache.invalidate(n1, n4)
......@@ -109,8 +110,8 @@ class CacheTests(ZODB.tests.util.TestCase):
(data1, n3, n4))
def testNonCurrent(self):
data1 = "data for n1"
data2 = "data for n2"
data1 = b"data for n1"
data2 = b"data for n2"
self.cache.store(n1, n4, None, data1)
self.cache.store(n1, n2, n3, data2)
# can't say anything about state before n2
......@@ -124,11 +125,11 @@ class CacheTests(ZODB.tests.util.TestCase):
self.assertEqual(self.cache.loadBefore(n2, n4), None)
def testException(self):
self.cache.store(n1, n2, None, "data")
self.cache.store(n1, n2, None, "data")
self.cache.store(n1, n2, None, b"data")
self.cache.store(n1, n2, None, b"data")
self.assertRaises(ValueError,
self.cache.store,
n1, n3, None, "data")
n1, n3, None, b"data")
def testEviction(self):
# Manually override the current maxsize
......@@ -136,7 +137,7 @@ class CacheTests(ZODB.tests.util.TestCase):
# Trivial test of eviction code. Doesn't test non-current
# eviction.
data = ["z" * i for i in range(100)]
data = [b"z" * i for i in range(100)]
for i in range(50):
n = p64(i)
cache.store(n, n, None, data[i])
......@@ -151,9 +152,9 @@ class CacheTests(ZODB.tests.util.TestCase):
# are handled correctly.
def testSerialization(self):
self.cache.store(n1, n2, None, "data for n1")
self.cache.store(n3, n3, n4, "non-current data for n3")
self.cache.store(n3, n4, n5, "more non-current data for n3")
self.cache.store(n1, n2, None, b"data for n1")
self.cache.store(n3, n3, n4, b"non-current data for n3")
self.cache.store(n3, n4, n5, b"more non-current data for n3")
path = tempfile.mktemp()
# Copy data from self.cache into path, reaching into the cache
......@@ -205,20 +206,20 @@ class CacheTests(ZODB.tests.util.TestCase):
def testVeryLargeCaches(self):
cache = ZEO.cache.ClientCache('cache', size=(1<<32)+(1<<20))
cache.store(n1, n2, None, "x")
cache.store(n1, n2, None, b"x")
cache.close()
cache = ZEO.cache.ClientCache('cache', size=(1<<33)+(1<<20))
self.assertEquals(cache.load(n1), ('x', n2))
self.assertEquals(cache.load(n1), (b'x', n2))
cache.close()
def testConversionOfLargeFreeBlocks(self):
f = open('cache', 'wb')
f.write(ZEO.cache.magic+
'\0'*8 +
'f'+struct.pack(">I", (1<<32)-12)
b'\0'*8 +
b'f'+struct.pack(">I", (1<<32)-12)
)
f.seek((1<<32)-1)
f.write('x')
f.write(b'x')
f.close()
cache = ZEO.cache.ClientCache('cache', size=1<<32)
cache.close()
......@@ -226,7 +227,7 @@ class CacheTests(ZODB.tests.util.TestCase):
cache.close()
f = open('cache', 'rb')
f.seek(12)
self.assertEquals(f.read(1), 'f')
self.assertEquals(f.read(1), b'f')
self.assertEquals(struct.unpack(">I", f.read(4))[0],
ZEO.cache.max_block_size)
f.close()
......@@ -241,11 +242,11 @@ class CacheTests(ZODB.tests.util.TestCase):
def test_clear_zeo_cache(self):
cache = self.cache
for i in range(10):
cache.store(p64(i), n2, None, str(i))
cache.store(p64(i), n1, n2, str(i)+'old')
cache.store(p64(i), n2, None, str(i).encode())
cache.store(p64(i), n1, n2, str(i).encode()+b'old')
self.assertEqual(len(cache), 20)
self.assertEqual(cache.load(n3), ('3', n2))
self.assertEqual(cache.loadBefore(n3, n2), ('3old', n1, n2))
self.assertEqual(cache.load(n3), (b'3', n2))
self.assertEqual(cache.loadBefore(n3, n2), (b'3old', n1, n2))
cache.clear()
self.assertEqual(len(cache), 0)
......@@ -254,7 +255,7 @@ class CacheTests(ZODB.tests.util.TestCase):
def testChangingCacheSize(self):
# start with a small cache
data = 'x'
data = b'x'
recsize = ZEO.cache.allocated_record_overhead+len(data)
for extra in (2, recsize-2):
......@@ -285,7 +286,7 @@ class CacheTests(ZODB.tests.util.TestCase):
# always get a free block after a new allocated block.
expected_len = small - 1
self.assertEquals(len(cache), expected_len)
expected_oids = set(range(11, 50)+range(100, 110))
expected_oids = set(list(range(11, 50))+list(range(100, 110)))
self.assertEquals(
set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
......@@ -316,7 +317,7 @@ class CacheTests(ZODB.tests.util.TestCase):
# We use large-2 for the same reason we used small-1 above.
expected_len = large-2
self.assertEquals(len(cache), expected_len)
expected_oids = set(range(11, 50)+range(106, 110)+range(200, 305))
expected_oids = set(list(range(11, 50))+list(range(106, 110))+list(range(200, 305)))
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
......@@ -346,8 +347,13 @@ isn't corrupted. To see this, we'll write a little script that
writes records to a cache file repeatedly.
>>> import os, random, sys, time
>>> open('t', 'w').write('''
... import os, random, sys, thread, time
>>> with open('t', 'w') as file:
... _ = file.write('''
... import os, random, sys, time
... try:
... import thread
... except ImportError:
... import _thread as thread
... sys.path = %r
...
... def suicide():
......@@ -363,7 +369,7 @@ writes records to a cache file repeatedly.
... while 1:
... oid += 1
... t += 1
... data = 'X' * random.randint(5000,25000)
... data = b'X' * random.randint(5000,25000)
... cache.store(p64(oid), p64(t), None, data)
...
... ''' % sys.path)
......@@ -387,11 +393,11 @@ still be used.
>>> import ZEO.cache
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> data = 'X' * (1000 - ZEO.cache.ZEC_HEADER_SIZE - 41)
>>> data = b'X' * (1000 - ZEO.cache.ZEC_HEADER_SIZE - 41)
>>> cache.store(p64(1), p64(1), None, data)
>>> cache.close()
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache.store(p64(2), p64(2), None, 'XXX')
>>> cache.store(p64(2), p64(2), None, b'XXX')
>>> cache.close()
"""
......@@ -400,7 +406,8 @@ def cannot_open_same_cache_file_twice():
r"""
>>> import ZEO.cache
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache2 = ZEO.cache.ClientCache('cache', 1000)
>>> cache2 = ZEO.cache.ClientCache('cache', 1000) \
... # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
LockError: Couldn't lock 'cache.lock'
......@@ -415,7 +422,7 @@ def thread_safe():
>>> cache = ZEO.cache.ClientCache('cache', 1000000)
>>> for i in range(100):
... cache.store(ZODB.utils.p64(i), ZODB.utils.p64(1), None, '0')
... cache.store(ZODB.utils.p64(i), ZODB.utils.p64(1), None, b'0')
>>> import random, sys, threading
>>> random = random.Random(0)
......@@ -441,20 +448,20 @@ def thread_safe():
... for oid in range(100):
... oid = ZODB.utils.p64(oid)
... cache.invalidate(oid, ZODB.utils.p64(tid))
... cache.store(oid, ZODB.utils.p64(tid), None, str(tid))
... cache.store(oid, ZODB.utils.p64(tid), None, str(tid).encode())
>>> stop = True
>>> thread.join()
>>> if read_failure:
... print 'Read failure:'
... print('Read failure:')
... import traceback
... traceback.print_exception(*read_failure)
>>> expected = '9', ZODB.utils.p64(9)
>>> expected = b'9', ZODB.utils.p64(9)
>>> for oid in range(100):
... loaded = cache.load(ZODB.utils.p64(oid))
... if loaded != expected:
... print oid, loaded
... print(oid, loaded)
>>> cache.close()
......@@ -475,7 +482,7 @@ log, but will ignore the error and keep going.
>>> handler = logging.StreamHandler(sys.stdout)
>>> logger.addHandler(handler)
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache.store(ZODB.utils.p64(1), ZODB.utils.p64(1), None, '0')
>>> cache.store(ZODB.utils.p64(1), ZODB.utils.p64(1), None, b'0')
>>> cache.invalidate(ZODB.utils.p64(1), ZODB.utils.p64(2))
>>> cache._del_noncurrent(ZODB.utils.p64(1), ZODB.utils.p64(2))
... # doctest: +NORMALIZE_WHITESPACE
......@@ -512,12 +519,12 @@ Check to make sure the cache analysis scripts work.
>>> for i in range(1000):
... serial += 1
... oid = random.randint(i+1000, i+6000)
... history.append(('s', p64(oid), p64(serial),
... 'x'*random.randint(200,2000)))
... history.append((b's', p64(oid), p64(serial),
... b'x'*random.randint(200,2000)))
... for j in range(10):
... oid = random.randint(i+1000, i+6000)
... history.append(('l', p64(oid), p64(serial),
... 'x'*random.randint(200,2000)))
... history.append((b'l', p64(oid), p64(serial),
... b'x'*random.randint(200,2000)))
>>> def cache_run(name, size):
... serial = 1
......@@ -527,7 +534,7 @@ Check to make sure the cache analysis scripts work.
... cache = ZEO.cache.ClientCache(name, size*(1<<20))
... for action, oid, serial, data in history:
... now += 1
... if action == 's':
... if action == b's':
... cache.invalidate(oid, serial)
... cache.store(oid, serial, None, data)
... else:
......@@ -1036,10 +1043,10 @@ Set up evicted and then invalidated oid
>>> os.environ["ZEO_CACHE_TRACE"] = 'yes'
>>> cache = ZEO.cache.ClientCache('cache', 1<<21)
>>> cache.store(p64(1), p64(1), None, 'x')
>>> cache.store(p64(1), p64(1), None, b'x')
>>> for i in range(10):
... cache.store(p64(2+i), p64(1), None, 'x'*(1<<19)) # Evict 1
>>> cache.store(p64(1), p64(1), None, 'x')
... cache.store(p64(2+i), p64(1), None, b'x'*(1<<19)) # Evict 1
>>> cache.store(p64(1), p64(1), None, b'x')
>>> cache.invalidate(p64(1), p64(2))
>>> cache.load(p64(1))
>>> cache.close()
......@@ -1062,7 +1069,7 @@ Now try to do simulation:
def invalidations_with_current_tid_dont_wreck_cache():
"""
>>> cache = ZEO.cache.ClientCache('cache', 1000)
>>> cache.store(p64(1), p64(1), None, 'data')
>>> cache.store(p64(1), p64(1), None, b'data')
>>> import logging, sys
>>> handler = logging.StreamHandler(sys.stdout)
>>> logging.getLogger().addHandler(handler)
......@@ -1081,7 +1088,8 @@ def rename_bad_cache_file():
"""
An attempt to open a bad cache file will cause it to be dropped and recreated.
>>> open('cache', 'w').write('x'*100)
>>> with open('cache', 'w') as file:
... _ = file.write('x'*100)
>>> import logging, sys
>>> handler = logging.StreamHandler(sys.stdout)
>>> logging.getLogger().addHandler(handler)
......@@ -1094,32 +1102,33 @@ An attempt to open a bad cache file will cause it to be dropped and recreated.
...
ValueError: unexpected magic number: 'xxxx'
>>> cache.store(p64(1), p64(1), None, 'data')
>>> cache.store(p64(1), p64(1), None, b'data')
>>> cache.close()
>>> f = open('cache')
>>> f.seek(0, 2)
>>> print f.tell()
>>> _ = f.seek(0, 2)
>>> print(f.tell())
1000
>>> f.close()
>>> open('cache', 'w').write('x'*200)
>>> with open('cache', 'w') as file:
... _ = file.write('x'*200)
>>> cache = ZEO.cache.ClientCache('cache', 1000) # doctest: +ELLIPSIS
Removing bad cache file: 'cache' (prev bad exists).
Traceback (most recent call last):
...
ValueError: unexpected magic number: 'xxxx'
>>> cache.store(p64(1), p64(1), None, 'data')
>>> cache.store(p64(1), p64(1), None, b'data')
>>> cache.close()
>>> f = open('cache')
>>> f.seek(0, 2)
>>> print f.tell()
>>> _ = f.seek(0, 2)
>>> print(f.tell())
1000
>>> f.close()
>>> f = open('cache.bad')
>>> f.seek(0, 2)
>>> print f.tell()
>>> _ = f.seek(0, 2)
>>> print(f.tell())
100
>>> f.close()
......@@ -1134,8 +1143,9 @@ def test_suite():
doctest.DocTestSuite(
setUp=zope.testing.setupstack.setUpDirectory,
tearDown=zope.testing.setupstack.tearDown,
checker=zope.testing.renormalizing.RENormalizing([
(re.compile(r'31\.3%'), '31.2%'),
checker=ZODB.tests.util.checker + \
zope.testing.renormalizing.RENormalizing([
(re.compile(r'31\.3%'), '31.2%'),
]),
)
)
......
src/ZEO/tests/zeoserver.py
View file @ d8bc34cc
......@@ -109,7 +109,7 @@ class ZEOTestServer(asyncore.dispatcher):
sock.close()
os._exit(0)
self.log('continuing')
sock.send('X')
sock.send(b'X')
self._count -= 1
def register_socket(self, sock):
......@@ -190,7 +190,7 @@ def main():
try:
log(label, 'creating the test server, keep: %s', keep)
t = ZEOTestServer(test_addr, server, keep)
except socket.error, e:
except socket.error as e:
if e[0] != errno.EADDRINUSE:
raise
log(label, 'addr in use, closing and exiting')
......
src/ZEO/zeopasswd.py
View file @ d8bc34cc
......@@ -29,6 +29,8 @@ Additional options:
-d/--delete -- delete user instead of updating password
"""
from __future__ import print_function
from __future__ import print_function
import getopt
import getpass
......@@ -39,8 +41,8 @@ import ZConfig
import ZEO
def usage(msg):
print __doc__
print msg
print(__doc__)
print(msg)
sys.exit(2)
def options(args):
......@@ -50,7 +52,7 @@ def options(args):
"protocol=",
"filename=",
"realm"])
except getopt.error, msg:
except getopt.error as msg:
usage(msg)
config = None
delete = 0
......
src/ZEO/zrpc/_hmac.py
View file @ d8bc34cc
......@@ -5,6 +5,8 @@
Implements the HMAC algorithm as described by RFC 2104.
"""
from six.moves import map
from six.moves import zip
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
......
src/ZEO/zrpc/client.py
View file @ d8bc34cc
......@@ -28,6 +28,8 @@ from ZEO.zrpc.error import DisconnectedError
from ZODB.POSException import ReadOnlyError
from ZODB.loglevels import BLATHER
from six.moves import map
from six.moves import zip
def client_timeout():
......@@ -49,7 +51,7 @@ def client_loop(map):
try:
r, w, e = select.select(r, w, e, client_timeout())
except select.error, err:
except select.error as err:
if err[0] != errno.EINTR:
if err[0] == errno.EBADF:
......@@ -481,7 +483,7 @@ class ConnectThread(threading.Thread):
try:
r, w, x = select.select([], connecting, connecting, 1.0)
log("CT: select() %d, %d, %d" % tuple(map(len, (r,w,x))))
except select.error, msg:
except select.error as msg:
log("CT: select failed; msg=%s" % str(msg),
level=logging.WARNING)
continue
......@@ -547,7 +549,7 @@ class ConnectWrapper:
log("CW: attempt to connect to %s" % repr(addr))
try:
self.sock = socket.socket(domain, socket.SOCK_STREAM)
except socket.error, err:
except socket.error as err:
log("CW: can't create socket, domain=%s: %s" % (domain, err),
level=logging.ERROR)
self.close()
......@@ -560,7 +562,7 @@ class ConnectWrapper:
if self.state in ("opened", "connecting"):
try:
err = self.sock.connect_ex(self.addr)
except socket.error, msg:
except socket.error as msg:
log("CW: connect_ex(%r) failed: %s" % (self.addr, msg),
level=logging.ERROR)
self.close()
......
src/ZEO/zrpc/connection.py
View file @ d8bc34cc
......@@ -426,7 +426,7 @@ class Connection(smac.SizedMessageAsyncConnection, object):
ret = self.obj.loadEx(*args)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, msg:
except Exception as msg:
if not isinstance(msg, self.unlogged_exception_types):
self.log("%s() raised exception: %s" % (name, msg),
logging.ERROR, exc_info=True)
......@@ -471,7 +471,7 @@ class Connection(smac.SizedMessageAsyncConnection, object):
self.waiting_for_reply = False
except (SystemExit, KeyboardInterrupt):
raise
except Exception, msg:
except Exception as msg:
if not isinstance(msg, self.unlogged_exception_types):
self.log("%s() raised exception: %s" % (name, msg),
logging.ERROR, exc_info=True)
......@@ -661,11 +661,11 @@ def server_loop(map):
while len(map) > 1:
try:
asyncore.poll(30.0, map)
except Exception, v:
except Exception as v:
if v.args[0] != errno.EBADF:
raise
for o in map.values():
for o in tuple(map.values()):
o.close()
class ManagedClientConnection(Connection):
......
src/ZEO/zrpc/marshal.py
View file @ d8bc34cc
......@@ -11,10 +11,9 @@
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from cPickle import Unpickler, Pickler
from cStringIO import StringIO
import logging
from ZEO._compat import Unpickler, Pickler, BytesIO, PY3
from ZEO.zrpc.error import ZRPCError
from ZEO.zrpc.log import log, short_repr
......@@ -31,25 +30,38 @@ def encode(*args): # args: (msgid, flags, name, args)
# being represented by \xij escapes in proto 0).
# Undocumented: cPickle.Pickler accepts a lone protocol argument;
# pickle.py does not.
pickler = Pickler(1)
pickler.fast = 1
return pickler.dump(args, 1)
@apply
def fast_encode():
# Only use in cases where you *know* the data contains only basic
# Python objects
pickler = Pickler(1)
pickler.fast = 1
dump = pickler.dump
def fast_encode(*args):
return dump(args, 1)
return fast_encode
if PY3:
# XXX: Py3: Needs optimization.
f = BytesIO()
pickler = Pickler(f, 1)
pickler.fast = 1
pickler.dump(args)
res = f.getvalue()
return res
else:
pickler = Pickler(1)
pickler.fast = 1
return pickler.dump(args, 1)
if PY3:
# XXX: Py3: Needs optimization.
fast_encode = encode
else:
def fast_encode():
# Only use in cases where you *know* the data contains only basic
# Python objects
pickler = Pickler(1)
pickler.fast = 1
dump = pickler.dump
def fast_encode(*args):
return dump(args, 1)
return fast_encode
fast_encode = fast_encode()
def decode(msg):
"""Decodes msg and returns its parts"""
unpickler = Unpickler(StringIO(msg))
unpickler = Unpickler(BytesIO(msg))
unpickler.find_global = find_global
try:
......@@ -61,7 +73,7 @@ def decode(msg):
def server_decode(msg):
"""Decodes msg and returns its parts"""
unpickler = Unpickler(StringIO(msg))
unpickler = Unpickler(BytesIO(msg))
unpickler.find_global = server_find_global
try:
......@@ -80,7 +92,7 @@ def find_global(module, name):
"""Helper for message unpickler"""
try:
m = __import__(module, _globals, _globals, _silly)
except ImportError, msg:
except ImportError as msg:
raise ZRPCError("import error %s: %s" % (module, msg))
try:
......@@ -104,7 +116,7 @@ def server_find_global(module, name):
if module != 'ZopeUndo.Prefix':
raise ImportError
m = __import__(module, _globals, _globals, _silly)
except ImportError, msg:
except ImportError as msg:
raise ZRPCError("import error %s: %s" % (module, msg))
try:
......
src/ZEO/zrpc/server.py
View file @ d8bc34cc
......@@ -82,7 +82,7 @@ class Dispatcher(asyncore.dispatcher):
def handle_accept(self):
try:
sock, addr = self.accept()
except socket.error, msg:
except socket.error as msg:
log("accepted failed: %s" % msg)
return
......
src/ZEO/zrpc/smac.py
View file @ d8bc34cc
......@@ -27,10 +27,11 @@ is set to 1 and the MAC immediately follows the length.
import asyncore
import errno
import six
try:
import hmac
except ImportError:
import _hmac as hmac
from . import _hmac as hmac
import socket
import struct
import threading
......@@ -62,7 +63,7 @@ del tmp_dict
# that we could pass to send() without blocking.
SEND_SIZE = 60000
MAC_BIT = 0x80000000L
MAC_BIT = 0x80000000
_close_marker = object()
......@@ -165,7 +166,7 @@ class SizedMessageAsyncConnection(asyncore.dispatcher):
# Use a single __inp buffer and integer indexes to make this fast.
try:
d = self.recv(8192)
except socket.error, err:
except socket.error as err:
if err[0] in expected_socket_read_errors:
return
raise
......@@ -272,7 +273,7 @@ class SizedMessageAsyncConnection(asyncore.dispatcher):
return self.close()
else:
try:
message = message.next()
message = six.advance_iterator(message)
except StopIteration:
messages.pop(0)
else:
......@@ -284,7 +285,7 @@ class SizedMessageAsyncConnection(asyncore.dispatcher):
try:
n = self.send(v)
except socket.error, err:
except socket.error as err:
# Fix for https://bugs.launchpad.net/zodb/+bug/182833
# ensure the above mentioned "output" invariant
output.insert(0, v)
......
src/ZEO/zrpc/trigger.py
View file @ d8bc34cc
from __future__ import print_function
##############################################################################
#
# Copyright (c) 2001-2005 Zope Foundation and Contributors.
......@@ -17,10 +18,10 @@ from __future__ import with_statement
import asyncore
import os
import socket
import thread
import errno
from ZODB.utils import positive_id
from ZEO._compat import thread, get_ident
# Original comments follow; they're hard to follow in the context of
# ZEO's use of triggers. TODO: rewrite from a ZEO perspective.
......@@ -128,8 +129,8 @@ class _triggerbase(object):
thunk[0](*thunk[1:])
except:
nil, t, v, tbinfo = asyncore.compact_traceback()
print ('exception in trigger thunk:'
' (%s:%s %s)' % (t, v, tbinfo))
print(('exception in trigger thunk:'
' (%s:%s %s)' % (t, v, tbinfo)))
def __repr__(self):
return '<select-trigger (%s) at %x>' % (self.kind, positive_id(self))
......@@ -158,7 +159,7 @@ if os.name == 'posix':
asyncore.file_dispatcher.close(self)
def _physical_pull(self):
os.write(self.trigger, 'x')
os.write(self.trigger, b'x')
else:
# Windows version; uses just sockets, because a pipe isn't select'able
......@@ -204,7 +205,7 @@ else:
try:
w.connect(connect_address)
break # success
except socket.error, detail:
except socket.error as detail:
if detail[0] != errno.WSAEADDRINUSE:
# "Address already in use" is the only error
# I've seen on two WinXP Pro SP2 boxes, under
......
tox.ini 0 → 100644
View file @ d8bc34cc
[tox]
envlist =
py26,py27,py33
[testenv]
commands =
python setup.py test -q
deps =
{toxinidir}/ZODB-4.0.0dev.tar.gz
ZConfig
manuel
persistent
transaction
zc.lockfile
zdaemon
zope.interface
zope.testing
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7