Commit c7d58894 authored by Jim Fulton's avatar Jim Fulton

Removed the Berkeley DB storage from the Zope (and ZODB 3) repository.

The storage was added to these repositories hoping that that would
encourage people to test it. The storage is not (yet) usable in
production due to Berkeley DB log handling problems.  The storage is
a constant source of irritation due to errors in implicit algorithms
to decide whether to build BDB extensions when building Python and
due to implicit algorithms used to decide whether the BDB extension
was suitable for use by the storage when running tests.  The BDB
storage tests are also rather time consuming.

The final straw was the recent ZODB change that caused many BDB
storage tests to fail. :(
parent e376b855
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Requirements:
#
# All: BerkeleyDB 4.1.25, available from www.sleepycat.com
# Python 2.2: PyBSDDB 4.1.3 or better, from pybsddb.sf.net
# Python 2.3: nothing extra
try:
from bsddb import db
except ImportError:
try:
from bsddb3 import db
except ImportError:
db = None
# This flag tells other components whether Berkeley storage is available
is_available = bool(db)
# Useful constants
ZERO = '\0'*8
/*****************************************************************************
Copyright (c) 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#include <Python.h>
/* This helper only works for Python 2.2 and beyond. If we're using an
* older version of Python, stop out now so we don't leave a broken, but
* compiled and importable module laying about. BDBFullStorage.py has a
* workaround for when this extension isn't available.
*/
#if PY_VERSION_HEX < 0x020200F0
#error "Must be using at least Python 2.2"
#endif
/* Increment an 8-byte unsigned integer (represented as an 8-byte raw string),
* by a Python integer.
* The arguments are an 8-byte Python string, and a Python int or long.
* The result is an 8-byte Python string, representing their sum.
* XXX It's unclear what this intends to do if the sum overflows an 8-byte
* XXX unsigned integer. _PyLong_AsByteArray should raise OverflowError then.
*/
static PyObject*
helper_incr(PyObject* self, PyObject* args)
{
PyObject *pylong = NULL, *incr, *sum = NULL, *result = NULL;
char *s, x[8];
int len, res;
if (!PyArg_ParseTuple(args, "s#O:incr", &s, &len, &incr))
return NULL;
assert(len == 8);
/* There seems to be no direct route from byte array to long long, so
* first convert it to a PyLongObject*, then convert /that/ thing to a
* long long.
*/
pylong = _PyLong_FromByteArray(s, len,
0 /* big endian */,
0 /* unsigned */);
if (!pylong)
return NULL;
sum = PyNumber_Add(pylong, incr);
if (!sum)
goto err;
res = _PyLong_AsByteArray((PyLongObject*)sum, x, 8,
0 /* big endian */,
0 /* unsigned */);
if (res < 0)
goto err;
result = PyString_FromStringAndSize(x, 8);
err:
Py_XDECREF(pylong);
Py_XDECREF(sum);
return result;
}
static PyMethodDef helper_methods[] = {
{"incr", helper_incr, METH_VARARGS},
{NULL, NULL} /* sentinel */
};
DL_EXPORT(void)
init_helper(void)
{
(void)Py_InitModule("_helper", helper_methods);
}
import os
import pprint
from hotshot.log import LogReader
prevloc = None
byline = {}
log = LogReader('profile.dat')
for what, place, tdelta in log:
byline[prevloc] = tdelta + byline.get(prevloc, 0)
byline.setdefault(place,0)
prevloc = place
# Sort
results = [(v, k) for k, v in byline.items()]
results.sort()
for usecs, place in results:
if not place:
print 'Bad unpack:', usecs, place
continue
filename, line, funcname = place
print '%08d' % usecs, os.path.split(filename)[1], line
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Basic test framework class for both the BDBFullStorage and BDBMinimalStorage
# Berkeley storages
import os
import shutil
import sys
import unittest
import BDBStorage
from BDBStorage.BerkeleyBase import BerkeleyConfig
from ZODB.tests.StorageTestBase import StorageTestBase
DBHOME = 'test-db'
class BerkeleyTestBase(StorageTestBase):
def _config(self, read_only=False):
# Checkpointing just slows the tests down because we have to wait for
# the thread to properly shutdown. This can take up to 10 seconds, so
# for the purposes of the test suite we shut off this thread.
config = BerkeleyConfig()
config.interval = 0
config.read_only = read_only
return config
def _envdir(self):
return DBHOME
def open(self, read_only=False):
self._storage = self.ConcreteStorage(
self._envdir(), config=self._config(read_only))
def _zap_dbhome(self, dir=None):
if dir is None:
dir = self._envdir()
if os.path.isdir(dir):
shutil.rmtree(dir)
def _mk_dbhome(self, dir=None):
if dir is None:
dir = self._get_envdir()
os.mkdir(dir)
try:
return self.ConcreteStorage(dir, config=self._config())
except:
self._zap_dbhome()
raise
def setUp(self):
StorageTestBase.setUp(self)
self._zap_dbhome()
self.open()
def tearDown(self):
StorageTestBase.tearDown(self)
self._zap_dbhome()
class MinimalTestBase(BerkeleyTestBase):
from BDBStorage import BDBMinimalStorage
ConcreteStorage = BDBMinimalStorage.BDBMinimalStorage
class FullTestBase(BerkeleyTestBase):
from BDBStorage import BDBFullStorage
ConcreteStorage = BDBFullStorage.BDBFullStorage
def makeSuite(*args,**kw):
prefix = kw.get('prefix','check')
level = kw.get('level')
suite = unittest.TestSuite()
if level:
suite.level = level
if BDBStorage.is_available:
for klass in args:
suite.addTest(unittest.makeSuite(klass, prefix))
else:
sys.stderr.write("BDBStorage not available, tests disabled\n")
return suite
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Base class for unit tests at the ZODB layer
import os
from ZODB import DB
from BDBStorage.tests.BerkeleyTestBase import BerkeleyTestBase
DBHOME = 'test-db'
class ZODBTestBase(BerkeleyTestBase):
def setUp(self):
BerkeleyTestBase.setUp(self)
self._db = None
try:
self._db = DB(self._storage)
self._conn = self._db.open()
self._root = self._conn.root()
except:
self.tearDown()
raise
def _close(self):
if self._db is not None:
self._db.close()
self._db = self._storage = self._conn = self._root = None
def tearDown(self):
# If the tests exited with any uncommitted objects, they'll blow up
# subsequent tests because the next transaction commit will try to
# commit those object. But they're tied to closed databases, so
# that's broken. Aborting the transaction now saves us the headache.
try:
get_transaction().abort()
self._close()
finally:
BerkeleyTestBase.tearDown(self)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
This diff is collapsed.
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Unit test for database creation
import os
import time
import unittest
import BDBStorage
if BDBStorage.is_available:
from BDBStorage.BerkeleyBase import BerkeleyConfig
from BDBStorage.BDBFullStorage import BDBFullStorage
from BDBStorage.tests import BerkeleyTestBase
class TestMixin:
def testDBHomeExists(self):
self.failUnless(os.path.isdir(BerkeleyTestBase.DBHOME))
class MinimalCreateTest(BerkeleyTestBase.MinimalTestBase, TestMixin):
pass
class FullCreateTest(BerkeleyTestBase.FullTestBase, TestMixin):
pass
class FullOpenExistingTest(BerkeleyTestBase.FullTestBase):
def testOpenWithExistingVersions(self):
version = 'test-version'
oid = self._storage.new_oid()
revid = self._dostore(oid, data=7, version=version)
# Now close the current storage and re-open it
self._storage.close()
self._storage = self.ConcreteStorage(BerkeleyTestBase.DBHOME)
self.assertEqual(self._storage.modifiedInVersion(oid), version)
def testOpenAddVersion(self):
eq = self.assertEqual
version1 = 'test-version'
oid1 = self._storage.new_oid()
revid = self._dostore(oid1, data=7, version=version1)
# Now close the current storage and re-open it
self._storage.close()
self._storage = self.ConcreteStorage(BerkeleyTestBase.DBHOME)
eq(self._storage.modifiedInVersion(oid1), version1)
# Now create a 2nd version string, then close/reopen
version2 = 'new-version'
oid2 = self._storage.new_oid()
revid = self._dostore(oid2, data=8, version=version2)
# Now close the current storage and re-open it
self._storage.close()
self._storage = self.ConcreteStorage(BerkeleyTestBase.DBHOME)
eq(self._storage.modifiedInVersion(oid1), version1)
# Now create a 2nd version string, then close/reopen
eq(self._storage.modifiedInVersion(oid2), version2)
class FullOpenCloseTest(BerkeleyTestBase.FullTestBase):
level = 2
def _mk_dbhome(self, dir):
config = BerkeleyConfig
config.interval = 10
os.mkdir(dir)
try:
return self.ConcreteStorage(dir, config=config)
except:
self._zap_dbhome(dir)
raise
def testCloseWithCheckpointingThread(self):
# All the interesting stuff happens in the setUp and tearDown
time.sleep(20)
class OpenRecoveryTest(BerkeleyTestBase.FullTestBase):
def open(self):
self._storage = None
def testOpenWithBogusConfig(self):
class C: pass
c = C()
# This instance won't have the necessary attributes, so the creation
# will fail. We want to be sure that everything gets cleaned up
# enough to fix that and create a proper storage.
dir = self._envdir()
self.assertRaises(AttributeError, BDBFullStorage, dir, config=c)
c = BerkeleyConfig()
s = BDBFullStorage(dir, config=c)
s.close()
def test_suite():
return BDBStorage.tests.BerkeleyTestBase.makeSuite(
MinimalCreateTest,
FullCreateTest,
FullOpenExistingTest,
FullOpenCloseTest,
OpenRecoveryTest,
prefix='test'
)
return suite
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Unit tests for basic storage functionality
import unittest
from ZODB import POSException
import BDBStorage
from BDBStorage.tests import BerkeleyTestBase
from ZODB.tests.BasicStorage import BasicStorage
from ZODB.tests.HistoryStorage import HistoryStorage
from ZODB.tests.IteratorStorage import IteratorStorage, ExtendedIteratorStorage
from ZODB.tests.MTStorage import MTStorage
from ZODB.tests.PackableStorage import PackableStorage, PackableUndoStorage
from ZODB.tests.PersistentStorage import PersistentStorage
from ZODB.tests.ReadOnlyStorage import ReadOnlyStorage
from ZODB.tests.RecoveryStorage import RecoveryStorage
from ZODB.tests.RevisionStorage import RevisionStorage
from ZODB.tests.Synchronization import SynchronizedStorage
from ZODB.tests.TransactionalUndoStorage import TransactionalUndoStorage
from ZODB.tests.TransactionalUndoVersionStorage import \
TransactionalUndoVersionStorage
from ZODB.tests.VersionStorage import VersionStorage
from ZODB.tests import ConflictResolution
class MinimalTest(BerkeleyTestBase.MinimalTestBase,
BasicStorage,
MTStorage,
PackableStorage,
ReadOnlyStorage,
SynchronizedStorage,
):
def checkVersionedStoreAndLoad(self):
# This storage doesn't support versions, so we should get an exception
oid = self._storage.new_oid()
self.assertRaises(POSException.Unsupported,
self._dostore,
oid, data=11, version='a version')
class FullTest(BerkeleyTestBase.FullTestBase, BasicStorage,
RevisionStorage, VersionStorage,
TransactionalUndoStorage,
TransactionalUndoVersionStorage,
PackableStorage, PackableUndoStorage,
HistoryStorage,
IteratorStorage, ExtendedIteratorStorage,
ConflictResolution.ConflictResolvingStorage,
ConflictResolution.ConflictResolvingTransUndoStorage,
SynchronizedStorage,
PersistentStorage,
MTStorage,
ReadOnlyStorage):
pass
DST_DBHOME = 'test-dst'
class FullRecoveryTest(BerkeleyTestBase.FullTestBase,
RecoveryStorage):
def setUp(self):
BerkeleyTestBase.FullTestBase.setUp(self)
self._zap_dbhome(DST_DBHOME)
self._dst = self._mk_dbhome(DST_DBHOME)
def tearDown(self):
BerkeleyTestBase.FullTestBase.tearDown(self)
self._dst.close()
self._zap_dbhome(DST_DBHOME)
def new_dest(self):
self._zap_dbhome(DST_DBHOME)
return self._mk_dbhome(DST_DBHOME)
def test_suite():
return BDBStorage.tests.BerkeleyTestBase.makeSuite(
FullTest,
FullRecoveryTest,
MinimalTest,
)
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Test creation of a brand new database, and insertion of root objects.
import unittest
import BDBStorage
from BDBStorage.tests.ZODBTestBase import ZODBTestBase
from persistent.mapping import PersistentMapping
class InsertMixin:
def checkIsEmpty(self):
self.failUnless(not self._root.has_key('names'))
def checkNewInserts(self):
self._root['names'] = names = PersistentMapping()
names['Warsaw'] = 'Barry'
names['Hylton'] = 'Jeremy'
get_transaction().commit()
class FullNewInsertsTest(ZODBTestBase, InsertMixin):
from BDBStorage import BDBFullStorage
ConcreteStorage = BDBFullStorage.BDBFullStorage
class MinimalNewInsertsTest(ZODBTestBase, InsertMixin):
from BDBStorage import BDBMinimalStorage
ConcreteStorage = BDBMinimalStorage.BDBMinimalStorage
def test_suite():
return BDBStorage.tests.BerkeleyTestBase.makeSuite(
MinimalNewInsertsTest,
FullNewInsertsTest,
)
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Whitebox testing of storage implementation details.
import unittest
from ZODB.utils import U64
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_unpickle
import BDBStorage
if BDBStorage.is_available:
from BDBStorage.BDBMinimalStorage import BDBMinimalStorage
from BDBStorage.BDBFullStorage import BDBFullStorage
else:
# Sigh
class FakeBaseClass: pass
BDBFullStorage = BDBMinimalStorage = FakeBaseClass
from BDBStorage.tests.ZODBTestBase import ZODBTestBase
from BDBStorage.tests.BerkeleyTestBase import BerkeleyTestBase
from persistent import Persistent
ZERO = '\0'*8
class Object(Persistent):
pass
class WhiteboxLowLevelMinimal(BerkeleyTestBase):
ConcreteStorage = BDBMinimalStorage
def checkTableConsistencyAfterCommit(self):
unless = self.failIf
eq = self.assertEqual
oid = self._storage.new_oid()
revid1 = self._dostore(oid, data=11)
revid2 = self._dostore(oid, revid=revid1, data=12)
revid3 = self._dostore(oid, revid=revid2, data=13)
# First off, there should be no entries in the pending table
unless(self._storage._pending.keys())
# Also, there should be no entries in the oids table
unless(self._storage._oids.keys())
# Now, there should be exactly one oid in the serials table, and
# exactly one record for that oid in the table too.
oids = {}
c = self._storage._serials.cursor()
try:
rec = c.first()
while rec:
oid, serial = rec
oids.setdefault(oid, []).append(serial)
rec = c.next()
finally:
c.close()
eq(len(oids), 1)
eq(len(oids[oids.keys()[0]]), 1)
# There should now be exactly one entry in the pickles table.
pickles = self._storage._pickles.items()
eq(len(pickles), 1)
key, data = pickles[0]
poid = key[:8]
pserial = key[8:]
eq(oid, poid)
eq(revid3, pserial)
obj = zodb_unpickle(data)
eq(obj.value, 13)
# Now verify the refcounts table, which should be empty because the
# stored object isn't referenced by any other objects.
eq(len(self._storage._refcounts.keys()), 0)
def checkStorageVersionAfterCreation(self):
from BDBStorage.BDBMinimalStorage import BDBMINIMAL_SCHEMA_VERSION
eq = self.assertEqual
eq(self._storage._info['version'], BDBMINIMAL_SCHEMA_VERSION)
self._storage.close()
self.open()
eq(self._storage._info['version'], BDBMINIMAL_SCHEMA_VERSION)
class WhiteboxHighLevelMinimal(ZODBTestBase):
ConcreteStorage = BDBMinimalStorage
def checkReferenceCounting(self):
eq = self.assertEqual
obj = MinPO(11)
self._root.obj = obj
get_transaction().commit()
obj.value = 12
get_transaction().commit()
obj.value = 13
get_transaction().commit()
# Make sure the databases have what we expect
eq(len(self._storage._serials.items()), 2)
eq(len(self._storage._pickles.items()), 2)
# And now refcount out the object
del self._root.obj
get_transaction().commit()
# Verification stage. Our serials table should have exactly one
# entry, oid == 0
keys = self._storage._serials.keys()
eq(len(keys), 1)
eq(len(self._storage._serials.items()), 1)
eq(keys[0], ZERO)
# The pickles table now should have exactly one revision of the root
# object, and no revisions of the MinPO object, which should have been
# collected away.
pickles = self._storage._pickles.items()
eq(len(pickles), 1)
rec = pickles[0]
key = rec[0]
data = rec[1]
eq(key[:8], ZERO)
# And that pickle should have no 'obj' attribute.
unobj = zodb_unpickle(data)
self.failIf(hasattr(unobj, 'obj'))
# Our refcounts table should have no entries in it, because the root
# object is an island.
eq(len(self._storage._refcounts.keys()), 0)
# And of course, oids and pendings should be empty too
eq(len(self._storage._oids.keys()), 0)
eq(len(self._storage._pending.keys()), 0)
def checkRecursiveReferenceCounting(self):
eq = self.assertEqual
obj1 = Object()
obj2 = Object()
obj3 = Object()
obj4 = Object()
self._root.obj = obj1
obj1.obj = obj2
obj2.obj = obj3
obj3.obj = obj4
get_transaction().commit()
# Make sure the databases have what we expect
eq(len(self._storage._serials.items()), 5)
eq(len(self._storage._pickles.items()), 5)
# And now refcount out the object
del self._root.obj
get_transaction().commit()
# Verification stage. Our serials table should have exactly one
# entry, oid == 0
keys = self._storage._serials.keys()
eq(len(keys), 1)
eq(len(self._storage._serials.items()), 1)
eq(keys[0], ZERO)
# The pickles table now should have exactly one revision of the root
# object, and no revisions of any other objects, which should have
# been collected away.
pickles = self._storage._pickles.items()
eq(len(pickles), 1)
rec = pickles[0]
key = rec[0]
data = rec[1]
eq(key[:8], ZERO)
# And that pickle should have no 'obj' attribute.
unobj = zodb_unpickle(data)
self.failIf(hasattr(unobj, 'obj'))
# Our refcounts table should have no entries in it, because the root
# object is an island.
eq(len(self._storage._refcounts.keys()), 0)
# And of course, oids and pendings should be empty too
eq(len(self._storage._oids.keys()), 0)
eq(len(self._storage._pending.keys()), 0)
class WhiteboxLowLevelFull(BerkeleyTestBase):
ConcreteStorage = BDBFullStorage
def checkStorageVersionAfterCreation(self):
from BDBStorage.BDBFullStorage import BDBFULL_SCHEMA_VERSION
eq = self.assertEqual
eq(self._storage._info['version'], BDBFULL_SCHEMA_VERSION)
self._storage.close()
self.open()
eq(self._storage._info['version'], BDBFULL_SCHEMA_VERSION)
class WhiteboxHighLevelFull(ZODBTestBase):
ConcreteStorage = BDBFullStorage
def checkReferenceCounting(self):
eq = self.assertEqual
# Make sure the databases have what we expect
eq(len(self._storage._serials.items()), 1)
eq(len(self._storage._pickles.items()), 1)
# Now store an object
obj = MinPO(11)
self._root.obj = obj
get_transaction().commit()
# Make sure the databases have what we expect
eq(len(self._storage._serials.items()), 2)
eq(len(self._storage._pickles.items()), 3)
obj.value = 12
get_transaction().commit()
# Make sure the databases have what we expect
eq(len(self._storage._serials.items()), 2)
eq(len(self._storage._pickles.items()), 4)
obj.value = 13
get_transaction().commit()
# Make sure the databases have what we expect
eq(len(self._storage._serials.items()), 2)
eq(len(self._storage._pickles.items()), 5)
# And now refcount out the object
del self._root.obj
get_transaction().commit()
# Verification stage. Our serials tabl should still have 2 entries,
# one for the root object and one for the now unlinked MinPO obj.
keys = self._storage._serials.keys()
eq(len(keys), 2)
eq(len(self._storage._serials.items()), 2)
eq(keys[0], ZERO)
# The pickles table should now have 6 entries, broken down like so:
# - 3 revisions of the root object: the initial database-open
# revision, the revision that got its obj attribute set, and the
# revision that got its obj attribute deleted.
# - 3 Three revisions of obj, corresponding to values 11, 12, and 13
pickles = self._storage._pickles.items()
eq(len(pickles), 6)
# Our refcounts table should have one entry in it for the MinPO that's
# referenced in an earlier revision of the root object
eq(len(self._storage._refcounts.keys()), 1)
# And of course, oids and pendings should be empty too
eq(len(self._storage._oids.keys()), 0)
eq(len(self._storage._pending.keys()), 0)
def test_suite():
return BDBStorage.tests.BerkeleyTestBase.makeSuite(
WhiteboxLowLevelMinimal,
WhiteboxHighLevelMinimal,
WhiteboxLowLevelFull,
WhiteboxHighLevelFull,
)
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Test some simple ZODB level stuff common to both the BDBMinimalStorage and
# BDBFullStorage storages, like transaction aborts and commits, changing
# objects, etc. Doesn't test undo, versions, or packing.
import time
import unittest
import BDBStorage
from BDBStorage.tests.ZODBTestBase import ZODBTestBase
from persistent.mapping import PersistentMapping
class CommitAndRead:
def checkCommit(self):
self.failUnless(not self._root)
names = self._root['names'] = PersistentMapping()
names['Warsaw'] = 'Barry'
names['Hylton'] = 'Jeremy'
get_transaction().commit()
def checkReadAfterCommit(self):
eq = self.assertEqual
self.checkCommit()
names = self._root['names']
eq(names['Warsaw'], 'Barry')
eq(names['Hylton'], 'Jeremy')
self.failUnless(names.get('Drake') is None)
def checkAbortAfterRead(self):
self.checkReadAfterCommit()
names = self._root['names']
names['Drake'] = 'Fred'
get_transaction().abort()
def checkReadAfterAbort(self):
self.checkAbortAfterRead()
names = self._root['names']
self.failUnless(names.get('Drake') is None)
def checkChangingCommits(self):
self.checkReadAfterAbort()
now = time.time()
# Make sure the last timestamp was more than 3 seconds ago
timestamp = self._root.get('timestamp')
if timestamp is None:
timestamp = self._root['timestamp'] = 0
get_transaction().commit()
self.failUnless(now > timestamp + 3)
self._root['timestamp'] = now
time.sleep(3)
class MinimalCommitAndRead(ZODBTestBase, CommitAndRead):
from BDBStorage import BDBMinimalStorage
ConcreteStorage = BDBMinimalStorage.BDBMinimalStorage
class FullCommitAndRead(ZODBTestBase, CommitAndRead):
from BDBStorage import BDBFullStorage
ConcreteStorage = BDBFullStorage.BDBFullStorage
def test_suite():
return BDBStorage.tests.BerkeleyTestBase.makeSuite(
MinimalCommitAndRead,
FullCommitAndRead
)
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Framework for running Unit tests
import unittest
MODULES = ('commitlog', 'create', 'virgin', 'zodb_simple', 'storage_api')
def suite():
alltests = unittest.TestSuite()
for modname in MODULES:
mod = __import__('test_'+modname)
alltests.addTest(mod.test_suite())
return alltests
def test_suite():
# Just to silence the top-level test.py
return None
if __name__ == '__main__':
unittest.main(defaultTest='suite')
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Time transaction commits and normalize vs. pickle size and #objects.
Actually just counts the size of pickles in the transaction via the iterator
protocol, so storage overheads aren't counted.
Usage: %(PROGRAM)s [options]
Options:
-h/--help
Print this message and exit.
-s filename
--source=filename
Use database in filename as the source (must be a FileStorage)
-d filename
--dest=filename
Use database in filename as the destination (must be a BDB storage)
-o filename
--output=filename
Print results in filename, otherwise stdout.
-m txncount
--max=txncount
Stop after committing txncount transactions.
-k txncount
--skip=txncount
Skip the first txncount transactions.
-p/--profile
Turn on specialized profiling.
-q/--quiet
Be quite.
"""
import sys
import os
import getopt
import time
import errno
import profile
import traceback
import marshal
from bsddb3 import db
from ZODB import utils
from persistent.TimeStamp import TimeStamp
from ZODB.FileStorage import FileStorage
from BDBStorage.BDBFullStorage import BDBFullStorage
PROGRAM = sys.argv[0]
ZERO = '\0'*8
def usage(code, msg=''):
print >> sys.stderr, __doc__ % globals()
if msg:
print >> sys.stderr, msg
sys.exit(code)
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'hs:d:qo:l:pm:k:',
['help', 'source=', 'dest=', 'quiet',
'output=', 'logfile=', 'profile',
'max=', 'skip='])
except getopt.error, msg:
usage(1, msg)
class Options:
source = None
dest = None
verbose = 1
outfile = None
logfile = None
profilep = 0
maxtxn = -1
skiptxn = -1
options = Options()
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-s', '--source'):
options.source = arg
elif opt in ('-d', '--dest'):
options.dest = arg
elif opt in ('-q', '--quiet'):
options.verbose = 0
elif opt in ('-o', '--output'):
options.outfile = arg
elif opt in ('-l', '--logfile'):
options.logfile = arg
elif opt in ('-p', '--profile'):
options.profilep = 1
elif opt in ('-m', '--max'):
options.maxtxn = int(arg)
elif opt in ('-k', '--skip'):
options.skiptxn = int(arg)
if args:
usage(1)
if not options.source or not options.dest:
usage(1, 'Source and destination databases must be provided')
# Open the output file
if options.outfile is None:
options.outfp = sys.stdout
options.outclosep = 0
else:
options.outfp = open(options.outfile, 'w')
options.outclosep = 1
# Open the logfile
if options.logfile is None:
options.logfp = sys.stdout
options.logclosep = 0
else:
options.logfp = open(options.logfile, 'w')
options.logclosep = 1
# Print a comment, this is a hack
print >> options.outfp, '# FS->BDB 3.3.11'
print >> options.outfp, '#', time.ctime()
print >>sys.stderr, 'Opening source FileStorage...'
t0 = time.time()
srcdb = FileStorage(options.source, read_only=1)
t1 = time.time()
print >>sys.stderr, 'Opening source FileStorage done. %s seconds' % (t1-t0)
#
# Uncomment this section to do a FS->BDB migration
#
print >>sys.stderr, 'Opening destination BDB...'
t0 = time.time()
dstdb = BDBFullStorage(options.dest)
t1 = time.time()
print >>sys.stderr, 'Opening destination BDB done. %s seconds' % (t1-t0)
#
# Uncomment this section to do a FS->FS migration
#
## print >>sys.stderr, 'Opening destination FileStorage...'
## t0 = time.time()
## dstdb = FileStorage(dest)
## t1 = time.time()
## print >>sys.stderr, 'Opening destination FileStorage done. %s seconds' % (
## t1-t0)
try:
t0 = time.time()
doit(srcdb, dstdb, options)
t1 = time.time()
print 'Total time:', t1-t0
finally:
# Done
srcdb.close()
dstdb.close()
if options.outclosep:
options.outfp.close()
if options.logclosep:
options.logfp.close()
def doit(srcdb, dstdb, options):
outfp = options.outfp
logfp = options.logfp
profilep = options.profilep
verbose = options.verbose
# some global information
largest_pickle = 0
largest_txn_in_size = 0
largest_txn_in_objects = 0
# Ripped from BaseStorage.copyTransactionsFrom()
ts = None
ok = 1
prevrevids = {}
counter = 0
skipper = 0
for txn in srcdb.iterator():
skipper += 1
if skipper <= options.skiptxn:
continue
counter += 1
if counter > options.maxtxn > 0:
break
tid = txn.tid
if ts is None:
ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= ts:
if ok:
print 'Time stamps are out of order %s, %s' % (ts, t)
ok = 0
ts = t.laterThan(ts)
tid = `ts`
else:
ts = t
if not ok:
print 'Time stamps are back in order %s' % t
ok = 1
if verbose:
print ts
prof = None
if profilep and (counter % 100) == 0:
prof = profile.Profile()
objects = 0
size = 0
t0 = time.time()
dstdb.tpc_begin(txn, tid, txn.status)
t1 = time.time()
try:
for r in txn:
oid = r.oid
objects += 1
thissize = len(r.data)
size += thissize
if thissize > largest_pickle:
largest_pickle = thissize
if verbose:
if not r.version:
vstr = 'norev'
else:
vstr = r.version
print utils.U64(oid), vstr, len(r.data)
oldrevid = prevrevids.get(oid, ZERO)
newrevid = dstdb.store(oid, oldrevid, r.data, r.version, txn)
prevrevids[oid] = newrevid
t2 = time.time()
dstdb.tpc_vote(txn)
t3 = time.time()
# Profile every 100 transactions
if prof:
prof.runcall(dstdb.tpc_finish, txn)
else:
dstdb.tpc_finish(txn)
t4 = time.time()
except KeyError, e:
traceback.print_exc(file=logfp)
# record the results
if objects > largest_txn_in_objects:
largest_txn_in_objects = objects
if size > largest_txn_in_size:
largest_txn_in_size = size
print >> outfp, utils.U64(tid), objects, size, t4-t0, \
t1-t0, t2-t1, t3-t2, t4-t3
if prof:
prof.create_stats()
fp = open('profile-%02d.txt' % (counter / 100), 'wb')
marshal.dump(prof.stats, fp)
fp.close()
print >> outfp, largest_pickle, largest_txn_in_size, largest_txn_in_objects
if __name__ == '__main__':
main()
This diff is collapsed.
......@@ -215,18 +215,6 @@ setup(
['App/www', ['App/www/*']]],
)
# BDBStorage
setup(
name='BDBStorage',
author=AUTHOR,
packages=['BDBStorage', 'BDBStorage.tests'],
ext_modules = [
Extension(name='BDBStorage._helper',
sources=['BDBStorage/_helper.c']),
]
)
# BTrees
setup(
name='BTrees',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment