Merge zodb4 conversion code from the zope3-zodb3-devel-branch.

trunk/src/ZODB/zodb4/__init__.py

+# This directory contains a Python package.

trunk/src/ZODB/zodb4/conversion.py

+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Nitty-gritty conversion of a ZODB 4 FileStorage to a ZODB 3 FileStorage."""
+
+from cPickle import dumps, Pickler, Unpickler
+from cStringIO import StringIO
+
+from ZODB.FileStorage import FileStorage
+from ZODB.zodb4 import z4iterator
+
+
+class Conversion:
+
+    def __init__(self, input_path, output_path):
+        """Initialize a ZODB4->ZODB3 FileStorage converter."""
+        self.instore = IterableFileIterator(input_path)
+        self.outstore = FileStorage(output_path)
+
+    def run(self):
+        self.instore._read_metadata()
+        self.outstore.copyTransactionsFrom(self.instore)
+        self.outstore.close()
+        self.instore.close()
+
+
+class IterableFileIterator(z4iterator.FileIterator):
+
+    def iterator(self):
+        return self
+
+    def __iter__(self):
+        baseiter = z4iterator.FileIterator.__iter__(self)
+        for txn in baseiter:
+            yield DataRecordConvertingTxn(txn)
+
+
+class DataRecordConvertingTxn(object):
+
+    def __init__(self, txn):
+        self._txn = txn
+        self.user = str8(txn.user)
+        self.description = str8(txn.description)
+
+    def __getattr__(self, name):
+        return getattr(self._txn, name)
+
+    def __iter__(self):
+        for record in self._txn:
+            record.tid = record.serial
+            # transform the data record format
+            # (including persistent references)
+            sio = StringIO(record.data)
+            up = Unpickler(sio)
+            up.persistent_load = PersistentIdentifier
+            classmeta = up.load()
+            state = up.load()
+            sio = StringIO()
+            p = Pickler(sio, 1)
+            p.persistent_id = get_persistent_id
+            p.dump(classmeta)
+            p.dump(state)
+            record.data = sio.getvalue()
+            yield record
+
+
+class PersistentIdentifier:
+    def __init__(self, ident):
+        if isinstance(ident, tuple):
+            self._oid, (self._class, args) = ident
+            if args:
+                # we have args from __getnewargs__(), but can just
+                # lose them since they're an optimization to allow
+                # ghost construction
+                self._class = None
+        else:
+            assert isinstance(ident, str)
+            self._oid = ident
+            self._class = None
+
+
+def get_persistent_id(ob):
+    if isinstance(ob, PersistentIdentifier):
+        if ob._class is None:
+            return ob._oid
+        else:
+            return ob._oid, ob._class
+    else:
+        return None
+
+
+def str8(s):
+    # convert unicode strings to 8-bit strings
+    if isinstance(s, unicode):
+        # Should we use UTF-8 or ASCII?  Not sure.
+        return s.encode("ascii")
+    else:
+        return s

trunk/src/ZODB/zodb4/main.py

+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Script to convert a ZODB 4 file storage to a ZODB 3 file storage.
+
+This is needed since Zope 3 is being changed to use ZODB 3 instead of
+ZODB 4.
+
+"""
+
+import getopt
+import os
+import sys
+
+try:
+    __file__
+except NameError:
+    __file__ = os.path.realpath(sys.argv[0])
+
+here = os.path.dirname(__file__)
+topdir = os.path.dirname(os.path.dirname(here))
+
+# Make sure that if we're run as a script, we can import the ZODB
+# package and our sibling modules.
+try:
+    import ZODB.zodb4
+except ImportError:
+    sys.path.append(topdir)
+    import ZODB.zodb4
+
+from ZODB.lock_file import LockFile
+from ZODB.zodb4 import conversion
+
+
+class ConversionApp:
+
+    def __init__(self, name=None, args=None):
+        if name is None:
+            name = os.path.basename(sys.argv[0])
+        if args is None:
+            args = sys.argv[1:]
+        self.name = name
+        self.verbosity = 0
+        self.dbfile = None
+        self.parse_args(args)
+
+    def run(self):
+        if not os.path.exists(self.dbfile):
+            self.error("input database does not exist: %s" % self.dbfile)
+        base, ext = os.path.splitext(self.dbfile)
+        if ext != ".fs":
+            base = self.dbfile
+        self.dbindex = self.dbfile + ".index"
+        self.bakfile = base + ".fs4"
+        self.bakindex = self.bakfile + ".index"
+        if os.path.exists(self.bakfile):
+            self.error("backup database already exists: %s\n"
+                       "please move aside and try again" % self.bakfile)
+        if os.path.exists(self.bakindex):
+            self.error("backup database index already exists: %s\n"
+                       "please move aside and try again" % self.bakindex)
+        self.convert()
+
+    def convert(self):
+        lock = LockFile(self.bakfile + ".lock")
+        try:
+            # move the ZODB 4 database to be the backup
+            os.rename(self.dbfile, self.bakfile)
+            if os.path.exists(self.dbindex):
+                try:
+                    os.rename(self.dbindex, self.bakindex)
+                except:
+                    # we couldn't rename *both*, so try to make sure we
+                    # don't rename either
+                    os.rename(self.bakfile, self.dbfile)
+                    raise
+            # go:
+            converter = conversion.Conversion(self.bakfile, self.dbfile)
+            converter.run()
+        finally:
+            lock.close()
+
+    def parse_args(self, args):
+        opts, args = getopt.getopt(args, "v", ["verbose"])
+        for opt, arg in opts:
+            if opt in ("-v", "--verbose"):
+                self.verbosity += 1
+        if len(args) == 0:
+            # use default location for Data.fs
+            self.dbfile = os.path.join(topdir, "Data.fs")
+        elif len(args) == 1:
+            self.dbfile = args[0]
+        else:
+            self.error("too many command-line arguments", rc=2)
+
+    def error(self, message, rc=1):
+        print >>sys.stderr, "%s: %s" % (self.name, message)
+        sys.exit(rc)
+
+
+def main():
+    ConversionApp().run()
+
+if __name__ == "__main__":
+    main()

trunk/src/ZODB/zodb4/tests/__init__.py

+# This directory contains a Python package.

trunk/src/ZODB/zodb4/tests/test_z4utils.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Test the routines to convert between long and 64-bit strings"""
+
+# originally zodb.tests.test_utils
+
+import random
+import unittest
+
+NUM = 100
+
+from ZODB.zodb4.z4utils import p64, u64
+
+class TestUtils(unittest.TestCase):
+
+    small = [random.randrange(1, 1L<<32, int=long)
+             for i in range(NUM)]
+    large = [random.randrange(1L<<32, 1L<<64, int=long)
+             for i in range(NUM)]
+    all = small + large
+
+    def test_LongToStringToLong(self):
+        for num in self.all:
+            s = p64(num)
+            n2 = u64(s)
+            self.assertEquals(num, n2, "u64() failed")
+
+    def test_KnownConstants(self):
+        self.assertEquals("\000\000\000\000\000\000\000\001", p64(1))
+        self.assertEquals("\000\000\000\001\000\000\000\000", p64(1L<<32))
+        self.assertEquals(u64("\000\000\000\000\000\000\000\001"), 1)
+        self.assertEquals(u64("\000\000\000\001\000\000\000\000"), 1L<<32)
+
+def test_suite():
+    return unittest.makeSuite(TestUtils)
+
+if __name__ == "__main__":
+    unittest.main(defaultTest="test_suite")

trunk/src/ZODB/zodb4/z4base.py

+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+"""Convenience function extracted from ZODB4's zodb.storage.base."""
+
+
+def splitrefs(refstr, oidlen=8):
+    # refstr is a packed string of reference oids.  Always return a list of
+    # oid strings.  Most storages use fixed oid lengths of 8 bytes, but if
+    # the oids in refstr are a different size, use oidlen to specify.  This
+    # does /not/ support variable length oids in refstr.
+    if not refstr:
+        return []
+    num, extra = divmod(len(refstr), oidlen)
+    fmt = '%ds' % oidlen
+    assert extra == 0, refstr
+    return list(struct.unpack('>' + (fmt * num), refstr))

trunk/src/ZODB/zodb4/z4errors.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""FileStorage-specific exceptions."""
+
+from ZODB.zodb4.z4interfaces import _fmt_oid, POSError
+
+# originally from zodb.storage.interfaces
+
+class StorageError(POSError):
+    """Base class for storage based exceptions."""
+
+class StorageSystemError(StorageError):
+    """Panic! Internal storage error!"""
+
+
+# originally zodb.storage.file.errors
+
+class FileStorageError(StorageError):
+    pass
+
+class PackError(FileStorageError):
+    pass
+
+class FileStorageFormatError(FileStorageError):
+    """Invalid file format
+
+    The format of the given file is not valid.
+    """
+
+class CorruptedError(FileStorageError, StorageSystemError):
+    """Corrupted file storage."""
+
+class CorruptedDataError(CorruptedError):
+
+    def __init__(self, oid=None, buf=None, pos=None):
+        self.oid = oid
+        self.buf = buf
+        self.pos = pos
+
+    def __str__(self):
+        if self.oid:
+            msg = "Error reading oid %s.  Found %r" % (_fmt_oid(self.oid),
+                                                       self.buf)
+        else:
+            msg = "Error reading unknown oid.  Found %r" % self.buf
+        if self.pos:
+            msg += " at %d" % self.pos
+        return msg
+
+class FileStorageQuotaError(FileStorageError, StorageSystemError):
+    """File storage quota exceeded."""
+

trunk/src/ZODB/zodb4/z4iterator.py

+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Iterator support for ZODB 4 databases."""
+
+from cPickle import loads
+from struct import unpack
+
+from ZODB.zodb4.z4interfaces import ZERO
+from ZODB.zodb4.z4utils import u64, splitrefs
+from ZODB.zodb4.z4format import FileStorageFormatter, DataHeader, TxnHeader
+from ZODB.zodb4.z4format import TRANS_HDR, TRANS_HDR_LEN, DATA_HDR
+from ZODB.zodb4.z4format import DATA_HDR_LEN, DATA_VERSION_HDR_LEN
+
+# originally from zodb.storage.file.main
+
+class FileIterator(FileStorageFormatter):
+    """Iterate over the transactions in a FileStorage file."""
+    _ltid = ZERO
+
+##    implements(IStorageIterator)
+
+    def __init__(self, file):
+        # - removed start and stop arguments
+        if isinstance(file, str):
+            file = open(file, 'rb')
+        self._file = file
+        self._read_metadata()
+        self._file.seek(0,2)
+        self._file_size = self._file.tell()
+        self._pos = self._metadata_size
+
+    def close(self):
+        file = self._file
+        if file is not None:
+            self._file = None
+            file.close()
+
+    def __iter__(self):
+        if self._file is None:
+            # A closed iterator.  XXX: Is IOError the best we can do?  For
+            # now, mimic a read on a closed file.
+            raise IOError("iterator is closed")
+        file = self._file
+        seek = file.seek
+        read = file.read
+
+        pos = self._pos
+        while True:
+            # Read the transaction record
+            seek(pos)
+            h = read(TRANS_HDR_LEN)
+            if len(h) < TRANS_HDR_LEN:
+                break
+
+            tid, tl, status, ul, dl, el = unpack(TRANS_HDR,h)
+            if el < 0:
+                el = (1L<<32) - el
+
+            if tid <= self._ltid:
+                warn("%s time-stamp reduction at %s", self._file.name, pos)
+            self._ltid = tid
+
+            if pos+(tl+8) > self._file_size or status=='c':
+                # Hm, the data were truncated or the checkpoint flag wasn't
+                # cleared.  They may also be corrupted,
+                # in which case, we don't want to totally lose the data.
+                warn("%s truncated, possibly due to damaged records at %s",
+                     self._file.name, pos)
+                break
+
+            if status not in ' p':
+                warn('%s has invalid status, %s, at %s', self._file.name,
+                     status, pos)
+
+            if tl < (TRANS_HDR_LEN+ul+dl+el):
+                # We're in trouble. Find out if this is bad data in
+                # the middle of the file, or just a turd that Win 9x
+                # dropped at the end when the system crashed.  Skip to
+                # the end and read what should be the transaction
+                # length of the last transaction.
+                seek(-8, 2)
+                rtl = u64(read(8))
+                # Now check to see if the redundant transaction length is
+                # reasonable:
+                if self._file_size - rtl < pos or rtl < TRANS_HDR_LEN:
+                    logger.critical('%s has invalid transaction header at %s',
+                                    self._file.name, pos)
+                    warn("It appears that there is invalid data at the end of "
+                         "the file, possibly due to a system crash.  %s "
+                         "truncated to recover from bad data at end.",
+                         self._file.name)
+                    break
+                else:
+                    warn('%s has invalid transaction header at %s',
+                         self._file.name, pos)
+                    break
+
+            tpos = pos
+            tend = tpos+tl
+
+            pos = tpos+(TRANS_HDR_LEN+ul+dl+el)
+            # user and description are utf-8 encoded strings
+            user = read(ul).decode('utf-8')
+            description = read(dl).decode('utf-8')
+            e = {}
+            if el:
+                try:
+                    e = loads(read(el))
+                # XXX can we do better?
+                except:
+                    pass
+
+            result = RecordIterator(tid, status, user, description, e, pos,
+                                    tend, file, tpos)
+            pos = tend
+
+            # Read the (intentionally redundant) transaction length
+            seek(pos)
+            l = u64(read(8))
+            if l != tl:
+                warn("%s redundant transaction length check failed at %s",
+                     self._file.name, pos)
+                break
+            pos += 8
+            yield result
+
+
+class RecordIterator(FileStorageFormatter):
+    """Iterate over data records for a transaction in a FileStorage."""
+
+##    implements(ITransactionRecordIterator, ITransactionAttrs)
+
+    def __init__(self, tid, status, user, desc, ext, pos, tend, file, tpos):
+        self.tid = tid
+        self.status = status
+        self.user = user
+        self.description = desc
+        self._extension = ext
+        self._pos = pos
+        self._tend = tend
+        self._file = file
+        self._tpos = tpos
+
+    def __iter__(self):
+        pos = self._pos
+        while pos < self._tend:
+            # Read the data records for this transaction
+            h = self._read_data_header(pos)
+            dlen = h.recordlen()
+            if pos + dlen > self._tend or h.tloc != self._tpos:
+                warn("%s data record exceeds transaction record at %s",
+                     file.name, pos)
+                return
+
+            pos += dlen
+            prev_txn = None
+
+            if h.plen:
+                refsdata = self._file.read(h.nrefs * 8)
+                refs = splitrefs(refsdata)
+                data = self._file.read(h.plen)
+            else:
+                if not h.back:
+                    # If the backpointer is 0, then this transaction
+                    # undoes the object creation.  It either aborts
+                    # the version that created the object or undid the
+                    # transaction that created it.  Return None
+                    # for data and refs because the backpointer has
+                    # the real data and refs.
+                    data = None
+                    refs = None
+                else:
+                    data, refs, _s, tid = self._loadBackTxn(h.oid, h.back)
+                    prev_txn = self.getTxnFromData(h.oid, h.back)
+
+            yield Record(h.oid, h.serial, h.version, data, prev_txn, refs)
+
+class Record:
+    """An abstract database record."""
+
+##    implements(IDataRecord)
+
+    def __init__(self, oid, serial, version, data, data_txn, refs):
+        self.oid = oid
+        self.serial = serial
+        self.version = version
+        self.data = data
+        self.data_txn = data_txn
+        self.refs = refs

trunk/src/ZODB/zodb4/z4utils.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+# originally zodb.utils
+
+from persistent.TimeStamp import TimeStamp
+import struct
+import time
+
+from sets import Set
+
+def p64(v):
+    """Pack an integer or long into a 8-byte string"""
+    return struct.pack(">Q", v)
+
+def u64(v):
+    """Unpack an 8-byte string into a 64-bit long integer."""
+    return struct.unpack(">Q", v)[0]
+
+def cp(f1, f2, l):
+    read = f1.read
+    write = f2.write
+    n = 8192
+
+    while l > 0:
+        if n > l:
+            n = l
+        d = read(n)
+        if not d:
+            break
+        write(d)
+        l = l - len(d)
+
+
+# originally from zodb.storage.base
+
+def splitrefs(refstr, oidlen=8):
+    # refstr is a packed string of reference oids.  Always return a list of
+    # oid strings.  Most storages use fixed oid lengths of 8 bytes, but if
+    # the oids in refstr are a different size, use oidlen to specify.  This
+    # does /not/ support variable length oids in refstr.
+    if not refstr:
+        return []
+    num, extra = divmod(len(refstr), oidlen)
+    fmt = '%ds' % oidlen
+    assert extra == 0, refstr
+    return list(struct.unpack('>' + (fmt * num), refstr))