X move test DB generation code to zodbtools/test

go/zodb/storage/fs1/py/gen-testdata

@@ -18,22 +18,13 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-# TODO move gen .fs & .index part to zodbtools and use it there for tests
-# NOTE as of 14 Mar 2017 FileStorage cannot commit transactions with non-ASCII
-#      metadata - so it is not tested
 """generate reference database and index for tests"""
 
 from ZODB.FileStorage import FileStorage
-from ZODB import DB
-from ZODB.POSException import UndoError
-from persistent import Persistent
-import transaction
+from zodbtools.test.gen_testdata import gen_testdb
 from zodbtools.util import escapeqq
 
 import struct
-import time
-import random
-import logging
 
 # convert numeric oid to/from str
 def p64(num):
@@ -45,143 +36,13 @@ def unpack64(packed):
 def hex64(packed):
     return '0x%016x' % unpack64(packed)
 
-# make time.time() predictable
-_xtime = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
-def xtime():
-    global _xtime
-    _xtime += 1.1
-    return _xtime
-time.time = xtime
-
-
-# prepare transaction for a commit
-def precommit(user, description, extension):
-    txn = transaction.get()
-    txn.user = user
-    txn.description = description
-    txn.extension = extension
-    return txn
-
-def commit(user, description, extension):
-    txn = precommit(user, description, extension)
-    txn.commit()
-
-
-class Object(Persistent):
-    # .value
-    def __init__(self, value):
-        self.value = value
-
-    def __getstate__(self):
-        return self.value
-
-    def __setstate__(self, state):
-        self.value = state
-
-# prepare extension dictionary for subject
-alnum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-def ext(subj):
-    d = {"x-generator": "zodb/py2 (%s)" % subj}
-
-    # also add some random 'x-cookie'
-    cooklen = 5
-    cookie = ""
-    for _ in range(cooklen):
-        cookie += random.choice(alnum)
-
-    xcookie = "x-cookie" + random.choice(alnum)
-    d[xcookie] = cookie
-
-    # shufle extension dict randomly - to likely trigger different ordering on save
-    keyv = d.keys()
-    random.shuffle(keyv)
-    ext = {}
-    for key in keyv:
-        ext[key] = d[key]
-
-    return ext
 
 def main():
-    logging.basicConfig()
-
     outfs = "testdata/1.fs"
-
-    # generate random changes to objects hooked to top-level root by a/b/c/... key
-    random.seed(0)
-
-    namev = [_ for _ in "abcdefg"]
-    Niter = 2
-    for i in range(Niter):
-        stor = FileStorage(outfs, create=(i == 0))
-        db   = DB(stor)
-        conn = db.open()
-        root = conn.root()
-        assert root._p_oid == p64(0), `root._p_oid`
-
-        for j in range(25):
-            name = random.choice(namev)
-            if name in root:
-                obj = root[name]
-            else:
-                root[name] = obj = Object(None)
-
-            obj.value = "%s%i.%i" % (name, i, j)
-
-            commit(u"user%i.%i" % (i,j), u"step %i.%i" % (i, j), ext(name))
-
-        # undo a transaction one step before a latest one a couple of times
-        for j in range(2):
-            # XXX undoLog, despite what its interface says:
-            #   https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/interfaces.py#L472
-            # just returns log of all transactions in specified range:
-            #   https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/FileStorage/FileStorage.py#L1008
-            #   https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/FileStorage/FileStorage.py#L2103
-            # so we retry undoing next log's txn on conflict.
-            for ul in db.undoLog(1, 20):
-                try:
-                    db.undo(ul["id"])
-                    commit(u"root%i.%i\nYour\nMagesty " % (i, j),
-                           u"undo %i.%i\nmore detailed description\n\nzzz ..." % (i, j) + "\t"*(i+j),
-                           ext("undo %s" % ul["id"]))
-                except UndoError:
-                    transaction.abort()
-                    continue
-
-                break
-
-        # delete an object
-        name = random.choice(root.keys())
-        obj = root[name]
-        root[name] = Object("%s%i*" % (name, i))
-        # NOTE user/ext are kept empty on purpose - to also test this case
-        commit(u"", u"predelete %s" % unpack64(obj._p_oid), {})
-
-        # XXX obj in db could be changed by above undo, but ZODB does not automatically
-        # propagate undo changes to live objects - so obj._p_serial can be stale.
-        # Get serial via history.
-        obj_tid_lastchange = db.history(obj._p_oid)[0]['tid']
-
-        txn = precommit(u"root%i\nYour\nRoyal\nMagesty' " % i +
-                            ''.join(chr(_) for _ in range(32)),     # <- NOTE all control characters
-                        u"delete %i\nalpha beta gamma'delta\"lambda\n\nqqq ..." % i,
-                        ext("delete %s" % unpack64(obj._p_oid)))
-        stor.tpc_begin(txn)
-        stor.deleteObject(obj._p_oid, obj_tid_lastchange, txn)
-        stor.tpc_vote(txn)
-        # TODO different txn status vvv
-        # XXX vvv it does the thing, but py fs iterator treats this txn as EOF
-        #if i != Niter-1:
-        #    stor.tpc_finish(txn)
-        stor.tpc_finish(txn)
-
-        # close db & rest not to get conflict errors after we touched stor
-        # directly a bit. everything will be reopened on next iteration.
-        conn.close()
-        db.close()
-        stor.close()
-
+    gen_testdb(outfs)
 
     # dump to go what to expect
+    stor = FileStorage(outfs, read_only=True)
     with open("ztestdata_expect_test.go", "w") as f:
         def emit(v):
             print >>f, v
@@ -197,8 +58,6 @@ def main():
         emit("}")
 
         # database records
-        stor = FileStorage(outfs, read_only=True)
-
         emit("\nvar _1fs_dbEntryv = [...]dbEntry{")
         txnLenPrev = -1
         for txn in stor.iterator():     # txn is TransactionRecord

go/zodb/zodbtools/dump.go

@@ -47,6 +47,8 @@ Dump format:
 
 quote:      quote string with " with non-printable and control characters \-escaped
 hashfunc:   one of sha1, sha256, sha512 ...
+
+TODO also protect txn record by hash.
 */
 
 package zodbtools