debug/junk

.gitignore

 /dist
 /zodbtools.egg-info
-
+/**/**.pyc
 /.tox/
 /venv
 /venv2

zodbtools/zodbcmp.py

@@ -36,6 +36,12 @@ from zodbtools.util import ashex, inf, nextitem, txnobjv, parse_tidrange, TidRan
 from time import time
 from golang import func, defer
 
+from ZODB.utils import readable_tid_repr
+ashex = readable_tid_repr
+
+from ZODB.serialize import referencesf
+
+
 # compare two storage transactions
 # 0 - equal, 1 - non-equal
 def txncmp(txn1, txn2):
@@ -44,12 +50,14 @@ def txncmp(txn1, txn2):
         attr1 = getattr(txn1, attr)
         attr2 = getattr(txn2, attr)
         if attr1 != attr2:
+            import pdb; pdb.set_trace()
             return 1
 
     # data
     objv1 = txnobjv(txn1)
     objv2 = txnobjv(txn2)
     if len(objv1) != len(objv2):
+        import pdb; pdb.set_trace()
         return 1
 
     for obj1, obj2 in zip(objv1, objv2):
@@ -57,6 +65,7 @@ def txncmp(txn1, txn2):
             attr1 = getattr(obj1, attr)
             attr2 = getattr(obj2, attr)
             if attr1 != attr2:
+                import pdb; pdb.set_trace()
                 return 1
 
     return 0
@@ -105,7 +114,9 @@ def storcmp(stor1, stor2, tidmin, tidmax, verbose=False):
         tcmp = txncmp(txn1, txn2)
         if tcmp:
             if verbose:
-                print("not-equal: transaction %s is different")
+                print("not-equal: transaction %s is different" % ashex(txn1.tid))
+                print([o1.oid for (o1, o2) in zip(txn1, txn2) if o1.data != o2.data])
+                import pdb; pdb.set_trace()
             return 1
 
 

zodbtools/zodbcommit.py

@@ -40,7 +40,7 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`.
 
 from __future__ import print_function
 from zodbtools import zodbdump
-from zodbtools.util import ashex, fromhex, storageFromURL, asbinstream
+from zodbtools.util import ashex, fromhex, sha1, storageFromURL, asbinstream
 from ZODB.interfaces import IStorageRestoreable
 from ZODB.utils import p64, u64, z64
 from ZODB.POSException import POSKeyError
@@ -48,6 +48,7 @@ from ZODB._compat import BytesIO
 from golang import func, defer, panic, b
 import warnings
 
+from ZODB.utils import readable_tid_repr
 
 # zodbcommit commits new transaction into ZODB storage with data specified by
 # zodbdump transaction.
@@ -90,14 +91,24 @@ def zodbcommit(stor, at, txn):
             copy_from = None
             if isinstance(obj, zodbdump.ObjectCopy):
                 copy_from = obj.copy_from
-                try:
-                    xdata = stor.loadBefore(obj.oid, p64(u64(obj.copy_from)+1))
-                except POSKeyError:
-                    xdata = None
-                if xdata is None:
-                    raise ValueError("%s: object %s: copy from @%s: no data" %
-                            (runctx, ashex(obj.oid), ashex(obj.copy_from)))
-                data, _, _ = xdata
+                if hasattr(obj, 'data'):
+                    data = obj.data
+                    if ashex(txn.tid) == '03a6166ee64176bb' and ashex(obj.oid) == '0000000000083b45':
+                        print("%s %s => %s" % (readable_tid_repr(txn.tid), ashex(obj.oid), ashex(sha1(obj.data))))
+                        import pdb; pdb.set_trace()
+                else:
+                    boom
+                    try:
+                        xdata = stor.loadBefore(obj.oid, p64(u64(obj.copy_from)+1))
+                    except POSKeyError:
+                        xdata = None
+                    if ashex(txn.tid) == '03a6166ee64176bb' and obj.oid==b'\x00\x00\x00\x00\x00\x08;E':
+                        print(ashex(txn.tid), xdata)
+                        #import pdb; pdb.set_trace()
+                    if xdata is None:
+                        raise ValueError("%s: object %s: copy from @%s: no data" %
+                                (runctx, ashex(obj.oid), ashex(obj.copy_from)))
+                    data, _, _ = xdata
 
             elif isinstance(obj, zodbdump.ObjectDelete):
                 data = None
@@ -115,9 +126,19 @@ def zodbcommit(stor, at, txn):
             # we have the data -> restore/store the object.
             # if it will be ConflictError - we just fail and let the caller retry.
             if data is None:
-                stor.deleteObject(obj.oid, current_serial(obj.oid), txn)
+                try:
+                    curser = current_serial(obj.oid)
+                    stor.deleteObject(obj.oid, curser, txn)
+                except:
+                    #import pdb; pdb.set_trace()
+                    raise
             else:
                 if want_restore and have_restore:
+                    if 0 and txn.status == 'p':
+                        warnings.warn(
+                          "%s is a pack transaction (status p), it will be restored "
+                          "as a commit transaction (status c)" % ashex(txn.tid))
+                        import pdb; pdb.set_trace()
                     stor.restore(obj.oid, txn.tid, data, '', copy_from, txn)
                 else:
                     # FIXME we don't handle copy_from on commit

zodbtools/zodbdump.py

@@ -108,6 +108,9 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, pretty='raw', out=asbinstream
     for txn in stor.iterator(tidmin, tidmax):
         # XXX .status not covered by IStorageTransactionInformation
         # XXX but covered by BaseStorage.TransactionRecord
+        if '0000000000083b45' not in [ashex(o.oid) for o in txn]:
+          continue
+
         out.write(b"txn %s %s\nuser %s\ndescription %s\n" % (
             ashex(txn.tid), qq(txn.status),
             qq(txn.user),
@@ -136,6 +139,9 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, pretty='raw', out=asbinstream
 
         for obj in objv:
             entry = b"obj %s " % ashex(obj.oid)
+            if b'0000000000083b45' not in entry:
+              continue
+
             write_data = False
 
             if obj.data is None:
@@ -143,7 +149,10 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, pretty='raw', out=asbinstream
 
             # was undo and data taken from obj.data_txn
             elif obj.data_txn is not None:
-                entry += b"from %s" % ashex(obj.data_txn)
+                from ZODB.utils import p64, u64, z64
+                xdata = stor.loadBefore(obj.oid, p64(u64(obj.data_txn)+1))
+                data, _, _ = xdata
+                entry += b"from %s # %s" % (ashex(obj.data_txn), ashex(sha1(data)))
 
             else:
                 # XXX sha1 is hardcoded for now. Dump format allows other hashes.
@@ -529,6 +538,10 @@ class Object(object):
     def __init__(self, oid):
         self.oid = oid
 
+    def __repr__(self):
+        return "<%s object for %s at %x>" % (
+            self.__class__.__name__, ashex(self.oid), id(self))
+
 # ObjectDelete represents objects deletion.
 class ObjectDelete(Object):
 
@@ -541,9 +554,11 @@ class ObjectDelete(Object):
 # ObjectCopy represents object data copy.
 class ObjectCopy(Object):
     # .copy_from    tid         copy object data from object's revision tid
-    def __init__(self, oid, copy_from):
+    def __init__(self, oid, copy_from, data=None):
         super(ObjectCopy, self).__init__(oid)
         self.copy_from = copy_from
+        self.data      = data
+
 
     def zdump(self):
         return b'obj %s from %s\n' % (ashex(self.oid), ashex(self.copy_from))
@@ -553,6 +568,7 @@ class ObjectData(Object):
     # .data         HashOnly | bytes
     # .hashfunc     bstr            hash function used for integrity
     # .hash_        bytes           hash of the object's data
+    # XXX
     def __init__(self, oid, data, hashfunc, hash_):
         super(ObjectData, self).__init__(oid)
         self.data       = data

zodbtools/zodbsync.py

@@ -29,10 +29,18 @@ from zodbtools.zodbdump import Transaction, ObjectCopy, ObjectData, ObjectDelete
 from zodbtools.util import ashex, parse_tid, storageFromURL, txnobjv
 
 from ZODB.interfaces import IStorageIteration
+from ZODB.TimeStamp import TimeStamp
 from ZODB.utils import z64, readable_tid_repr
 
 from golang import func, defer
 
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
+
+def xxxzodbsync(primary_store, secondary_store, until, verbosity):
+    secondary_store.copyTransactionsFrom(primary_store)# verbosity > 0)
+
 
 def zodbsync(primary_store, secondary_store, until, verbosity):
     at = secondary_store.lastTransaction()
@@ -41,7 +49,7 @@ def zodbsync(primary_store, secondary_store, until, verbosity):
         print('secondary at', readable_tid_repr(at))
         print("replicating from", readable_tid_repr(at), end='')
         if until:
-            print("until", readable_tid_repr(until), end='')
+            print(" until", readable_tid_repr(until), end='')
         print()
 
     start = datetime.datetime.now()
@@ -54,11 +62,12 @@ def zodbsync(primary_store, secondary_store, until, verbosity):
             continue
 
         objv = []
-        for obj in txnobjv(t):
+        for obj in t: #(txnobjv(t)):
             if obj.data is None:
+                assert not obj.data_txn
                 objv.append(ObjectDelete(obj.oid))
             elif obj.data_txn is not None:
-                objv.append(ObjectCopy(obj.oid, obj.data_txn))
+                objv.append(ObjectCopy(obj.oid, obj.data_txn, data=obj.data))
             else:
                 objv.append(ObjectData(obj.oid, obj.data, 'null', None))
 
@@ -72,12 +81,18 @@ def zodbsync(primary_store, secondary_store, until, verbosity):
         zodbcommit(secondary_store, at, txn)
         transaction_count += 1
 
+
+        if verbosity > 0:
+            behind = TimeStamp(primary_store.lastTransaction()).timeTime() - TimeStamp(at).timeTime()
+            print("behind=%s" % behind)
+
         if verbosity > 1:
-            print(readable_tid_repr(txn.tid), t.user, t.description, len(objv))
+            print(readable_tid_repr(txn.tid), t.user, t.description, len(objv), str(datetime.timedelta(seconds=behind)))
         elif verbosity > 0:
             print(readable_tid_repr(txn.tid))
         at = txn.tid
 
+
     if verbosity:
         print("replicated %d transactions in %s" % (
             transaction_count, datetime.datetime.now() - start))
@@ -128,7 +143,7 @@ def main(argv):
         sys.exit(2)
 
 
-    primary_store = storageFromURL(argv[0], read_only=True)
+    primary_store = storageFromURL(argv[0]) #read_only=True)
     defer(primary_store.close)
     secondary_store = storageFromURL(argv[1])
     defer(secondary_store.close)