py3 wip

zodbtools/test/test_commit.py

@@ -41,8 +41,8 @@ def test_zodbcommit():
     # commit some transactions via zodbcommit and verify if storage dump gives
     # what is expected.
     t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [
-        ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
-        ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
+        ObjectData(p64(1), b'data1', 'sha1', sha1(b'data1')),
+        ObjectData(p64(2), b'data2', 'sha1', sha1(b'data2'))])
 
     t1.tid = zodbcommit(stor, head, t1)
 
@@ -52,7 +52,9 @@ def test_zodbcommit():
     t2.tid = zodbcommit(stor, t1.tid, t2)
 
 
-    buf = BytesIO()
+   # buf = BytesIO()
+    from io import StringIO
+    buf = StringIO()
     zodbdump(stor, p64(u64(head)+1), maxtid, out=buf)
     dumped = buf.getvalue()
 

zodbtools/test/test_dump.py

@@ -21,9 +21,11 @@ from zodbtools.zodbdump import (
         zodbdump, DumpReader, Transaction, ObjectDelete, ObjectCopy,
         ObjectData, HashOnly
     )
+from zodbtools.util import fromhex
+
 from ZODB.FileStorage import FileStorage
 from ZODB.utils import p64
-from cStringIO import StringIO
+from io import BytesIO
 
 from os.path import dirname
 
@@ -37,7 +39,7 @@ def test_zodbdump():
     with open('%s/testdata/1.zdump.ok' % tdir) as f:
         dumpok = f.read()
 
-    out = StringIO()
+    out = BytesIO()
     zodbdump(stor, None, None, out=out)
 
     assert out.getvalue() == dumpok
@@ -67,10 +69,10 @@ extension "qqq"
 
 """
 
-    r = DumpReader(StringIO(in_))
+    r = DumpReader(BytesIO(in_))
     t1 = r.readtxn()
     assert isinstance(t1, Transaction)
-    assert t1.tid == '0123456789abcdef'.decode('hex')
+    assert t1.tid == fromhex('0123456789abcdef')
     assert t1.user              == b'my name'
     assert t1.description       == b'o la-la...'
     assert t1.extension_bytes   == b'zzz123 def'
@@ -81,29 +83,29 @@ extension "qqq"
     _ = t1.objv[1]
     assert isinstance(_, ObjectCopy)
     assert _.oid        == p64(2)
-    assert _.copy_from  == '0123456789abcdee'.decode('hex')
+    assert _.copy_from  == fromhex('0123456789abcdee')
     _ = t1.objv[2]
     assert isinstance(_, ObjectData)
     assert _.oid        == p64(3)
     assert _.data       == HashOnly(54)
     assert _.hashfunc   == 'adler32'
-    assert _.hash_      == '01234567'.decode('hex')
+    assert _.hash_      == fromhex('01234567')
     _ = t1.objv[3]
     assert isinstance(_, ObjectData)
     assert _.oid        == p64(4)
     assert _.data       == b'ZZZZ'
     assert _.hashfunc   == 'sha1'
-    assert _.hash_      == '9865d483bc5a94f2e30056fc256ed3066af54d04'.decode('hex')
+    assert _.hash_      == fromhex('9865d483bc5a94f2e30056fc256ed3066af54d04')
     _ = t1.objv[4]
     assert isinstance(_, ObjectData)
     assert _.oid        == p64(5)
     assert _.data       == b'ABC\n\nDEF!'
     assert _.hashfunc   == 'crc32'
-    assert _.hash_      == '52fdeac5'.decode('hex')
+    assert _.hash_      == fromhex('52fdeac5')
 
     t2 = r.readtxn()
     assert isinstance(t2, Transaction)
-    assert t2.tid == '0123456789abcdf0'.decode('hex')
+    assert t2.tid == fromhex('0123456789abcdf0')
     assert t2.user              == b'author2'
     assert t2.description       == b'zzz'
     assert t2.extension_bytes   == b'qqq'
@@ -112,10 +114,10 @@ extension "qqq"
     assert r.readtxn() == None
 
     z = ''.join([_.zdump() for _ in (t1, t2)])
-    assert z == in_
+    assert z.encode() == in_
 
     # unknown hash function
-    r = DumpReader(StringIO("""\
+    r = DumpReader(BytesIO(b"""\
 txn 0000000000000000 " "
 user ""
 description ""
@@ -125,10 +127,11 @@ obj 0000000000000001 1 xyz:0123 -
 """))
     with raises(RuntimeError) as exc:
         r.readtxn()
-    assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",)
+                                                                                # XXX b ?
+    assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" (b'obj 0000000000000001 1 xyz:0123 -')""",)
 
     # data integrity error
-    r = DumpReader(StringIO("""\
+    r = DumpReader(BytesIO(b"""\
 txn 0000000000000000 " "
 user ""
 description ""

zodbtools/util.py

@@ -18,16 +18,19 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
-import hashlib, struct, codecs
+import hashlib, struct, codecs, binascii
 import zodburi
 from six.moves.urllib_parse import urlsplit, urlunsplit
 from zlib import crc32, adler32
 
 def ashex(s):
+    return binascii.hexlify(s).decode()
     return s.encode('hex')
+#ashex = binascii.hexlify
 
 def fromhex(s):
     return codecs.decode(s, 'hex')
+fromhex = binascii.unhexlify
 
 def sha1(data):
     m = hashlib.sha1()
@@ -74,8 +77,8 @@ def parse_tidrange(tidrange):
         raise TidRangeInvalid(tidrange)
 
     try:
-        tidmin = tidmin.decode("hex")
-        tidmax = tidmax.decode("hex")
+        tidmin = binascii.unhexlify(tidmin)
+        tidmax = binascii.unhexlify(tidmax)
     except TypeError:   # hex decoding error
         raise TidRangeInvalid(tidrange)
 
@@ -131,7 +134,7 @@ class Adler32Hasher:
     digest_size = 4
 
     def __init__(self):
-        self._h = adler32('')
+        self._h = adler32(b'')
 
     def update(self, data):
         self._h = adler32(data, self._h)
@@ -148,7 +151,7 @@ class CRC32Hasher:
     digest_size = 4
 
     def __init__(self):
-        self._h = crc32('')
+        self._h = crc32(b'')
 
     def update(self, data):
         self._h = crc32(data, self._h)

zodbtools/zodbdump.py

@@ -78,9 +78,9 @@ def txn_raw_extension(stor, txn):
     # in a rational way
     stor_name = "(%s, %s)" % (type(stor).__name__, stor.getName())
     if stor_name not in _already_warned_notxnraw:
-        logging.warn("%s: storage does not provide IStorageTransactionInformationRaw ...", stor_name)
-        logging.warn("... will do best-effort to dump pickles in stable order but this cannot be done 100% correctly")
-        logging.warn("... please upgrade your ZODB & storage: see https://github.com/zopefoundation/ZODB/pull/183 for details.")
+        logging.warning("%s: storage does not provide IStorageTransactionInformationRaw ...", stor_name)
+        logging.warning("... will do best-effort to dump pickles in stable order but this cannot be done 100% correctly")
+        logging.warning("... please upgrade your ZODB & storage: see https://github.com/zopefoundation/ZODB/pull/183 for details.")
         _already_warned_notxnraw.add(stor_name)
 
     return serializeext(txn.extension)
@@ -125,7 +125,7 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=sys.stdout):
                     out.write(" -")
                 else:
                     out.write("\n")
-                    out.write(obj.data)
+                    out.write(obj.data.decode())
 
             out.write("\n")
 
@@ -306,7 +306,7 @@ class DumpReader(object):
 
     def _readline(self):
         l = self._r.readline()
-        if l == '':
+        if l == b'':
             self._line = None
             return None # EOF
 
@@ -347,7 +347,7 @@ class DumpReader(object):
         objv = []
         while 1:
             l = self._readline()
-            if l == '':
+            if l == b'':
                 break   # empty line - end of transaction
 
             if l is None or not l.startswith(b'obj '):
@@ -371,7 +371,7 @@ class DumpReader(object):
 
             else:
                 size     = int(m.group('size'))
-                hashfunc = m.group('hashfunc')
+                hashfunc = m.group('hashfunc').decode()
                 hashok   = fromhex(m.group('hash'))
                 hashonly = m.group('hashonly') is not None
                 data     = None # see vvv
@@ -390,7 +390,7 @@ class DumpReader(object):
                         chunk = self._r.read(n)
                         data += chunk
                         n -= len(chunk)
-                    self.lineno += data.count('\n')
+                    self.lineno += data.count(b'\n')
                     self._line = None
                     if data[-1:] != b'\n':
                         raise RuntimeError('%s+%d: no LF after obj data' % (_ioname(self._r), self.lineno))
@@ -504,7 +504,7 @@ class ObjectData(Object):
             z += ' -'
         else:
             z += '\n'
-            z += data
+            z += data.decode()
         z += '\n'
         return z
 

[Kirill Smelkov]

@jerome, if I recall correctly when trying this on py3, it is better to use codecs directly -then it will work on both py2 and py3.

[Kirill Smelkov]

(codecs module)

[Kirill Smelkov]

@jerome, thanks, I suggest we extract non-controversial bits from here (e.g. ashex, fromhex, BytesIO) and merge them to master without waiting for full py3 testing coverage. Would you please prepare the patches? Thanks beforehand, Kirill.