More python3 support

Flushing changes from yet another attempt. Still not completely there yet, but closer.

Reviewed-by: @jerome
Reviewed-on: nexedi/zodbtools!16

zodbtools/test/gen_testdata.py

@@ -113,7 +113,7 @@ def ext4subj(subj):
     d[xcookie] = cookie
 
     # shufle extension dict randomly - to likely trigger different ordering on save
-    keyv = d.keys()
+    keyv = list(d.keys())
     random.shuffle(keyv)
     ext = {}
     for key in keyv:
@@ -227,7 +227,7 @@ def _gen_testdb(outfs_path, zext):
                 break
 
         # delete an object
-        name = random.choice(root.keys())
+        name = random.choice(list(root.keys()))
         obj = root[name]
         root[name] = Object("%s%i*" % (name, i))
         # NOTE user/ext are kept empty on purpose - to also test this case
@@ -276,7 +276,7 @@ def main():
             dbname += "_!zext"
         gen_testdb("%s.fs" % dbname, zext=zext)
         stor = FileStorage("%s.fs" % dbname, read_only=True)
-        with open("%s.zdump.ok" % dbname, "w") as f:
+        with open("%s.zdump.ok" % dbname, "wb") as f:
             zodbdump(stor, None, None, out=f)
 
 if __name__ == '__main__':

zodbtools/test/test_commit.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2019  Nexedi SA and Contributors.
+# Copyright (C) 2018-2020  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #                          Jérome Perrin <jerome@nexedi.com>
 #
@@ -58,7 +58,7 @@ def test_zodbcommit(zext):
     zodbdump(stor, p64(u64(head)+1), None, out=buf)
     dumped = buf.getvalue()
 
-    assert dumped == ''.join([_.zdump() for _ in (t1, t2)])
+    assert dumped == b''.join([_.zdump() for _ in (t1, t2)])
 
     # ObjectCopy. XXX zodbcommit handled ObjectCopy by actually copying data,
     # not referencing previous transaction via backpointer.

zodbtools/test/test_dump.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2017-2019  Nexedi SA and Contributors.
+# Copyright (C) 2017-2020  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #                          Jérome Perrin <jerome@nexedi.com>
 #
@@ -39,7 +39,7 @@ def test_zodbdump(zext):
     zkind = '_!zext' if zext.disabled else ''
     stor  = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
 
-    with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
+    with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind), 'rb') as f:
         dumpok = f.read()
 
     out = BytesIO()
@@ -116,11 +116,11 @@ extension "qqq"
 
     assert r.readtxn() == None
 
-    z = ''.join([_.zdump() for _ in (t1, t2)])
+    z = b''.join([_.zdump() for _ in (t1, t2)])
     assert z == in_
 
     # unknown hash function
-    r = DumpReader(BytesIO("""\
+    r = DumpReader(BytesIO(b"""\
 txn 0000000000000000 " "
 user ""
 description ""
@@ -130,10 +130,10 @@ obj 0000000000000001 1 xyz:0123 -
 """))
     with raises(RuntimeError) as exc:
         r.readtxn()
-    assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",)
+    assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ("obj 0000000000000001 1 xyz:0123 -")""",)
 
     # data integrity error
-    r = DumpReader(BytesIO("""\
+    r = DumpReader(BytesIO(b"""\
 txn 0000000000000000 " "
 user ""
 description ""

zodbtools/test/test_tidrange.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2019 Nexedi SA and Contributors.
+# Copyright (C) 2019-2020 Nexedi SA and Contributors.
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -27,6 +27,7 @@ from freezegun import freeze_time
 import tzlocal
 
 from zodbtools.util import TidRangeInvalid, TidInvalid, ashex, parse_tid, parse_tidrange
+from golang import b
 
 
 @pytest.fixture
@@ -94,7 +95,7 @@ def test_parse_tid():
     assert exc.value.args == ('', )
 
 
-test_parameters = []
+test_parameters = [] # of (reference_time, reference_tid, input_time)
 with open(
         os.path.join(
             os.path.dirname(__file__), "testdata",
@@ -109,7 +110,7 @@ with open(
                          test_parameters)
 def test_parse_tid_time_format(fake_time, reference_time, reference_tid,
                                input_time):
-    assert reference_tid == ashex(parse_tid(input_time))
+    assert b(reference_tid) == ashex(parse_tid(input_time))
     # check that the reference_tid matches the reference time, mainly
     # to check that input is defined correctly.
-    assert reference_tid == ashex(parse_tid(reference_time))
+    assert b(reference_tid) == ashex(parse_tid(reference_time))

zodbtools/zodbcommit.py

-# Copyright (C) 2018-2019  Nexedi SA and Contributors.
+# Copyright (C) 2018-2020  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -159,7 +159,7 @@ def main(argv):
     stor = storageFromURL(storurl)
     defer(stor.close)
 
-    zin = 'txn 0000000000000000 " "\n'  # artificial transaction header
+    zin = b'txn 0000000000000000 " "\n'  # artificial transaction header
     zin += sys.stdin.read()
     zin = BytesIO(zin)
     zr = zodbdump.DumpReader(zin)

zodbtools/zodbdump.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2019  Nexedi SA and Contributors.
+# Copyright (C) 2016-2020  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #                          Jérome Perrin <jerome@nexedi.com>
 #
@@ -67,7 +67,7 @@ import sys
 import logging as log
 import re
 from golang.gcompat import qq
-from golang import func, defer, strconv
+from golang import func, defer, strconv, b
 
 # txn_raw_extension returns raw extension from txn metadata
 def txn_raw_extension(stor, txn):
@@ -96,7 +96,7 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
     for txn in stor.iterator(tidmin, tidmax):
         # XXX .status not covered by IStorageTransactionInformation
         # XXX but covered by BaseStorage.TransactionRecord
-        out.write("txn %s %s\nuser %s\ndescription %s\nextension %s\n" % (
+        out.write(b"txn %s %s\nuser %s\ndescription %s\nextension %s\n" % (
             ashex(txn.tid), qq(txn.status),
             qq(txn.user),
             qq(txn.description),
@@ -105,33 +105,33 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
         objv = txnobjv(txn)
 
         for obj in objv:
-            entry = "obj %s " % ashex(obj.oid)
+            entry = b"obj %s " % ashex(obj.oid)
             write_data = False
 
             if obj.data is None:
-                entry += "delete"
+                entry += b"delete"
 
             # was undo and data taken from obj.data_txn
             elif obj.data_txn is not None:
-                entry += "from %s" % ashex(obj.data_txn)
+                entry += b"from %s" % ashex(obj.data_txn)
 
             else:
                 # XXX sha1 is hardcoded for now. Dump format allows other hashes.
-                entry += "%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data)))
+                entry += b"%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data)))
                 write_data = True
 
-            out.write(entry)
+            out.write(b(entry))
 
             if write_data:
                 if hashonly:
-                    out.write(" -")
+                    out.write(b" -")
                 else:
-                    out.write("\n")
+                    out.write(b"\n")
                     out.write(obj.data)
 
-            out.write("\n")
+            out.write(b"\n")
 
-        out.write("\n")
+        out.write(b"\n")
 
 # ----------------------------------------
 # XPickler is Pickler that tries to save objects stably
@@ -309,7 +309,7 @@ class DumpReader(object):
 
     def _readline(self):
         l = self._r.readline()
-        if l == '':
+        if l == b'':
             self._line = None
             return None # EOF
 
@@ -320,7 +320,7 @@ class DumpReader(object):
 
     # report a problem found around currently-read line
     def _badline(self, msg):
-        raise RuntimeError("%s+%d: invalid line: %s (%r)" % (_ioname(self._r), self.lineno, msg, self._line))
+        raise RuntimeError("%s+%d: invalid line: %s (%s)" % (_ioname(self._r), self.lineno, msg, qq(self._line)))
 
     # readtxn reads one transaction record from input stream and returns
     # Transaction instance or None at EOF.
@@ -350,7 +350,7 @@ class DumpReader(object):
         objv = []
         while 1:
             l = self._readline()
-            if l == '':
+            if l == b'':
                 break   # empty line - end of transaction
 
             if l is None or not l.startswith(b'obj '):
@@ -393,7 +393,7 @@ class DumpReader(object):
                         chunk = self._r.read(n)
                         data += chunk
                         n -= len(chunk)
-                    self.lineno += data.count('\n')
+                    self.lineno += data.count(b'\n')
                     self._line = None
                     if data[-1:] != b'\n':
                         raise RuntimeError('%s+%d: no LF after obj data' % (_ioname(self._r), self.lineno))
@@ -452,15 +452,15 @@ class Transaction(object):
     def _extension(self):
         return self.extension
 
-    # zdump returns text representation of a record in zodbdump format.
-    def zdump(self):
-        z  = 'txn %s %s\n' % (ashex(self.tid), qq(self.status))
-        z += 'user %s\n' % qq(self.user)
-        z += 'description %s\n' % qq(self.description)
-        z += 'extension %s\n' % qq(self.extension_bytes)
+    # zdump returns semi text-binary representation of a record in zodbdump format.
+    def zdump(self): # -> bytes
+        z  = b'txn %s %s\n' % (ashex(self.tid), qq(self.status))
+        z += b'user %s\n' % qq(self.user)
+        z += b'description %s\n' % qq(self.description)
+        z += b'extension %s\n' % qq(self.extension_bytes)
         for obj in self.objv:
             z += obj.zdump()
-        z += '\n'
+        z += b'\n'
         return z
 
 
@@ -477,7 +477,7 @@ class ObjectDelete(Object):
         super(ObjectDelete, self).__init__(oid)
 
     def zdump(self):
-        return 'obj %s delete\n' % (ashex(self.oid))
+        return b'obj %s delete\n' % (ashex(self.oid))
 
 # ObjectCopy represents object data copy.
 class ObjectCopy(Object):
@@ -487,7 +487,7 @@ class ObjectCopy(Object):
         self.copy_from = copy_from
 
     def zdump(self):
-        return 'obj %s from %s\n' % (ashex(self.oid), ashex(self.copy_from))
+        return b'obj %s from %s\n' % (ashex(self.oid), ashex(self.copy_from))
 
 # ObjectData represents record with object data.
 class ObjectData(Object):
@@ -507,13 +507,13 @@ class ObjectData(Object):
             size = data.size
         else:
             size = len(data)
-        z = 'obj %s %d %s:%s' % (ashex(self.oid), size, self.hashfunc, ashex(self.hash_))
+        z = b'obj %s %d %s:%s' % (ashex(self.oid), size, self.hashfunc, ashex(self.hash_))
         if hashonly:
-            z += ' -'
+            z += b' -'
         else:
-            z += '\n'
+            z += b'\n'
             z += data
-        z += '\n'
+        z += b'\n'
         return z
 
 # HashOnly indicated that this ObjectData record contains only hash and does not contain object data.