More python3 compatibility

@jerome, I was trying to make zodbtools work with Python3 and along that road picked some bits of your work from nexedi/zodbtools!12. At present the migration to Python3 is not complete, and even though now I have the answer to how handle strings in both python2/3 in compatible and reasonable way (I can share details if you are interested), I have to put that work on hold for some time and use https://pypi.org/project/pep3134 directly in wcfs tests, since getting all string details right, even after figuring on how to do it, will take time. Anyway the bits presented here should be ready for master and could be merged now. Could you please have a look?

Thanks beforehand,  
Kirill

/reviewed-on nexedi/zodbtools!13

setup.py

@@ -23,7 +23,7 @@ setup(
     install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six', 'dateparser'],
 
     extras_require = {
-                  'test': ['pytest', 'freezegun', 'pytz'],
+                  'test': ['pytest', 'freezegun', 'pytz', 'mock;python_version<="2.7"'],
     },
 
     entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']},

tox.ini

@@ -8,6 +8,7 @@ deps	=
     pytest
     freezegun
     pytz
+    mock;python_version<="2.7"
 
     # latest ZODB from ZODB3 series
     ZODB3:  ZODB3 >=3.10, <3.11dev

zodbtools/test/test_commit.py

-# Copyright (C) 2018  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# -*- coding: utf-8 -*-
+# Copyright (C) 2018-2019  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#                          Jérome Perrin <jerome@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -41,8 +43,8 @@ def test_zodbcommit(zext):
     # commit some transactions via zodbcommit and verify if storage dump gives
     # what is expected.
     t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
-        ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
-        ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
+        ObjectData(p64(1), b'data1', 'sha1', sha1(b'data1')),
+        ObjectData(p64(2), b'data2', 'sha1', sha1(b'data2'))])
 
     t1.tid = zodbcommit(stor, head, t1)
 

zodbtools/test/test_dump.py

-# Copyright (C) 2017-2018  Nexedi SA and Contributors.
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
+#                          Jérome Perrin <jerome@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -21,9 +23,10 @@ from zodbtools.zodbdump import (
         zodbdump, DumpReader, Transaction, ObjectDelete, ObjectCopy,
         ObjectData, HashOnly
     )
+from zodbtools.util import fromhex
 from ZODB.FileStorage import FileStorage
 from ZODB.utils import p64
-from cStringIO import StringIO
+from io import BytesIO
 
 from os.path import dirname
 
@@ -39,7 +42,7 @@ def test_zodbdump(zext):
     with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
         dumpok = f.read()
 
-    out = StringIO()
+    out = BytesIO()
     zodbdump(stor, None, None, out=out)
 
     assert out.getvalue() == dumpok
@@ -69,10 +72,10 @@ extension "qqq"
 
 """
 
-    r = DumpReader(StringIO(in_))
+    r = DumpReader(BytesIO(in_))
     t1 = r.readtxn()
     assert isinstance(t1, Transaction)
-    assert t1.tid == '0123456789abcdef'.decode('hex')
+    assert t1.tid == fromhex('0123456789abcdef')
     assert t1.user              == b'my name'
     assert t1.description       == b'o la-la...'
     assert t1.extension_bytes   == b'zzz123 def'
@@ -83,29 +86,29 @@ extension "qqq"
     _ = t1.objv[1]
     assert isinstance(_, ObjectCopy)
     assert _.oid        == p64(2)
-    assert _.copy_from  == '0123456789abcdee'.decode('hex')
+    assert _.copy_from  == fromhex('0123456789abcdee')
     _ = t1.objv[2]
     assert isinstance(_, ObjectData)
     assert _.oid        == p64(3)
     assert _.data       == HashOnly(54)
     assert _.hashfunc   == 'adler32'
-    assert _.hash_      == '01234567'.decode('hex')
+    assert _.hash_      == fromhex('01234567')
     _ = t1.objv[3]
     assert isinstance(_, ObjectData)
     assert _.oid        == p64(4)
     assert _.data       == b'ZZZZ'
     assert _.hashfunc   == 'sha1'
-    assert _.hash_      == '9865d483bc5a94f2e30056fc256ed3066af54d04'.decode('hex')
+    assert _.hash_      == fromhex('9865d483bc5a94f2e30056fc256ed3066af54d04')
     _ = t1.objv[4]
     assert isinstance(_, ObjectData)
     assert _.oid        == p64(5)
     assert _.data       == b'ABC\n\nDEF!'
     assert _.hashfunc   == 'crc32'
-    assert _.hash_      == '52fdeac5'.decode('hex')
+    assert _.hash_      == fromhex('52fdeac5')
 
     t2 = r.readtxn()
     assert isinstance(t2, Transaction)
-    assert t2.tid == '0123456789abcdf0'.decode('hex')
+    assert t2.tid == fromhex('0123456789abcdf0')
     assert t2.user              == b'author2'
     assert t2.description       == b'zzz'
     assert t2.extension_bytes   == b'qqq'
@@ -117,7 +120,7 @@ extension "qqq"
     assert z == in_
 
     # unknown hash function
-    r = DumpReader(StringIO("""\
+    r = DumpReader(BytesIO("""\
 txn 0000000000000000 " "
 user ""
 description ""
@@ -130,7 +133,7 @@ obj 0000000000000001 1 xyz:0123 -
     assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",)
 
     # data integrity error
-    r = DumpReader(StringIO("""\
+    r = DumpReader(BytesIO("""\
 txn 0000000000000000 " "
 user ""
 description ""

zodbtools/test/test_zodb.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Jérome Perrin <jerome@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import sys
+try:
+    from unittest import mock
+except ImportError:
+    # BBB python2
+    import mock
+
+import pytest
+
+from zodbtools import zodb
+from zodbtools import help as help_module
+
+
+# zodbrun runs zodb.main with argv and returns exit code + captured stdout/stderr.
+def zodbrun(capsys, *argv):
+    with mock.patch.object(sys, 'argv', ('zodb',) + argv), \
+         pytest.raises(SystemExit) as excinfo:
+        zodb.main()
+    assert len(excinfo.value.args) == 1
+    ecode = excinfo.value.args[0]
+    return ecode, capsys.readouterr()
+
+
+def test_main(capsys):
+    e, _ = zodbrun(capsys)
+    assert e == 2
+    assert "" == _.out
+    assert "Zodb is a tool for managing ZODB databases." in _.err
+
+    e, _ = zodbrun(capsys, '-h')
+    assert e == 0
+    assert "Zodb is a tool for managing ZODB databases." in _.out
+    assert "" == _.err
+
+
+@pytest.mark.parametrize(
+    "help_topic",
+    tuple(zodb.command_dict) + tuple(help_module.topic_dict))
+def test_help(capsys, help_topic):
+    e, _ = zodbrun(capsys, 'help', help_topic)
+    assert e == 0
+    assert _.err == ""
+    assert _.out != ""

zodbtools/util.py

+# -*- coding: utf-8 -*-
 # zodbtools - various utility routines
-# Copyright (C) 2016-2018  Nexedi SA and Contributors.
+# Copyright (C) 2016-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
+#                          Jérome Perrin <jerome@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -18,7 +20,7 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
-import hashlib, struct, codecs
+import hashlib, struct, codecs, io
 import zodburi
 from six.moves.urllib_parse import urlsplit, urlunsplit
 from zlib import crc32, adler32
@@ -26,12 +28,15 @@ from ZODB.TimeStamp import TimeStamp
 import dateparser
 
 def ashex(s):
-    return s.encode('hex')
+    # type: (bytes) -> bytes
+    return codecs.encode(s, 'hex')
 
 def fromhex(s):
+    # type: (Union[str,bytes]) -> bytes
     return codecs.decode(s, 'hex')
 
 def sha1(data):
+    # type: (bytes) -> bytes
     m = hashlib.sha1()
     m.update(data)
     return m.digest()
@@ -183,7 +188,7 @@ class Adler32Hasher:
     digest_size = 4
 
     def __init__(self):
-        self._h = adler32('')
+        self._h = adler32(b'')
 
     def update(self, data):
         self._h = adler32(data, self._h)
@@ -200,7 +205,7 @@ class CRC32Hasher:
     digest_size = 4
 
     def __init__(self):
-        self._h = crc32('')
+        self._h = crc32(b'')
 
     def update(self, data):
         self._h = crc32(data, self._h)
@@ -220,3 +225,13 @@ hashRegistry = {
     "sha256":   hashlib.sha256,
     "sha512":   hashlib.sha512,
 }
+
+# ---- IO ----
+
+# asbinstream return binary stream associated with stream.
+# For example on py3 sys.stdout is io.TextIO which does not allow to write binary data to it.
+def asbinstream(stream):
+    # type: (IO) -> BinaryIO
+    if isinstance(stream, io.TextIOBase):
+        return stream.buffer
+    return stream

zodbtools/zodb.py

 #!/usr/bin/env python
-# Copyright (C) 2017-2018  Nexedi SA and Contributors.
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
+#                          Jérome Perrin <jerome@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -51,10 +53,7 @@ Usage:
 The commands are:
 """, file=out)
 
-    cmdv = command_dict.keys()
-    cmdv.sort()
-    for cmd in cmdv:
-        cmd_module = command_dict[cmd]
+    for cmd, cmd_module in sorted(command_dict.items()):
         print("    %-11s %s" % (cmd, cmd_module.summary), file=out)
 
     print("""\

zodbtools/zodbcommit.py

-# Copyright (C) 2018  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2018-2019  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -40,7 +40,7 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`.
 
 from __future__ import print_function
 from zodbtools import zodbdump
-from zodbtools.util import ashex, storageFromURL
+from zodbtools.util import ashex, fromhex, storageFromURL
 from ZODB.utils import p64, u64, z64
 from ZODB.POSException import POSKeyError
 from ZODB._compat import BytesIO
@@ -154,7 +154,7 @@ def main(argv):
         sys.exit(2)
 
     storurl = argv[0]
-    at = argv[1].decode('hex')
+    at = fromhex(argv[1])
 
     stor = storageFromURL(storurl)
     defer(stor.close)

zodbtools/zodbdump.py

-# Copyright (C) 2016-2018  Nexedi SA and Contributors.
+# -*- coding: utf-8 -*-
+# Copyright (C) 2016-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
+#                          Jérome Perrin <jerome@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -54,7 +56,7 @@ TODO also protect txn record by hash.
 
 from __future__ import print_function
 from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRangeInvalid,   \
-        storageFromURL, hashRegistry
+        storageFromURL, hashRegistry, asbinstream
 from ZODB._compat import loads, _protocol, BytesIO
 from zodbpickle.slowpickle import Pickler as pyPickler
 #import pickletools
@@ -90,7 +92,7 @@ _already_warned_notxnraw = set()
 
 # zodbdump dumps content of a ZODB storage to a file.
 # please see module doc-string for dump format and details
-def zodbdump(stor, tidmin, tidmax, hashonly=False, out=sys.stdout):
+def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
     for txn in stor.iterator(tidmin, tidmax):
         # XXX .status not covered by IStorageTransactionInformation
         # XXX but covered by BaseStorage.TransactionRecord