Commit b44f9c0d authored by Kirill Smelkov's avatar Kirill Smelkov

More python3 compatibility

@jerome, I was trying to make zodbtools work with Python3 and along that road picked some bits of your work from nexedi/zodbtools!12. At present the migration to Python3 is not complete, and even though now I have the answer to how handle strings in both python2/3 in compatible and reasonable way (I can share details if you are interested), I have to put that work on hold for some time and use https://pypi.org/project/pep3134 directly in wcfs tests, since getting all string details right, even after figuring on how to do it, will take time. Anyway the bits presented here should be ready for master and could be merged now. Could you please have a look?

Thanks beforehand,  
Kirill

/reviewed-on nexedi/zodbtools!13
parents bcaf3984 c5f20201
......@@ -23,7 +23,7 @@ setup(
install_requires = ['ZODB', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six', 'dateparser'],
extras_require = {
'test': ['pytest', 'freezegun', 'pytz'],
'test': ['pytest', 'freezegun', 'pytz', 'mock;python_version<="2.7"'],
},
entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']},
......
......@@ -8,6 +8,7 @@ deps =
pytest
freezegun
pytz
mock;python_version<="2.7"
# latest ZODB from ZODB3 series
ZODB3: ZODB3 >=3.10, <3.11dev
......
# Copyright (C) 2018 Nexedi SA and Contributors.
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -41,8 +43,8 @@ def test_zodbcommit(zext):
# commit some transactions via zodbcommit and verify if storage dump gives
# what is expected.
t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
ObjectData(p64(1), b'data1', 'sha1', sha1(b'data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1(b'data2'))])
t1.tid = zodbcommit(stor, head, t1)
......
# Copyright (C) 2017-2018 Nexedi SA and Contributors.
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -21,9 +23,10 @@ from zodbtools.zodbdump import (
zodbdump, DumpReader, Transaction, ObjectDelete, ObjectCopy,
ObjectData, HashOnly
)
from zodbtools.util import fromhex
from ZODB.FileStorage import FileStorage
from ZODB.utils import p64
from cStringIO import StringIO
from io import BytesIO
from os.path import dirname
......@@ -39,7 +42,7 @@ def test_zodbdump(zext):
with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
dumpok = f.read()
out = StringIO()
out = BytesIO()
zodbdump(stor, None, None, out=out)
assert out.getvalue() == dumpok
......@@ -69,10 +72,10 @@ extension "qqq"
"""
r = DumpReader(StringIO(in_))
r = DumpReader(BytesIO(in_))
t1 = r.readtxn()
assert isinstance(t1, Transaction)
assert t1.tid == '0123456789abcdef'.decode('hex')
assert t1.tid == fromhex('0123456789abcdef')
assert t1.user == b'my name'
assert t1.description == b'o la-la...'
assert t1.extension_bytes == b'zzz123 def'
......@@ -83,29 +86,29 @@ extension "qqq"
_ = t1.objv[1]
assert isinstance(_, ObjectCopy)
assert _.oid == p64(2)
assert _.copy_from == '0123456789abcdee'.decode('hex')
assert _.copy_from == fromhex('0123456789abcdee')
_ = t1.objv[2]
assert isinstance(_, ObjectData)
assert _.oid == p64(3)
assert _.data == HashOnly(54)
assert _.hashfunc == 'adler32'
assert _.hash_ == '01234567'.decode('hex')
assert _.hash_ == fromhex('01234567')
_ = t1.objv[3]
assert isinstance(_, ObjectData)
assert _.oid == p64(4)
assert _.data == b'ZZZZ'
assert _.hashfunc == 'sha1'
assert _.hash_ == '9865d483bc5a94f2e30056fc256ed3066af54d04'.decode('hex')
assert _.hash_ == fromhex('9865d483bc5a94f2e30056fc256ed3066af54d04')
_ = t1.objv[4]
assert isinstance(_, ObjectData)
assert _.oid == p64(5)
assert _.data == b'ABC\n\nDEF!'
assert _.hashfunc == 'crc32'
assert _.hash_ == '52fdeac5'.decode('hex')
assert _.hash_ == fromhex('52fdeac5')
t2 = r.readtxn()
assert isinstance(t2, Transaction)
assert t2.tid == '0123456789abcdf0'.decode('hex')
assert t2.tid == fromhex('0123456789abcdf0')
assert t2.user == b'author2'
assert t2.description == b'zzz'
assert t2.extension_bytes == b'qqq'
......@@ -117,7 +120,7 @@ extension "qqq"
assert z == in_
# unknown hash function
r = DumpReader(StringIO("""\
r = DumpReader(BytesIO("""\
txn 0000000000000000 " "
user ""
description ""
......@@ -130,7 +133,7 @@ obj 0000000000000001 1 xyz:0123 -
assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",)
# data integrity error
r = DumpReader(StringIO("""\
r = DumpReader(BytesIO("""\
txn 0000000000000000 " "
user ""
description ""
......
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Jérome Perrin <jerome@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import sys
try:
from unittest import mock
except ImportError:
# BBB python2
import mock
import pytest
from zodbtools import zodb
from zodbtools import help as help_module
# zodbrun runs zodb.main with argv and returns exit code + captured stdout/stderr.
def zodbrun(capsys, *argv):
with mock.patch.object(sys, 'argv', ('zodb',) + argv), \
pytest.raises(SystemExit) as excinfo:
zodb.main()
assert len(excinfo.value.args) == 1
ecode = excinfo.value.args[0]
return ecode, capsys.readouterr()
def test_main(capsys):
e, _ = zodbrun(capsys)
assert e == 2
assert "" == _.out
assert "Zodb is a tool for managing ZODB databases." in _.err
e, _ = zodbrun(capsys, '-h')
assert e == 0
assert "Zodb is a tool for managing ZODB databases." in _.out
assert "" == _.err
@pytest.mark.parametrize(
"help_topic",
tuple(zodb.command_dict) + tuple(help_module.topic_dict))
def test_help(capsys, help_topic):
e, _ = zodbrun(capsys, 'help', help_topic)
assert e == 0
assert _.err == ""
assert _.out != ""
# -*- coding: utf-8 -*-
# zodbtools - various utility routines
# Copyright (C) 2016-2018 Nexedi SA and Contributors.
# Copyright (C) 2016-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -18,7 +20,7 @@
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import hashlib, struct, codecs
import hashlib, struct, codecs, io
import zodburi
from six.moves.urllib_parse import urlsplit, urlunsplit
from zlib import crc32, adler32
......@@ -26,12 +28,15 @@ from ZODB.TimeStamp import TimeStamp
import dateparser
def ashex(s):
return s.encode('hex')
# type: (bytes) -> bytes
return codecs.encode(s, 'hex')
def fromhex(s):
# type: (Union[str,bytes]) -> bytes
return codecs.decode(s, 'hex')
def sha1(data):
# type: (bytes) -> bytes
m = hashlib.sha1()
m.update(data)
return m.digest()
......@@ -183,7 +188,7 @@ class Adler32Hasher:
digest_size = 4
def __init__(self):
self._h = adler32('')
self._h = adler32(b'')
def update(self, data):
self._h = adler32(data, self._h)
......@@ -200,7 +205,7 @@ class CRC32Hasher:
digest_size = 4
def __init__(self):
self._h = crc32('')
self._h = crc32(b'')
def update(self, data):
self._h = crc32(data, self._h)
......@@ -220,3 +225,13 @@ hashRegistry = {
"sha256": hashlib.sha256,
"sha512": hashlib.sha512,
}
# ---- IO ----
# asbinstream return binary stream associated with stream.
# For example on py3 sys.stdout is io.TextIO which does not allow to write binary data to it.
def asbinstream(stream):
# type: (IO) -> BinaryIO
if isinstance(stream, io.TextIOBase):
return stream.buffer
return stream
#!/usr/bin/env python
# Copyright (C) 2017-2018 Nexedi SA and Contributors.
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -51,10 +53,7 @@ Usage:
The commands are:
""", file=out)
cmdv = command_dict.keys()
cmdv.sort()
for cmd in cmdv:
cmd_module = command_dict[cmd]
for cmd, cmd_module in sorted(command_dict.items()):
print(" %-11s %s" % (cmd, cmd_module.summary), file=out)
print("""\
......
# Copyright (C) 2018 Nexedi SA and Contributors.
# Copyright (C) 2018-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -40,7 +40,7 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`.
from __future__ import print_function
from zodbtools import zodbdump
from zodbtools.util import ashex, storageFromURL
from zodbtools.util import ashex, fromhex, storageFromURL
from ZODB.utils import p64, u64, z64
from ZODB.POSException import POSKeyError
from ZODB._compat import BytesIO
......@@ -154,7 +154,7 @@ def main(argv):
sys.exit(2)
storurl = argv[0]
at = argv[1].decode('hex')
at = fromhex(argv[1])
stor = storageFromURL(storurl)
defer(stor.close)
......
# Copyright (C) 2016-2018 Nexedi SA and Contributors.
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -54,7 +56,7 @@ TODO also protect txn record by hash.
from __future__ import print_function
from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRangeInvalid, \
storageFromURL, hashRegistry
storageFromURL, hashRegistry, asbinstream
from ZODB._compat import loads, _protocol, BytesIO
from zodbpickle.slowpickle import Pickler as pyPickler
#import pickletools
......@@ -90,7 +92,7 @@ _already_warned_notxnraw = set()
# zodbdump dumps content of a ZODB storage to a file.
# please see module doc-string for dump format and details
def zodbdump(stor, tidmin, tidmax, hashonly=False, out=sys.stdout):
def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
for txn in stor.iterator(tidmin, tidmax):
# XXX .status not covered by IStorageTransactionInformation
# XXX but covered by BaseStorage.TransactionRecord
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment