dump, commit: Test for both non-empty and empty transaction extensions

Currently we exercise zodbdump and zodbcommit+zodbdump with non-empty
extensions, which works if ZODB is patched for txn.extension_bytes
support, but fails on pristine ZODB.

Support for txn.extension_bytes cannot get into upstream ZODB for
more than a year:

https://github.com/zopefoundation/ZODB/pull/183
https://github.com/zopefoundation/ZODB/pull/207

and  even if it somehow will make it, it will likely be only in ZODB5,
while we still care to support ZODB4 and ZODB3.

Skipping zodbdump / zodbcommit tests, if a ZODB does not have
txn.extension_bytes support, would result in significant reduction of
zodbtools test coverage, because practically that is the current
situation with all upstream ZODB{3,4,5}. Dropping test coverage for
non-empty extensions is neither a good option.

For those reason, let's rework the tests and test both zodbdump and
zodbcommit with two scenarios:

1. on a test database where transactions extensions are always empty.
   This should work on all ZODB irregardless of whether
   txn.extension_bytes patch is there or not.

2. on a test database where transactions extensions are present.
   This should work if ZODB has txn.extension_bytes support, but if not,
   we can mark this case as xfail, since the failure is expected.

This way we make the testsuite pass irregardless of whether
txn.extension_bytes support is there, and we don't abandon dump/commit
testing coverage.

zodbtools/test/conftest.py

+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import pytest
+from zodbtools.test.testutil import zext_supported
+
+# zext is a test fixture function object that allows to exercise 2 cases:
+#
+# - when ZODB does not have txn.extension_bytes support
+# - when ZODB might have    txn.extension_bytes support
+#
+# in a test, zext should be used as as follows:
+#
+#   def test_something(zext):
+#       # bytes for an extension dict
+#       raw_ext = dumps({...})
+#
+#       # will be either same as raw_ext, or b'' if ZODB lacks txn.extension_bytes support
+#       raw_ext = zext(raw_ext)
+#
+#       # zext.disabled indicates whether testing for non-empty extension was disabled.
+#       if zext.disabled:
+#           ...
+@pytest.fixture(params=['!zext', 'zext'])
+def zext(request):
+    if request.param == '!zext':
+        # txn.extension_bytes is not working - always test with empty extension
+        def _(ext):
+            return b''
+        _.disabled = True
+        return _
+    else:
+        # txn.extension_bytes might be working - test with given extension and
+        # xfail if ZODB does not have necessary support.
+        def _(ext):
+            if not zext_supported():
+                pytest.xfail('ZODB does not have txn.extension_bytes support')
+            return ext
+        _.disabled = False
+        return _

zodbtools/test/gen_testdata.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2017-2019  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -60,7 +60,12 @@ def hex64(packed):
     return '0x%016x' % unpack64(packed)
 
 # make time.time() predictable
-_xtime = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
+_xtime0 = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
+def xtime_reset():
+    global _xtime
+    _xtime = _xtime0
+xtime_reset()
+
 def xtime():
     global _xtime
     _xtime += 1.1
@@ -115,8 +120,16 @@ def ext(subj):
 
     return ext
 
-# gen_testdb generates test FileStorage database @ outfs_path
-def gen_testdb(outfs_path):
+# gen_testdb generates test FileStorage database @ outfs_path.
+#
+# zext indicates whether or not to include non-empty extension into transactions.
+def gen_testdb(outfs_path, zext=True):
+    xtime_reset()
+
+    ext = globals()['ext']
+    if not zext:
+        def ext(subj): return {}
+
     logging.basicConfig()
 
     # generate random changes to objects hooked to top-level root by a/b/c/... key
@@ -196,13 +209,22 @@ def gen_testdb(outfs_path):
 # ----------------------------------------
 
 from zodbtools.zodbdump import zodbdump
+from zodbtools.test.testutil import zext_supported
 
 def main():
+    # check that ZODB supports txn.extension_bytes; refuse to work if not.
+    if not zext_supported():
+        raise RuntimeError("gen_testdata must be used with ZODB that supports txn.extension_bytes")
+
     out = "testdata/1"
-    gen_testdb("%s.fs" % out)
-    stor = FileStorage("%s.fs" % out, read_only=True)
-    with open("%s.zdump.ok" % out, "w") as f:
-        zodbdump(stor, None, None, out=f)
+    for zext in [True, False]:
+        dbname = out
+        if not zext:
+            dbname += "_!zext"
+        gen_testdb("%s.fs" % dbname, zext=zext)
+        stor = FileStorage("%s.fs" % dbname, read_only=True)
+        with open("%s.zdump.ok" % dbname, "w") as f:
+            zodbdump(stor, None, None, out=f)
 
 if __name__ == '__main__':
     main()

zodbtools/test/test_commit.py

@@ -29,7 +29,7 @@ from golang import func, defer
 
 # verify zodbcommit.
 @func
-def test_zodbcommit():
+def test_zodbcommit(zext):
     tmpd = mkdtemp('', 'zodbcommit.')
     defer(lambda: rmtree(tmpd))
 
@@ -40,7 +40,7 @@ def test_zodbcommit():
 
     # commit some transactions via zodbcommit and verify if storage dump gives
     # what is expected.
-    t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [
+    t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
         ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
         ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
 

zodbtools/test/test_dump.py

@@ -27,20 +27,28 @@ from cStringIO import StringIO
 
 from os.path import dirname
 
-from pytest import raises
+from zodbtools.test.testutil import zext_supported
+from pytest import raises, xfail
 
 # verify zodbdump output against golden
-def test_zodbdump():
-    tdir = dirname(__file__)
-    stor = FileStorage('%s/testdata/1.fs' % tdir, read_only=True)
+def test_zodbdump(zext):
+    tdir  = dirname(__file__)
+    zkind = '_!zext' if zext.disabled else ''
+    stor  = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
 
-    with open('%s/testdata/1.zdump.ok' % tdir) as f:
+    with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
         dumpok = f.read()
 
     out = StringIO()
     zodbdump(stor, None, None, out=out)
 
-    assert out.getvalue() == dumpok
+    try:
+        assert out.getvalue() == dumpok
+    except AssertionError:
+        if not zext.disabled and not zext_supported():
+            xfail('zodbdump cannot read raw extension without ZODB txn.extension_bytes support')
+        raise
+
 
 
 # verify zodbdump.DumpReader

zodbtools/test/testutil.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""utilities for testing"""
+
+from ZODB.FileStorage import FileStorage
+from ZODB import DB
+import transaction
+
+from tempfile import mkdtemp
+from shutil import rmtree
+from golang import func, defer
+
+# zext_supported checks whether ZODB supports txn.extension_bytes .
+_zext_supported_memo = None
+def zext_supported():
+    global _zext_supported_memo
+    if _zext_supported_memo is not None:
+        return _zext_supported_memo
+
+    _ = _zext_supported_memo = _zext_supported()
+    return _
+
+@func
+def _zext_supported():
+    tmpd = mkdtemp('', 'zext_check.')
+    defer(lambda: rmtree(tmpd))
+    dbfs = tmpd + '/1.fs'
+
+    stor = FileStorage(dbfs, create=True)
+    db   = DB(stor)
+    conn = db.open()
+    root = conn.root()
+    root._p_changed = True
+
+    txn = transaction.get()
+    txn.setExtendedInfo('a', 'b')
+    txn.commit()
+
+    for last_txn in stor.iterator(start=stor.lastTransaction()):
+        break
+    else:
+        assert False, "cannot see details of last transaction"
+
+    assert last_txn.extension == {'a': 'b'}
+    return hasattr(last_txn, 'extension_bytes')