wcfs: zdata: Test ZBlk loading on all py2/py3 ZODB kinds of data we care about

Previously we were testing ZBlk loading on Go side only with data generated by
python2 and pickle protocol=2. However even on py2 there are more pickle
protocols that are in use, and also there is python3.

-> Modernize testdata/zblk_test_gen.py to use run_with_all_zodb_pickle_kinds
   that was recently added as part of nexedi/zodbtools@f9d36ba7
   and generate test data with both python2 and python3. It is handy to
   use py2py3-venv(*) to prepare python environment to do that.

   Adjust tests on Go side to verify how ZBlk is loaded for all generated zkinds.

py2_pickle1, py2_pickle2 and py2_pickle3 are handled well.
ZBlk test for py3_pickle3 currently fails with

        --- FAIL: TestZBlk/py3_pickle3 (0.01s)
    panic: ZBlk0(0000000000000002): loadBlkData: wendelin.bigfile.file_zodb.ZBlk0(0000000000000002): activate: pysetstate: expect str; got ogórek.Bytes [recovered]

and so is marked with "xfail".

We will fix tests for py3_pickle3 in follow-up patches.

(*) see nexedi/zodbtools@fac2f190

wcfs/internal/zdata/testdata/.gitignore

-/*.lock
-/*.tmp
-/*.tr[0-9]
+*.lock
+*.tmp
+*.tr[0-9]

wcfs/internal/zdata/testdata/zblk_test_gen.py

-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2021  Nexedi SA and Contributors.
+# Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -20,25 +20,33 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """zblk_test_gen.py generates test data for zblk_test.go"""
 
+from __future__ import print_function
+
 from ZODB.DB import DB
 from ZODB.utils import u64
 from wendelin.bigfile.file_zodb import ZBlk0, ZBlk1, ZBigFile
 from BTrees.IOBTree import IOBTree, IOBucket
 from numpy import arange
-import os, os.path, transaction
+import os, os.path, shutil, transaction
 
 import zodbtools.test.gen_testdata  # to make time predictable
-from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
+from zodbtools.test.gen_testdata import run_with_all_zodb_pickle_kinds, current_zkind, xtime_reset
 
 K = 1024
 
 def main():
-    run_with_zodb4py2_compat(main2)
+    run_with_all_zodb_pickle_kinds(main2)
 
 def main2():
-    outfs = "testdata/zblk.fs"
-    rm_f(outfs)
-    rm_f(outfs + ".index")
+    xtime_reset()
+
+    zkind = current_zkind()
+    prefix = "testdata/%s" % zkind
+    if os.path.exists(prefix):
+        shutil.rmtree(prefix)
+    os.makedirs(prefix)
+
+    outfs = "%s/zblk.fs" % prefix
     db = DB(outfs)
     conn = db.open()
     root = conn.root()
@@ -67,22 +75,27 @@ def main2():
 
     transaction.commit()
 
-    with open("ztestdata_zblk_test.go", "w") as f:
+    with open("ztestdata_zblk_%s_test.go" % zkind, "w") as f:
         def emit(v):
-            print >>f, v
-        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+            print(v, file=f)
+        emit("// Code generated by %s; DO NOT EDIT." % os.path.relpath(__file__))
         emit("package zdata\n")
         emit('import "lab.nexedi.com/kirr/neo/go/zodb"\n')
 
-        emit("const zf_blksize = %d" % zf.blksize)
-        emit("const zf_size    = %d" % ((zf.blktab.maxKey()+1)*zf.blksize))
-        emit("const z0_oid     = zodb.Oid(%d)" % u64(z0._p_oid))
-        emit("const z1_oid     = zodb.Oid(%d)" % u64(z1._p_oid))
-        emit("const zf_oid     = zodb.Oid(%d)" % u64(zf._p_oid))
-        emit("const z0_rev     = zodb.Tid(0x%x)"  % u64(z0._p_serial))
-        emit("const z1_rev     = zodb.Tid(0x%x)"  % u64(z1._p_serial))
-        emit("const z0_len     = %d" % len(z0.loadblkdata()))
-        emit("const z1_htlen   = %d" % z1ht)
+        emit("func init() {")
+        emit('\tzblkTestDataRegistry["%s"] = ZBlkTestData{' % zkind)
+        emit('\t\tdata_fs    : "%s",' % outfs)
+        emit("\t\tzf_blksize : %d," % zf.blksize)
+        emit("\t\tzf_size    : %d," % ((zf.blktab.maxKey()+1)*zf.blksize))
+        emit("\t\tz0_oid     : zodb.Oid(%d)," % u64(z0._p_oid))
+        emit("\t\tz1_oid     : zodb.Oid(%d)," % u64(z1._p_oid))
+        emit("\t\tzf_oid     : zodb.Oid(%d)," % u64(zf._p_oid))
+        emit("\t\tz0_rev     : zodb.Tid(0x%x),"  % u64(z0._p_serial))
+        emit("\t\tz1_rev     : zodb.Tid(0x%x),"  % u64(z1._p_serial))
+        emit("\t\tz0_len     : %d," % len(z0.loadblkdata()))
+        emit("\t\tz1_htlen   : %d," % z1ht)
+        emit("\t}")
+        emit("}")
 
 
     conn.close()
@@ -119,11 +132,6 @@ def assertIOBTreeHas2Buckets(t):
     assert isinstance(_[0][2], IOBucket), _[0][2] # bucket1
     assert isinstance(_[1],    IOBucket), _[1]    # .firstbucket
 
-# rm_f is like `rm -f` in shell.
-def rm_f(path):
-    if os.path.exists(path):
-        os.remove(path)
-
 
 if __name__ == '__main__':
     main()

wcfs/internal/zdata/zblk_test.go

-// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+// Copyright (C) 2018-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -19,12 +19,14 @@
 
 package zdata
 
-//go:generate ./testdata/zblk_test_gen.py
+//go:generate python2 ./testdata/zblk_test_gen.py
+//go:generate python3 ./testdata/zblk_test_gen.py
 
 import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"sort"
 	"testing"
 
 	"lab.nexedi.com/kirr/go123/exc"
@@ -35,13 +37,49 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+// ZBlkTestData describes one ZBlk test data generated by zblk_test_gen.py for particular zkind .
+type ZBlkTestData struct {
+	data_fs    string   // e.g. "testdata/zblk-py2-pickle3.fs"
+	zf_blksize int64    // root.zbigf.blksize
+	zf_size	   int64    // size of zbigf data
+	z0_oid     zodb.Oid // oid of root.zblk0
+	z1_oid     zodb.Oid // oid of root.zblk1
+	zf_oid     zodb.Oid // oid of root.zbigf
+	z0_rev     zodb.Tid // serial of zblk0
+	z1_rev     zodb.Tid // serial of zblk1
+	z0_len     int      // len of zblk0 data
+	z1_htlen   int      // len of head and tail in zblk1
+}
+
+// each generated testdata kind is registered in zblkTestDataRegistry.
+var zblkTestDataRegistry = map[/*zkind*/string]ZBlkTestData{}
+
+
 // TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go.
-// TODO also test with data saved by Python3.
 func TestZBlk(t *testing.T) {
+	// run the test for all generated testdata kinds
+	zkindv := []string{}
+	for zkind := range zblkTestDataRegistry {
+		zkindv = append(zkindv, zkind)
+	}
+	sort.Strings(zkindv)
+
+	for _, zkind := range zkindv {
+		z := zblkTestDataRegistry[zkind]
+		t.Run(zkind, func(t *testing.T) {
+			if zkind == "py3_pickle3" {
+				t.Skip("xfail")
+			}
+			_TestZBlk(t, z)
+		})
+	}
+}
+
+func _TestZBlk(t *testing.T, z ZBlkTestData) {
 	X := exc.Raiseif
 	assert := require.New(t)
 	ctx := context.Background()
-	stor, err := zodb.Open(ctx, "testdata/zblk.fs", &zodb.OpenOptions{ReadOnly: true}); X(err)
+	stor, err := zodb.Open(ctx, z.data_fs, &zodb.OpenOptions{ReadOnly: true}); X(err)
 	db := zodb.NewDB(stor, &zodb.DBOptions{})
 	defer func() {
 		err := db.Close();	X(err)
@@ -53,9 +91,9 @@ func TestZBlk(t *testing.T) {
 
 	conn, err := db.Open(ctx, &zodb.ConnOptions{});	X(err)
 
-	xz0, err := conn.Get(ctx, z0_oid);	X(err)
-	xz1, err := conn.Get(ctx, z1_oid);	X(err)
-	xzf, err := conn.Get(ctx, zf_oid);	X(err)
+	xz0, err := conn.Get(ctx, z.z0_oid);	X(err)
+	xz1, err := conn.Get(ctx, z.z1_oid);	X(err)
+	xzf, err := conn.Get(ctx, z.zf_oid);	X(err)
 
 	z0, ok := xz0.(*ZBlk0)
 	if !ok {
@@ -82,22 +120,22 @@ func TestZBlk(t *testing.T) {
 
 
 	z0Data, z0Rev, err := z0.LoadBlkData(ctx);	X(err)
-	z0DataOK := brange32(z0_len)
+	z0DataOK := brange32(z.z0_len)
 	assert.Equal(z0Data, z0DataOK,   "ZBlk0 data wrong")
-	assert.Equal(z0Rev,  z0_rev,	 "ZBlk0 rev wrong")
+	assert.Equal(z0Rev,  z.z0_rev,   "ZBlk0 rev wrong")
 
 	z1Data, z1Rev, err := z1.LoadBlkData(ctx);	X(err)
-	z1DataOK := make([]byte, zf_blksize)                                  // zeros
-	copy(z1DataOK[0:],                      brange32(z1_htlen))           // head
-	copy(z1DataOK[len(z1DataOK)-z1_htlen:], breverse(brange32(z1_htlen))) // tail
+	z1DataOK := make([]byte, z.zf_blksize)                                    // zeros
+	copy(z1DataOK[0:],                        brange32(z.z1_htlen))           // head
+	copy(z1DataOK[len(z1DataOK)-z.z1_htlen:], breverse(brange32(z.z1_htlen))) // tail
 	z1DataOK = bytes.TrimRight(z1DataOK, "\x00") // trailing 0 are not persisted
 	assert.Equal(z1Data, z1DataOK,   "ZBlk1 data wrong")
-	assert.Equal(z1Rev,  z1_rev,     "ZBlk1 rev wrong")
+	assert.Equal(z1Rev,  z.z1_rev,   "ZBlk1 rev wrong")
 
 
 	xactivate(zf)
-	if zf.blksize != zf_blksize {
-		t.Fatalf("zf: blksize=%d;  want %d", zf.blksize, zf_blksize)
+	if zf.blksize != z.zf_blksize {
+		t.Fatalf("zf: blksize=%d;  want %d", zf.blksize, z.zf_blksize)
 	}
 
 	z0_, ok, err := zf.blktab.Get(ctx, 1);	X(err)
@@ -111,7 +149,7 @@ func TestZBlk(t *testing.T) {
 	}
 
 	size, _, _, err := zf.Size(ctx);	X(err)
-	assert.Equal(size, int64(zf_size),	"ZBigFile size wrong")
+	assert.Equal(size, z.zf_size,	"ZBigFile size wrong")
 
 
 	// LoadBlk

wcfs/internal/zdata/ztestdata_zblk_py2_pickle1_test.go

+// Code generated by testdata/zblk_test_gen.py; DO NOT EDIT.
+package zdata
+
+import "lab.nexedi.com/kirr/neo/go/zodb"
+
+func init() {
+	zblkTestDataRegistry["py2_pickle1"] = ZBlkTestData{
+		data_fs    : "testdata/py2_pickle1/zblk.fs",
+		zf_blksize : 2097152,
+		zf_size    : 8388608,
+		z0_oid     : zodb.Oid(2),
+		z1_oid     : zodb.Oid(3),
+		zf_oid     : zodb.Oid(1),
+		z0_rev     : zodb.Tid(0x285cbac3851eb99),
+		z1_rev     : zodb.Tid(0x285cbac3851eb99),
+		z0_len     : 16384,
+		z1_htlen   : 131072,
+	}
+}

wcfs/internal/zdata/ztestdata_zblk_py2_pickle2_test.go

+// Code generated by testdata/zblk_test_gen.py; DO NOT EDIT.
+package zdata
+
+import "lab.nexedi.com/kirr/neo/go/zodb"
+
+func init() {
+	zblkTestDataRegistry["py2_pickle2"] = ZBlkTestData{
+		data_fs    : "testdata/py2_pickle2/zblk.fs",
+		zf_blksize : 2097152,
+		zf_size    : 8388608,
+		z0_oid     : zodb.Oid(2),
+		z1_oid     : zodb.Oid(3),
+		zf_oid     : zodb.Oid(1),
+		z0_rev     : zodb.Tid(0x285cbac3851eb99),
+		z1_rev     : zodb.Tid(0x285cbac3851eb99),
+		z0_len     : 16384,
+		z1_htlen   : 131072,
+	}
+}

wcfs/internal/zdata/ztestdata_zblk_py2_pickle3_test.go

+// Code generated by testdata/zblk_test_gen.py; DO NOT EDIT.
+package zdata
+
+import "lab.nexedi.com/kirr/neo/go/zodb"
+
+func init() {
+	zblkTestDataRegistry["py2_pickle3"] = ZBlkTestData{
+		data_fs    : "testdata/py2_pickle3/zblk.fs",
+		zf_blksize : 2097152,
+		zf_size    : 8388608,
+		z0_oid     : zodb.Oid(2),
+		z1_oid     : zodb.Oid(3),
+		zf_oid     : zodb.Oid(1),
+		z0_rev     : zodb.Tid(0x285cbac3851eb99),
+		z1_rev     : zodb.Tid(0x285cbac3851eb99),
+		z0_len     : 16384,
+		z1_htlen   : 131072,
+	}
+}

wcfs/internal/zdata/ztestdata_zblk_py3_pickle3_test.go

+// Code generated by testdata/zblk_test_gen.py; DO NOT EDIT.
+package zdata
+
+import "lab.nexedi.com/kirr/neo/go/zodb"
+
+func init() {
+	zblkTestDataRegistry["py3_pickle3"] = ZBlkTestData{
+		data_fs    : "testdata/py3_pickle3/zblk.fs",
+		zf_blksize : 2097152,
+		zf_size    : 8388608,
+		z0_oid     : zodb.Oid(2),
+		z1_oid     : zodb.Oid(3),
+		zf_oid     : zodb.Oid(1),
+		z0_rev     : zodb.Tid(0x285cbac3851eb99),
+		z1_rev     : zodb.Tid(0x285cbac3851eb99),
+		z0_len     : 16384,
+		z1_htlen   : 131072,
+	}
+}

wcfs/internal/zdata/ztestdata_zblk_test.go

-// Code generated by ./testdata/zblk_test_gen.py; DO NOT EDIT.
-package zdata
-
-import "lab.nexedi.com/kirr/neo/go/zodb"
-
-const zf_blksize = 2097152
-const zf_size    = 8388608
-const z0_oid     = zodb.Oid(2)
-const z1_oid     = zodb.Oid(3)
-const zf_oid     = zodb.Oid(1)
-const z0_rev     = zodb.Tid(0x285cbac3851eb99)
-const z1_rev     = zodb.Tid(0x285cbac3851eb99)
-const z0_len     = 16384
-const z1_htlen   = 131072