wcfs: Add zdata package to load ZBlk/ZBigFile data

Add functionality to load objects from ZODB as saved by py wendelin.core.
Mostly straightforward code.
The main part is in zblk.go .

Contrary to python implementation, go can load ZBlk1's subobjects in
parallel, which, given scalable ZODB storage, can be significantly
faster compared to serially loading all ZData subobjects as py code
does.

TODO test wrt data saved by Python3.

Some preliminary history:

kirr/wendelin.core@878b2787 X draft loading
kirr/wendelin.core@bf9a7405 X No longer rely on ZODB cache invariant for invalidations
kirr/wendelin.core@0d62b05e X Adjust to btree.VGet & friends signature change to include keycov in visit callback
kirr/wendelin.core@b74dda09 X Start switching Track from Track(key) to Track(keycov)

wcfs/go.mod

 module lab.nexedi.com/nexedi/wendelin.core/wcfs
 
 go 1.14
+
+require (
+	github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c
+	github.com/kisielk/og-rek v1.1.1-0.20210310094122-8def3d024dac
+	github.com/stretchr/testify v1.7.0
+	lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408
+	lab.nexedi.com/kirr/neo/go v0.0.0-20211004111643-c74a5a3cd0d0
+)

wcfs/internal/pycompat/pycompat.go

+// XXX dup from neo/go/zodb/internal/pickletools
+
+package pycompat
+
+import (
+	"math/big"
+)
+
+// Int64 tries to convert unpickled Python value to int64.
+//
+// (ogórek decodes python long as big.Int)
+//
+// XXX + support for float?
+func Int64(xv interface{}) (v int64, ok bool) {
+        switch v := xv.(type) {
+        case int64:
+                return v, true
+        case *big.Int:
+                if v.IsInt64() {
+                        return v.Int64(), true
+                }
+        }
+
+        return 0, false
+}

wcfs/internal/xzodb/xzodb.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xzodb compements package zodb.
+package xzodb
+
+import (
+	"fmt"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+
+// TypeOf returns string for object's type.
+//
+//	- for ZODB objects, it uses zodb.ClassOf, which in particular supports
+//	  printing zodb.Broken with details properly.
+//
+//	- for other objects, it uses %T.
+func TypeOf(obj interface{}) string {
+	switch obj := obj.(type) {
+	case zodb.IPersistent:
+		return zodb.ClassOf(obj)
+
+	default:
+		return fmt.Sprintf("%T", obj)
+	}
+}

wcfs/internal/zdata/misc.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zdata
+
+import (
+	"fmt"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+// tidmax returns latest revision.
+func tidmax(a, b zodb.Tid) zodb.Tid {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// tidmin returns earliest revision.
+func tidmin(a, b zodb.Tid) zodb.Tid {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func panicf(format string, argv ...interface{}) {
+	panic(fmt.Sprintf(format, argv...))
+}

wcfs/internal/zdata/testdata/.gitignore

+/*.lock
+/*.tmp
+/*.tr[0-9]

wcfs/internal/zdata/testdata/zblk_test_gen.py

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright (C) 2018-2021  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""zblk_test_gen.py generates test data for zblk_test.go"""
+
+from ZODB.DB import DB
+from ZODB.utils import u64
+from wendelin.bigfile.file_zodb import ZBlk0, ZBlk1, ZBigFile
+from BTrees.IOBTree import IOBTree, IOBucket
+from numpy import arange
+import os, os.path, transaction
+
+import zodbtools.test.gen_testdata  # to make time predictable
+from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
+
+K = 1024
+
+def main():
+    run_with_zodb4py2_compat(main2)
+
+def main2():
+    outfs = "testdata/zblk.fs"
+    rm_f(outfs)
+    rm_f(outfs + ".index")
+    db = DB(outfs)
+    conn = db.open()
+    root = conn.root()
+
+    # one ZBigFile with two data blocks
+    root['zbigf'] = zf = ZBigFile(2*1024*K)
+    root['zblk0'] = z0 = ZBlk0()
+    root['zblk1'] = z1 = ZBlk1()    # also covers ZData
+
+    # zblk0 with small data
+    z0.setblkdata(brange32(16*K))
+    # zblk1 with head + large hole + tail.
+    # head and tail are 128K because if smaller, z1.chunktab would serialize without buckets.
+    z1ht = 128*K
+    z1.setblkdata(brange32(z1ht) + bzeros(zf.blksize - 2*z1ht) + breverse(brange32(z1ht)))
+    zf.blktab[1] = z0
+    zf.blktab[3] = z1
+
+    # make sure there is at least one bucket in z1's chunktab
+    #
+    # we need at least one bucket to verify how zblk.go handles chunktab bucket loading;
+    # more than 1 bucket just wastes space, however with only 1 bucket a tree
+    # is serialized to have leaf nodes directly in the tree state.
+    # -> verify for ==2 buckets.
+    assertIOBTreeHas2Buckets(z1.chunktab)
+
+    transaction.commit()
+
+    with open("ztestdata_zblk_test.go", "w") as f:
+        def emit(v):
+            print >>f, v
+        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+        emit("package zdata\n")
+        emit('import "lab.nexedi.com/kirr/neo/go/zodb"\n')
+
+        emit("const zf_blksize = %d" % zf.blksize)
+        emit("const zf_size    = %d" % ((zf.blktab.maxKey()+1)*zf.blksize))
+        emit("const z0_oid     = zodb.Oid(%d)" % u64(z0._p_oid))
+        emit("const z1_oid     = zodb.Oid(%d)" % u64(z1._p_oid))
+        emit("const zf_oid     = zodb.Oid(%d)" % u64(zf._p_oid))
+        emit("const z0_rev     = zodb.Tid(0x%x)"  % u64(z0._p_serial))
+        emit("const z1_rev     = zodb.Tid(0x%x)"  % u64(z1._p_serial))
+        emit("const z0_len     = %d" % len(z0.loadblkdata()))
+        emit("const z1_htlen   = %d" % z1ht)
+
+
+    conn.close()
+    db.close()
+
+
+# brange32 returns bytes with big-endian uint32 sequence filling them.
+# returned bytes has len == size.
+def brange32(size):
+    # 0, 1, 2, ... as u32
+    return arange(0, size//4, dtype='>u4').tobytes()
+
+# bzeros returns bytes of requested size with 0 filling them.
+def bzeros(size):
+    return b'\0'*size
+
+# breverse returns bytes in the reverse order.
+def breverse(b):
+    assert isinstance(b, bytes)
+    _ = bytearray(b)
+    _.reverse()
+    return bytes(_)
+
+# assertIOBTreeHas2Buckets asserts that IOBTree has 2 buckets.
+def assertIOBTreeHas2Buckets(t):
+    assert isinstance(t, IOBTree)
+
+    # https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/BTreeTemplate.c#L1087-L1109
+    _ = t.__getstate__()
+    assert len(_) == 2, (len(_), _)
+    assert len(_[0]) == 3, _[0]                   # (bucket0, key1, bucket1)
+    assert isinstance(_[0][0], IOBucket), _[0][0] # bucket0
+    assert isinstance(_[0][1], int),      _[0][2] # key1
+    assert isinstance(_[0][2], IOBucket), _[0][2] # bucket1
+    assert isinstance(_[1],    IOBucket), _[1]    # .firstbucket
+
+# rm_f is like `rm -f` in shell.
+def rm_f(path):
+    if os.path.exists(path):
+        os.remove(path)
+
+
+if __name__ == '__main__':
+    main()

wcfs/internal/zdata/zblk_test.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package zdata
+
+//go:generate ./testdata/zblk_test_gen.py
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"testing"
+
+	"lab.nexedi.com/kirr/go123/exc"
+	"lab.nexedi.com/kirr/neo/go/transaction"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go.
+// TODO also test with data saved by Python3.
+func TestZBlk(t *testing.T) {
+	X := exc.Raiseif
+	assert := require.New(t)
+	ctx := context.Background()
+	stor, err := zodb.Open(ctx, "testdata/zblk.fs", &zodb.OpenOptions{ReadOnly: true}); X(err)
+	db := zodb.NewDB(stor, &zodb.DBOptions{})
+	defer func() {
+		err := db.Close();	X(err)
+		err  = stor.Close();	X(err)
+	}()
+
+	txn, ctx := transaction.New(ctx)
+	defer txn.Abort()
+
+	conn, err := db.Open(ctx, &zodb.ConnOptions{});	X(err)
+
+	xz0, err := conn.Get(ctx, z0_oid);	X(err)
+	xz1, err := conn.Get(ctx, z1_oid);	X(err)
+	xzf, err := conn.Get(ctx, zf_oid);	X(err)
+
+	z0, ok := xz0.(*ZBlk0)
+	if !ok {
+		t.Fatalf("z0: want ZBlk0; got %T", xz0)
+	}
+
+	z1, ok := xz1.(*ZBlk1)
+	if !ok {
+		t.Fatalf("z1: want ZBlk1; got %T", xz1)
+	}
+
+	zf, ok := xzf.(*ZBigFile)
+	if !ok {
+		t.Fatalf("zf: want ZBigFile; got %T", xzf)
+	}
+
+	xactivate := func(obj zodb.IPersistent) {
+		t.Helper()
+		err := obj.PActivate(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+
+	z0Data, z0Rev, err := z0.LoadBlkData(ctx);	X(err)
+	z0DataOK := brange32(z0_len)
+	assert.Equal(z0Data, z0DataOK,   "ZBlk0 data wrong")
+	assert.Equal(z0Rev,  z0_rev,	 "ZBlk0 rev wrong")
+
+	z1Data, z1Rev, err := z1.LoadBlkData(ctx);	X(err)
+	z1DataOK := make([]byte, zf_blksize)                                  // zeros
+	copy(z1DataOK[0:],                      brange32(z1_htlen))           // head
+	copy(z1DataOK[len(z1DataOK)-z1_htlen:], breverse(brange32(z1_htlen))) // tail
+	z1DataOK = bytes.TrimRight(z1DataOK, "\x00") // trailing 0 are not persisted
+	assert.Equal(z1Data, z1DataOK,   "ZBlk1 data wrong")
+	assert.Equal(z1Rev,  z1_rev,     "ZBlk1 rev wrong")
+
+
+	xactivate(zf)
+	if zf.blksize != zf_blksize {
+		t.Fatalf("zf: blksize=%d;  want %d", zf.blksize, zf_blksize)
+	}
+
+	z0_, ok, err := zf.blktab.Get(ctx, 1);	X(err)
+	if !(ok && z0_ == z0) {
+		t.Fatalf("zf: [0] -> %#v; want z0", z0_)
+	}
+
+	z1_, ok, err := zf.blktab.Get(ctx, 3);	X(err)
+	if !(ok && z1_ == z1) {
+		t.Fatalf("zf: [1] -> %#v; want z1", z1_)
+	}
+
+	size, _, _, err := zf.Size(ctx);	X(err)
+	assert.Equal(size, int64(zf_size),	"ZBigFile size wrong")
+
+
+	// LoadBlk
+	z0Data, _, _, _, _, err = zf.LoadBlk(ctx, 1);	X(err)
+	assert.Equal(len(z0Data), int(zf.blksize))
+	z0Data = bytes.TrimRight(z0Data, "\x00")
+	assert.Equal(z0Data, z0DataOK)
+
+	z1Data, _, _, _, _, err = zf.LoadBlk(ctx, 3);	X(err)
+	assert.Equal(len(z1Data), int(zf.blksize))
+	z1Data = bytes.TrimRight(z1Data, "\x00")
+	assert.Equal(z1Data, z1DataOK)
+}
+
+// TODO verify PyGetState vs PySetState
+
+// brange32 returns bytes with big-endian uint32 sequence filling them.
+// returned bytes has len == size.
+func brange32(size int) []byte {
+	data := make([]byte, size)
+	for i := 0; i < size / 4; i++ {
+		binary.BigEndian.PutUint32(data[i*4:], uint32(i))
+	}
+	return data
+}
+
+// breverse returns bytes in the reverse order.
+func breverse(b []byte) []byte {
+	r := make([]byte, len(b))
+	for i := range(b) {
+		r[i] = b[len(b)-i-1]
+	}
+	return r
+}

wcfs/internal/zdata/ztestdata_zblk_test.go

+// Code generated by ./testdata/zblk_test_gen.py; DO NOT EDIT.
+package zdata
+
+import "lab.nexedi.com/kirr/neo/go/zodb"
+
+const zf_blksize = 2097152
+const zf_size    = 8388608
+const z0_oid     = zodb.Oid(2)
+const z1_oid     = zodb.Oid(3)
+const zf_oid     = zodb.Oid(1)
+const z0_rev     = zodb.Tid(0x285cbac3851eb99)
+const z1_rev     = zodb.Tid(0x285cbac3851eb99)
+const z0_len     = 16384
+const z1_htlen   = 131072