wcfs: tests: Tree-based testing environment

Add treeenv.go that combines Treegen and client side access to ZODB with
committed trees as extension to testing.T . The environment allows to
easily see which tree update was committed, what is the difference in
terms of KV, what is the state of updated tree and state of pointed-to
ZBlk objects.

This will be used to test upcoming ΔBtail and ΔFtail.

Main functionality is in treeenv.go; the other added files are to
support that.

Some preliminary history:

f07502fc    X xbtreetest: Teach T & Commit to automatically provide At in symbolic form
0d62b05e    X Adjust to btree.VGet & friends signature change to include keycov in visit callback
588a512a    X zdata: Switch SliceByFileRev not to clone Zinblk
e9c4b619    X rebuild: tests: Random testing
43090ac7    X tests: Factor-out tree-test-env into tTreeEnv
d4a523b2    X δbtail: tests: Run much faster with live ZODB cache
271d953d    X rebuild: tests: Move ΔBtail.Clone test out of hot inner loop into separate test
c32055fc    X wcfs/xbtree: ΔBtail tests += ø -> Tree; Tree -> ø
5324547c    X wcfs/xbtree: root(a) must stay in trackSet even after treediff(a,ø)
8f6e2b1e    X rebuild: tests: Don't access ZODB in XGetδKV

wcfs/internal/xbtree/xbtreetest/init/init.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package xbtreetest/init (ex imported from package A_test) should be imported
+// in addition to xbtreetest (ex imported from package A) to initialize
+// xbtreetest at runtime.
+package init
+
+// ZBlk-related part of xbtreetest
+
+import (
+	"context"
+	"fmt"
+
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
+)
+
+type Tree     = xbtreetest.Tree
+type Node     = xbtreetest.Node
+type Key      = xbtreetest.Key
+type KeyRange = xbtreetest.KeyRange
+
+type ZBlk = zdata.ZBlk
+
+
+func init() {
+	xbtreetest.ZGetBlkData = _ZGetBlkData
+}
+
+
+// _ZGetBlkData loads block data from ZBlk object specified by its oid.
+func _ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
+	defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
+
+	xblk, err := zconn.Get(ctx, zblkOid)
+	if err != nil {
+		return "", err
+	}
+	zblk, ok := xblk.(ZBlk)
+	if !ok {
+		return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
+	}
+	bdata, _, err := zblk.LoadBlkData(ctx)
+	if err != nil {
+		return "", err
+	}
+	return string(bdata), nil
+}

wcfs/internal/xbtree/xbtreetest/kvdiff.go

@@ -18,6 +18,7 @@
 // See https://www.nexedi.com/licensing for rationale and options.
 
 package xbtreetest
+// kvdiff + friends
 
 import (
 	"fmt"
@@ -25,6 +26,31 @@ import (
 	"strings"
 )
 
+// KVDiff returns difference in between kv1 and kv2.
+const DEL = "ø" // DEL means deletion
+type Δstring struct {
+	Old string
+	New string
+}
+func KVDiff(kv1, kv2 map[Key]string) map[Key]Δstring {
+	delta := map[Key]Δstring{}
+	keys := setKey{}
+	for k := range kv1 { keys.Add(k) }
+	for k := range kv2 { keys.Add(k) }
+
+	for k := range keys {
+		v1, ok := kv1[k]
+		if !ok { v1 = DEL }
+		v2, ok := kv2[k]
+		if !ok { v2 = DEL }
+		if v1 != v2 {
+			delta[k] = Δstring{v1,v2}
+		}
+	}
+
+	return delta
+}
+
 // KVTxt returns string representation of {} kv.
 func KVTxt(kv map[Key]string) string {
 	if len(kv) == 0 {

wcfs/internal/xbtree/xbtreetest/kvdiff_test.go

@@ -20,9 +20,20 @@
 package xbtreetest
 
 import (
+	"reflect"
 	"testing"
 )
 
+func TestKVDiff(t *testing.T) {
+	kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
+	kv2 := map[Key]string{1:"b",        4:"d", 5:"e"}
+	got  := KVDiff(kv1, kv2)
+	want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("error:\ngot:  %v\nwant: %v", got, want)
+	}
+}
+
 func TestKVTxt(t *testing.T) {
 	kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
 	got  := KVTxt(kv)

wcfs/internal/xbtree/xbtreetest/rtree.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// RBucket + RBucketSet
+
+import (
+	"fmt"
+	"sort"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
+)
+
+// RBucketSet represents set of buckets covering whole [-∞,∞) range.
+type RBucketSet []*RBucket // k↑
+
+// RBucket represents Bucket node with values covering [lo, hi_] key range in its Tree.
+// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
+type RBucket struct {
+	Oid     zodb.Oid
+	Parent  *RTree
+	Keycov  blib.KeyRange
+	KV      map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
+}
+
+// RTree represents Tree node that RBucket refers to as parent.
+type RTree struct {
+	Oid    zodb.Oid
+	Parent *RTree
+}
+
+
+// Path returns path to this bucket from tree root.
+func (rb *RBucket) Path() []zodb.Oid {
+	path := []zodb.Oid{rb.Oid}
+	p := rb.Parent
+	for p != nil {
+		path = append([]zodb.Oid{p.Oid}, path...)
+		p = p.Parent
+	}
+	return path
+}
+
+// Get returns RBucket which covers key k.
+func (rbs RBucketSet) Get(k Key) *RBucket {
+	i := sort.Search(len(rbs), func(i int) bool {
+		return k <= rbs[i].Keycov.Hi_
+	})
+	if i == len(rbs) {
+		panicf("BUG: key %v not covered;  coverage: %s", k, rbs.coverage())
+	}
+
+	rb := rbs[i]
+	if !rb.Keycov.Has(k) {
+		panicf("BUG: get(%v) -> %s;  coverage: %s", k, rb.Keycov, rbs.coverage())
+	}
+
+	return rb
+}
+
+// coverage returns string representation of rbs coverage structure.
+func (rbs RBucketSet) coverage() string {
+	if len(rbs) == 0 {
+		return "ø"
+	}
+	s := ""
+	for _, rb := range rbs {
+		if s != "" {
+			s += " "
+		}
+		s += fmt.Sprintf("%s", rb.Keycov)
+	}
+	return s
+}
+
+// Flatten converts xkv with bucket structure into regular dict.
+func (xkv RBucketSet) Flatten() map[Key]string {
+	kv := make(map[Key]string)
+	for _, b := range xkv {
+		for k,v := range b.KV {
+			kv[k] = v
+		}
+	}
+	return kv
+}
+
+func (b *RBucket) String() string {
+	return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, KVTxt(b.KV))
+}

wcfs/internal/xbtree/xbtreetest/treeenv.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// T + friends
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"testing"
+
+	"lab.nexedi.com/kirr/go123/exc"
+	"lab.nexedi.com/kirr/neo/go/transaction"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
+)
+
+// T is tree-based testing environment.
+//
+// It combines TreeSrv and client side access to ZODB with committed trees.
+// It should be created via NewT().
+type T struct {
+	*testing.T
+
+	work    string   // working directory
+	treeSrv *TreeSrv
+	zstor   zodb.IStorage
+	DB      *zodb.DB
+
+	commitv []*Commit // all committed trees
+	at0idx  int       // index of current "at₀" in commitv
+}
+
+// Commit represent test commit changing a tree.
+type Commit struct {
+	T       *T                    // created via T.Commit
+	idx     int                   // lives in .T.commitv[idx]
+	Tree    string                // the tree in topology-encoding
+	Prev    *Commit               // previous commit
+	At      zodb.Tid              // committed revision
+	ΔZ      *zodb.EventCommit     // raw ZODB changes; δZ.tid == at
+	Xkv     RBucketSet            // full tree state as of @at
+	Δxkv    map[Key]Δstring       // full tree-diff against parent
+	ZBlkTab map[zodb.Oid]ZBlkInfo // full snapshot of all ZBlk name/data @at
+}
+
+// ZBlkInfo describes one ZBlk object.
+type ZBlkInfo struct {
+	Name string // this ZBlk comes under root['treegen/values'][Name]
+	Data string
+}
+
+// NewT creates new T.
+func NewT(t *testing.T) *T {
+	X := exc.Raiseif
+	t.Helper()
+
+	tt := &T{T: t, at0idx: 1 /* at₀ starts from first t.Commit */}
+
+	var err error
+	work := t.TempDir()
+	tt.treeSrv, err = StartTreeSrv(work + "/1.fs"); X(err)
+	t.Cleanup(func() {
+		err := tt.treeSrv.Close(); X(err)
+	})
+
+	tt.zstor, err = zodb.Open(context.Background(), tt.treeSrv.zurl, &zodb.OpenOptions{
+					 ReadOnly: true,
+		       }); X(err)
+	t.Cleanup(func() {
+		err := tt.zstor.Close(); X(err)
+	})
+
+	tt.DB = zodb.NewDB(tt.zstor, &zodb.DBOptions{
+		// We need objects to be cached, because otherwise it is too
+		// slow to run the test for many testcases, especially
+		// xverifyΔBTail_rebuild.
+		CacheControl: &tZODBCacheEverything{},
+	})
+	t.Cleanup(func() {
+		err := tt.DB.Close(); X(err)
+	})
+
+	head := tt.treeSrv.head
+	t1 := &Commit{
+		T:       tt,
+		idx:     0,
+		Tree:    "T/B:",	// treegen.py creates the tree as initially empty
+		Prev:    nil,
+		At:      head,
+		Xkv:     xGetTree(tt.DB, head, tt.Root()),
+		ZBlkTab: xGetBlkTab(tt.DB, head),
+		ΔZ:      nil,
+		Δxkv:    nil,
+	}
+	tt.commitv = []*Commit{t1}
+
+	return tt
+}
+
+// tZODBCacheEverything is workaround for ZODB/go not implementing real
+// live cache for now: Objects get dropped on PDeactivate if cache
+// control does not say we need the object to stay in the cache.
+type tZODBCacheEverything struct{}
+func (_ *tZODBCacheEverything) PCacheClassify(_ zodb.IPersistent) zodb.PCachePolicy {
+	return zodb.PCachePinObject | zodb.PCacheKeepState
+}
+
+// Root returns OID of root tree node.
+func (t *T) Root() zodb.Oid {
+	return t.treeSrv.treeRoot
+}
+
+// Head returns most-recently committed tree.
+func (t *T) Head() *Commit {
+	return t.commitv[len(t.commitv)-1]
+}
+
+// XGetCommit finds and returns Commit created with revision at.
+func (t *T) XGetCommit(at zodb.Tid) *Commit {
+	commit, _, _ := t.getCommit(at)
+	if commit == nil {
+		panicf("no commit corresponding to @%s", at)
+	}
+	return commit
+}
+
+func (t *T) getCommit(at zodb.Tid) (commit, cprev, cnext *Commit) {
+	l := len(t.commitv)
+	i := sort.Search(l, func(i int) bool {
+		return at <= t.commitv[i].At
+	})
+	if i < l {
+		commit = t.commitv[i]
+		if commit.At != at {
+			cnext = commit
+			commit = nil
+		} else if i+1 < l {
+			cnext = t.commitv[i+1]
+		}
+	}
+	if i > 0 {
+		cprev = t.commitv[i-1]
+	}
+	if commit != nil && commit.idx != i {
+		panicf("BUG: commit.idx (%d) != i (%d)", commit.idx, i)
+	}
+	return commit, cprev, cnext
+}
+
+// AtSymb returns symbolic representation of at, for example "at3".
+//
+// at should correspond to a Commit.
+func (t *T) AtSymb(at zodb.Tid) string {
+	commit, cprev, cnext := t.getCommit(at)
+	if commit != nil {
+		return commit.AtSymb()
+	}
+
+	// at does not correspond to commit - return something like ~at2<xxxx>at3
+	s := "~"
+	if cprev != nil {
+		s += cprev.AtSymb() + "<"
+	}
+	s += at.String()
+	if cnext != nil {
+		s += ">" + cnext.AtSymb()
+	}
+	return s
+}
+
+// AtSymb returns symbolic representation of c.At, for example "at3".
+func (c *Commit) AtSymb() string {
+	return fmt.Sprintf("at%d", c.idx - c.T.at0idx)
+}
+
+// AtSymbReset shifts symbolic numbers and adjust AtSymb setup so that c.AtSymb() returns "at<i>".
+func (t *T) AtSymbReset(c *Commit, i int) {
+	t.at0idx = c.idx - i
+}
+
+// Commit commits tree via treegen server and returns Commit object corresponding to committed transaction.
+func (t *T) Commit(tree string) *Commit {
+	X := exc.Raiseif
+	defer exc.Contextf("commit %s", tree)
+
+	watchq := make(chan zodb.Event)
+	at0 := t.zstor.AddWatch(watchq)
+	defer t.zstor.DelWatch(watchq)
+
+	tid, err := t.treeSrv.Commit(tree); X(err)
+
+	if !(tid > at0) {
+		exc.Raisef("treegen -> %s  ; want > %s", tid, at0)
+	}
+
+	zevent := <-watchq
+	δZ := zevent.(*zodb.EventCommit)
+	if δZ.Tid != tid {
+		exc.Raisef("treegen -> %s  ; watchq -> %s", tid, δZ)
+	}
+
+	// load tree structure from the db
+	// if the tree does not exist yet - report its structure as empty
+	var xkv RBucketSet
+	if tree != DEL {
+		xkv = xGetTree(t.DB, δZ.Tid, t.Root())
+	} else {
+		// empty tree with real treeRoot as oid even though the tree is
+		// deleted.  Having real oid in the root tests that after deletion,
+		// root of the tree stays in the tracking set. We need root to stay
+		// in trackSet because e.g. in
+		//
+		//	T1 -> ø -> T2
+		//
+		// where the tree is first deleted, then recreated, without root
+		// staying in trackSet after ->ø, treediff will notice nothing when
+		// it comes to ->T2.
+		xkv = RBucketSet{
+			&RBucket{
+				Oid:    zodb.InvalidOid,
+				Parent: &RTree{
+					Oid:    t.Root(), // NOTE oid is not InvalidOid
+					Parent: nil,
+				},
+				Keycov: blib.KeyRange{KeyMin, KeyMax},
+				KV:     map[Key]string{},
+			},
+		}
+	}
+
+	ttree := &Commit{
+		T:       t,
+		Tree:    tree,
+		At:      δZ.Tid,
+		ΔZ:      δZ,
+		Xkv:     xkv,
+		ZBlkTab: xGetBlkTab(t.DB, δZ.Tid),
+	}
+
+	tprev := t.Head()
+	ttree.Prev = tprev
+	ttree.Δxkv = KVDiff(tprev.Xkv.Flatten(), ttree.Xkv.Flatten())
+
+	ttree.idx = len(t.commitv)
+	t.commitv = append(t.commitv, ttree)
+
+	return ttree
+}
+
+// xGetBlkTab loads all ZBlk from db@at.
+//
+// it returns {} oid -> blkdata.
+func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
+	defer exc.Contextf("%s: @%s: get blktab", db.Storage().URL(), at)
+	X := exc.Raiseif
+
+	blkTab := map[zodb.Oid]ZBlkInfo{}
+
+	txn, ctx := transaction.New(context.Background())
+	defer txn.Abort()
+	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
+
+	xzroot, err := zconn.Get(ctx, 0); X(err)
+	zroot, ok := xzroot.(*zodb.Map)
+	if !ok {
+		exc.Raisef("root: expected %s, got %s", xzodb.TypeOf(zroot), xzodb.TypeOf(xzroot))
+	}
+
+	err = zroot.PActivate(ctx); X(err)
+	defer zroot.PDeactivate()
+
+	xzblkdir, ok := zroot.Data["treegen/values"]
+	if !ok {
+		exc.Raisef("root['treegen/values'] missing")
+	}
+	zblkdir, ok := xzblkdir.(*zodb.Map)
+	if !ok {
+		exc.Raisef("root['treegen/values']: expected %s, got %s", xzodb.TypeOf(zblkdir), xzodb.TypeOf(xzblkdir))
+	}
+
+	err = zblkdir.PActivate(ctx); X(err)
+	defer zblkdir.PDeactivate()
+
+	for xname, xzblk := range zblkdir.Data {
+		name, ok := xname.(string)
+		if !ok {
+			exc.Raisef("root['treegen/values']: key [%q]: expected str, got %T", xname, xname)
+		}
+
+		zblk, ok := xzblk.(zodb.IPersistent)
+		if !ok {
+			exc.Raisef("root['treegen/values'][%q]: expected IPersistent, got %s", name, xzodb.TypeOf(xzblk))
+		}
+
+		oid := zblk.POid()
+		data := xzgetBlkData(ctx, zconn, oid)
+		blkTab[oid] = ZBlkInfo{name, data}
+	}
+
+	return blkTab
+}
+
+// XGetBlkData loads blk data for ZBlk<oid> @c.at
+//
+// For speed the load is done via preloaded c.blkDataTab instead of access to the DB.
+func (c *Commit) XGetBlkData(oid zodb.Oid) string {
+	if oid == VDEL {
+		return DEL
+	}
+	zblki, ok := c.ZBlkTab[oid]
+	if !ok {
+		exc.Raisef("getBlkData ZBlk<%s> @%s: no such ZBlk", oid, c.At)
+	}
+	return zblki.Data
+}
+
+// XGetBlkByName returns ZBlk info associated with ZBlk<name>
+func (c *Commit) XGetBlkByName(name string) (zodb.Oid, ZBlkInfo) {
+	for oid, zblki := range c.ZBlkTab {
+		if zblki.Name == name {
+			return oid, zblki
+		}
+	}
+	panicf("ZBlk<%q> not found", name)
+	panic("")
+}
+
+// xGetTree loads Tree from zurl@at->obj<root>.
+//
+// Tree values must be ZBlk whose data is returned instead of references to ZBlk objects.
+// The tree is returned structured by buckets as
+//
+//	[] [lo,hi){k->v}  k↑
+func xGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
+	defer exc.Contextf("%s: @%s: get tree %s", db.Storage().URL(), at, root)
+	X := exc.Raiseif
+
+	txn, ctx := transaction.New(context.Background())
+	defer txn.Abort()
+	zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
+
+	xztree, err := zconn.Get(ctx, root); X(err)
+	ztree, ok := xztree.(*Tree)
+	if !ok {
+		exc.Raisef("expected %s, got %s", xzodb.TypeOf(ztree), xzodb.TypeOf(xztree))
+	}
+
+	rbucketv := RBucketSet{}
+	xwalkDFS(ctx, KeyMin, KeyMax, ztree, func(rb *RBucket) {
+		rbucketv = append(rbucketv, rb)
+	})
+	if len(rbucketv) == 0 { // empty tree -> [-∞,∞){}
+		etree   := &RTree{
+				Oid:    root,
+				Parent: nil,
+		}
+		ebucket := &RBucket{
+				Oid:    zodb.InvalidOid,
+				Parent: etree,
+				Keycov: blib.KeyRange{KeyMin, KeyMax},
+				KV:     map[Key]string{},
+		}
+		rbucketv = RBucketSet{ebucket}
+	}
+	return rbucketv
+}
+
+// xwalkDFS walks ztree in depth-first order emitting bvisit callback on visited bucket nodes.
+func xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, bvisit func(*RBucket)) {
+	_xwalkDFS(ctx, lo, hi_, ztree, /*rparent*/nil, bvisit)
+}
+func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bvisit func(*RBucket)) {
+	X := exc.Raiseif
+
+	err := ztree.PActivate(ctx); X(err)
+	defer ztree.PDeactivate()
+
+	rtree := &RTree{Oid: ztree.POid(), Parent: rparent}
+
+	// [i].Key ≤ [i].Child.*.Key < [i+1].Key  i ∈ [0, len([]))
+	//
+	// [0].Key       = -∞ ; always returned so
+	// [len(ev)].Key = +∞ ; should be assumed so
+	ev := ztree.Entryv()
+	for i := range ev {
+		xlo  := lo;  if i   > 0       { xlo  = ev[i].Key() }
+		xhi_ := hi_; if i+1 < len(ev) { xhi_ = ev[i+1].Key() - 1 }
+
+		tchild, ok := ev[i].Child().(*Tree)
+		if ok {
+			_xwalkDFS(ctx, xlo, xhi_, tchild, rtree, bvisit)
+			continue
+		}
+
+		zbucket := ev[i].Child().(*Bucket)
+		err = zbucket.PActivate(ctx); X(err)
+		defer zbucket.PDeactivate()
+
+		bkv := make(map[Key]string)
+		bentryv := zbucket.Entryv()
+
+		for _, __ := range bentryv {
+			k  := __.Key()
+			xv := __.Value()
+			pv, ok := xv.(zodb.IPersistent)
+			if !ok {
+				exc.Raisef("[%d] -> %s;  want IPersistent", k, xzodb.TypeOf(xv))
+			}
+			data, err := ZGetBlkData(ctx, pv.PJar(), pv.POid())
+			if err != nil {
+				exc.Raisef("[%d]: %s", k, err)
+			}
+			bkv[k] = data
+		}
+
+
+		b := &RBucket{
+			Oid:    zbucket.POid(),
+			Parent: rtree,
+			Keycov: blib.KeyRange{xlo, xhi_},
+			KV:     bkv,
+		}
+		bvisit(b)
+	}
+}

wcfs/internal/xbtree/xbtreetest/xbtreetest.go

@@ -23,6 +23,9 @@ package xbtreetest
 import (
 	"fmt"
 
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
 )
 
@@ -38,6 +41,10 @@ type KeyRange = blib.KeyRange
 const KeyMax  = blib.KeyMax
 const KeyMin  = blib.KeyMin
 
+type setKey = set.I64
+
+const VDEL  = zodb.InvalidOid
+
 
 func panicf(format string, argv ...interface{}) {
 	panic(fmt.Sprintf(format, argv...))

wcfs/internal/xbtree/xbtreetest/zdata.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xbtreetest
+// access to ZBlk data
+
+import (
+	"context"
+
+	"lab.nexedi.com/kirr/go123/exc"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
+)
+
+// ZBlk-related functions are imported at runtime by package xbtreetest/init
+var (
+	ZGetBlkData func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
+)
+
+func zassertInitDone() {
+	if ZGetBlkData == nil {
+		panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
+	}
+}
+
+// xzgetBlkData loads block data from ZBlk object specified by its oid.
+func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
+	zassertInitDone()
+	X := exc.Raiseif
+
+	if zblkOid == VDEL {
+		return DEL
+	}
+
+	data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
+	return string(data)
+}