Commit b87edcfe authored by Kirill Smelkov's avatar Kirill Smelkov

wcfs: tests: Tree-based testing environment

Add treeenv.go that combines Treegen and client side access to ZODB with
committed trees as extension to testing.T . The environment allows to
easily see which tree update was committed, what is the difference in
terms of KV, what is the state of updated tree and state of pointed-to
ZBlk objects.

This will be used to test upcoming ΔBtail and ΔFtail.

Main functionality is in treeenv.go; the other added files are to
support that.

Some preliminary history:

kirr/wendelin.core@f07502fc    X xbtreetest: Teach T & Commit to automatically provide At in symbolic form
kirr/wendelin.core@0d62b05e    X Adjust to btree.VGet & friends signature change to include keycov in visit callback
kirr/wendelin.core@588a512a    X zdata: Switch SliceByFileRev not to clone Zinblk
kirr/wendelin.core@e9c4b619    X rebuild: tests: Random testing
kirr/wendelin.core@43090ac7    X tests: Factor-out tree-test-env into tTreeEnv
kirr/wendelin.core@d4a523b2    X δbtail: tests: Run much faster with live ZODB cache
kirr/wendelin.core@271d953d    X rebuild: tests: Move ΔBtail.Clone test out of hot inner loop into separate test
kirr/wendelin.core@c32055fc    X wcfs/xbtree: ΔBtail tests += ø -> Tree; Tree -> ø
kirr/wendelin.core@5324547c    X wcfs/xbtree: root(a) must stay in trackSet even after treediff(a,ø)
kirr/wendelin.core@8f6e2b1e    X rebuild: tests: Don't access ZODB in XGetδKV
parent b13ee09b
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest/init (ex imported from package A_test) should be imported
// in addition to xbtreetest (ex imported from package A) to initialize
// xbtreetest at runtime.
package init
// ZBlk-related part of xbtreetest
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/xbtreetest"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata"
)
type Tree = xbtreetest.Tree
type Node = xbtreetest.Node
type Key = xbtreetest.Key
type KeyRange = xbtreetest.KeyRange
type ZBlk = zdata.ZBlk
func init() {
xbtreetest.ZGetBlkData = _ZGetBlkData
}
// _ZGetBlkData loads block data from ZBlk object specified by its oid.
func _ZGetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) (data string, err error) {
defer xerr.Contextf(&err, "@%s: get blkdata from obj %s", zconn.At(), zblkOid)
xblk, err := zconn.Get(ctx, zblkOid)
if err != nil {
return "", err
}
zblk, ok := xblk.(ZBlk)
if !ok {
return "", fmt.Errorf("expect ZBlk*; got %s", xzodb.TypeOf(xblk))
}
bdata, _, err := zblk.LoadBlkData(ctx)
if err != nil {
return "", err
}
return string(bdata), nil
}
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
// See https://www.nexedi.com/licensing for rationale and options. // See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest package xbtreetest
// kvdiff + friends
import ( import (
"fmt" "fmt"
...@@ -25,6 +26,31 @@ import ( ...@@ -25,6 +26,31 @@ import (
"strings" "strings"
) )
// KVDiff returns difference in between kv1 and kv2.
const DEL = "ø" // DEL means deletion
type Δstring struct {
Old string
New string
}
func KVDiff(kv1, kv2 map[Key]string) map[Key]Δstring {
delta := map[Key]Δstring{}
keys := setKey{}
for k := range kv1 { keys.Add(k) }
for k := range kv2 { keys.Add(k) }
for k := range keys {
v1, ok := kv1[k]
if !ok { v1 = DEL }
v2, ok := kv2[k]
if !ok { v2 = DEL }
if v1 != v2 {
delta[k] = Δstring{v1,v2}
}
}
return delta
}
// KVTxt returns string representation of {} kv. // KVTxt returns string representation of {} kv.
func KVTxt(kv map[Key]string) string { func KVTxt(kv map[Key]string) string {
if len(kv) == 0 { if len(kv) == 0 {
......
...@@ -20,9 +20,20 @@ ...@@ -20,9 +20,20 @@
package xbtreetest package xbtreetest
import ( import (
"reflect"
"testing" "testing"
) )
func TestKVDiff(t *testing.T) {
kv1 := map[Key]string{1:"a", 3:"c", 4:"d"}
kv2 := map[Key]string{1:"b", 4:"d", 5:"e"}
got := KVDiff(kv1, kv2)
want := map[Key]Δstring{1:{"a","b"}, 3:{"c",DEL}, 5:{DEL,"e"}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("error:\ngot: %v\nwant: %v", got, want)
}
}
func TestKVTxt(t *testing.T) { func TestKVTxt(t *testing.T) {
kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"} kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
got := KVTxt(kv) got := KVTxt(kv)
......
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// RBucket + RBucketSet
import (
"fmt"
"sort"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// RBucketSet represents set of buckets covering whole [-∞,∞) range.
type RBucketSet []*RBucket // k↑
// RBucket represents Bucket node with values covering [lo, hi_] key range in its Tree.
// NOTE it is not [lo,hi) but [lo,hi_] instead to avoid overflow at KeyMax.
type RBucket struct {
Oid zodb.Oid
Parent *RTree
Keycov blib.KeyRange
KV map[Key]string // bucket's k->v; values were ZBlk objects whose data is loaded instead.
}
// RTree represents Tree node that RBucket refers to as parent.
type RTree struct {
Oid zodb.Oid
Parent *RTree
}
// Path returns path to this bucket from tree root.
func (rb *RBucket) Path() []zodb.Oid {
path := []zodb.Oid{rb.Oid}
p := rb.Parent
for p != nil {
path = append([]zodb.Oid{p.Oid}, path...)
p = p.Parent
}
return path
}
// Get returns RBucket which covers key k.
func (rbs RBucketSet) Get(k Key) *RBucket {
i := sort.Search(len(rbs), func(i int) bool {
return k <= rbs[i].Keycov.Hi_
})
if i == len(rbs) {
panicf("BUG: key %v not covered; coverage: %s", k, rbs.coverage())
}
rb := rbs[i]
if !rb.Keycov.Has(k) {
panicf("BUG: get(%v) -> %s; coverage: %s", k, rb.Keycov, rbs.coverage())
}
return rb
}
// coverage returns string representation of rbs coverage structure.
func (rbs RBucketSet) coverage() string {
if len(rbs) == 0 {
return "ø"
}
s := ""
for _, rb := range rbs {
if s != "" {
s += " "
}
s += fmt.Sprintf("%s", rb.Keycov)
}
return s
}
// Flatten converts xkv with bucket structure into regular dict.
func (xkv RBucketSet) Flatten() map[Key]string {
kv := make(map[Key]string)
for _, b := range xkv {
for k,v := range b.KV {
kv[k] = v
}
}
return kv
}
func (b *RBucket) String() string {
return fmt.Sprintf("%sB%s{%s}", b.Keycov, b.Oid, KVTxt(b.KV))
}
This diff is collapsed.
...@@ -23,6 +23,9 @@ package xbtreetest ...@@ -23,6 +23,9 @@ package xbtreetest
import ( import (
"fmt" "fmt"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/set"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib" "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
) )
...@@ -38,6 +41,10 @@ type KeyRange = blib.KeyRange ...@@ -38,6 +41,10 @@ type KeyRange = blib.KeyRange
const KeyMax = blib.KeyMax const KeyMax = blib.KeyMax
const KeyMin = blib.KeyMin const KeyMin = blib.KeyMin
type setKey = set.I64
const VDEL = zodb.InvalidOid
func panicf(format string, argv ...interface{}) { func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...)) panic(fmt.Sprintf(format, argv...))
......
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// access to ZBlk data
import (
"context"
"lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/neo/go/zodb"
_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
)
// ZBlk-related functions are imported at runtime by package xbtreetest/init
var (
ZGetBlkData func(context.Context, *zodb.Connection, zodb.Oid) (string, error)
)
func zassertInitDone() {
if ZGetBlkData == nil {
panic("xbtreetest/zdata not initialized -> import xbtreetest/init to fix")
}
}
// xzgetBlkData loads block data from ZBlk object specified by its oid.
func xzgetBlkData(ctx context.Context, zconn *zodb.Connection, zblkOid zodb.Oid) string {
zassertInitDone()
X := exc.Raiseif
if zblkOid == VDEL {
return DEL
}
data, err := ZGetBlkData(ctx, zconn, zblkOid); X(err)
return string(data)
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment