Commit a8595565 authored by Kirill Smelkov's avatar Kirill Smelkov

wcfs: tests: Treegen functionality

treegen.go and treegen.py together provide a way

- to commit a particular BTree topology into ZODB, and
- to generate set of random tree topologies that all correspond to particular {k->v} dict.

this will be used in upcoming ΔBtail and ΔFtail tests.

See treegen.py documentation for details.

Some preliminary history:

9eca74ec    X Teach AllStructs to emit topologies with values
1b962f03    X Restructure: found bug that it was not marking objects as modified
2139af2c    X treegen: Verify that tree actually saved to storage is what was requested
b5e39d4a    X wcfs/treegen: allstructs: Do not keep all tree structures in memory
e9c4b619    X rebuild: tests: Random testing
c32055fc    X wcfs/xbtree: ΔBtail tests += ø -> Tree; Tree -> ø
4300d88a    X wcfs/xbtreetest/treegen.py: Fix it on ZODB4
parent 37fb6d28
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
import (
"fmt"
"sort"
"strings"
)
// KVTxt returns string representation of {} kv.
func KVTxt(kv map[Key]string) string {
if len(kv) == 0 {
return "ø"
}
keyv := []Key{}
for k := range kv { keyv = append(keyv, k) }
sort.Slice(keyv, func(i,j int) bool { return keyv[i] < keyv[j] })
sv := []string{}
for _, k := range keyv {
v := kv[k]
if strings.ContainsAny(v, " \n\t,:") {
panicf("[%v]=%q: invalid value", k, v)
}
sv = append(sv, fmt.Sprintf("%v:%s", k, v))
}
return strings.Join(sv, ",")
}
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
import (
"testing"
)
func TestKVTxt(t *testing.T) {
kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
got := KVTxt(kv)
want := "1:zzz,3:hello,4:world"
if got != want {
t.Fatalf("error:\ngot: %q\nwant: %q", got, want)
}
}
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// treegen.go provides functionality:
//
// - to commit a particular BTree topology into ZODB, and
// - to generate set of random tree topologies that all correspond to particular {k->v} dict.
//
// treegen.py is used as helper for both tasks.
import (
"bufio"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"lab.nexedi.com/kirr/go123/my"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// TreeGenSrv represents connection to running `treegen ...` server.
type TreeGenSrv struct {
argv []string
pysrv *exec.Cmd // spawned `treegen ...`
pyin io.WriteCloser // input to pysrv
pyoutRaw io.ReadCloser // output from pysrv
pyout *bufio.Reader // buffered ^^^
}
// TreeSrv represents connection to running `treegen trees` server.
//
// Create it with StartTreeSrv(zurl).
// - Commit(treeTopology) -> tid
type TreeSrv struct {
*TreeGenSrv
zurl string
treeRoot zodb.Oid // oid of the tree treegen works on
head zodb.Tid // last made commit
}
// AllStructsSrv represents connection to running `treegen allstructs` server.
//
// Create it with StartAllStructsSrv().
// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
type AllStructsSrv struct {
*TreeGenSrv
}
// startTreeGenSrv spawns `treegen ...` server.
func startTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
defer xerr.Contextf(&err, "treesrv %v: start", argv)
// spawn `treegen ...`
tg := &TreeGenSrv{argv: argv}
tg.pysrv = exec.Command(filepath.Dir(my.File())+"/treegen.py", argv...)
tg.pyin, err = tg.pysrv.StdinPipe()
if err != nil {
return nil, "", err
}
tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
if err != nil {
return nil, "", err
}
tg.pyout = bufio.NewReader(tg.pyoutRaw)
tg.pysrv.Stderr = os.Stderr // no redirection
err = tg.pysrv.Start()
if err != nil {
return nil, "", err
}
// wait for hello message and return it
defer func() {
if err != nil {
tg.Close() // ignore error
}
}()
defer xerr.Context(&err, "handshake")
hello, err = tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, "", err
}
hello = strings.TrimSuffix(hello, "\n")
return tg, hello, nil
}
// Close shutdowns treegen server.
func (tg *TreeGenSrv) Close() (err error) {
defer xerr.Contextf(&err, "treegen %v: close", tg.argv)
err1 := tg.pyin.Close()
err2 := tg.pyoutRaw.Close()
err3 := tg.pysrv.Wait()
return xerr.Merge(err1, err2, err3)
}
// StartTreeSrv spawns `treegen trees` server.
func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
tgSrv, hello, err := startTreeGenSrv("trees", zurl)
if err != nil {
return nil, err
}
tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
// tree.srv start @<at> tree=<root>
defer xerr.Contextf(&err, "invalid hello %q", hello)
startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
m := startRe.FindStringSubmatch(hello)
if m == nil {
return nil, fmt.Errorf("unexpected format")
}
tg.head, err = zodb.ParseTid(m[1]) // <at>
if err != nil {
return nil, fmt.Errorf("tid: %s", err)
}
tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
if err != nil {
return nil, fmt.Errorf("root: %s", err)
}
return tg, nil
}
// StartAllStructsSrv spawns `treegen allstructs` server.
func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
defer xerr.Context(&err, "allstructs.srv: start")
tgSrv, hello, err := startTreeGenSrv("allstructs")
if err != nil {
return nil, err
}
sg := &AllStructsSrv{TreeGenSrv: tgSrv}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
defer xerr.Contextf(&err, "invalid hello %q", hello)
if hello != "# allstructs.srv start" {
return nil, fmt.Errorf("unexpected format")
}
return sg, nil
}
// Commit creates new commit with underlying tree changed to specified tree topology.
//
// The following topologies are treated specially:
//
// - "ø" requests deletion of the tree, and
// - "øf" requests deletion of ZBigFile, that is referencing the tree.
// - "t<kv> D<data-kv>" changes both tree natively, and ZBlk data objects.
//
// see treegen.py for details.
func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)
_, err = io.WriteString(tg.pyin, tree + "\n")
if err != nil {
return zodb.InvalidTid, err
}
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return zodb.InvalidTid, err
}
reply = strings.TrimSuffix(reply, "\n")
tid, err := zodb.ParseTid(reply)
if err != nil {
return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
}
tg.head = tid
return tid, nil
}
// AllStructs returns response from `treegen allstructs`
func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, KVTxt(kv))
defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
_, err = io.WriteString(tg.pyin, req + "\n")
if err != nil {
return nil, err
}
structv := []string{}
for {
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
reply = strings.TrimSuffix(reply, "\n")
if reply == "# ----" {
return structv, nil // end of response
}
if strings.HasPrefix(reply, "#") {
continue // comment
}
structv = append(structv, reply)
}
}
This diff is collapsed.
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest provides infrastructure for testing LOBTree with ZBlk values.
package xbtreetest
import (
"fmt"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// XXX instead of generics
type Tree = blib.Tree
type Bucket = blib.Bucket
type Node = blib.Node
type TreeEntry = blib.TreeEntry
type BucketEntry = blib.BucketEntry
type Key = blib.Key
type KeyRange = blib.KeyRange
const KeyMax = blib.KeyMax
const KeyMin = blib.KeyMin
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment