Commit 4fb6bd0a authored by Kirill Smelkov's avatar Kirill Smelkov

go/zodb/demo: New package that provides base+δ storages overlay

For Load demo.Storage implementation is similar to DemoStorage in
ZODB/py with fixes "cherry-picked" from:

- https://github.com/zopefoundation/ZODB/issues/318
  (DemoStorage does not take whiteouts into account -> leading to data corruption)

- https://github.com/zopefoundation/ZODB/pull/323
  (loadAt + fix for the above issue)

For safety demo.Storage - contrary to DemoStorage/py - actually verifies
that for demo=base+δ δ comes strictly after base and that base remains
unchanged.

URI schema follows XRI Cross-references approach and is

	demo:(zurl_base)/(zurl_δ)

https://en.wikipedia.org/wiki/Extensible_Resource_Identifier provides
some related details and examples.

For ZODB/py corresponding pull-request for zodburi to support demo: URI
scheme has been made here: https://github.com/Pylons/zodburi/pull/29 .

Tests need:

- recent zodbtools with zodbrestore:

  https://lab.nexedi.com/nexedi/zodbtools/blob/129afa67/zodbtools/zodbrestore.py
  nexedi/zodbtools!19

- ZODB with support for DemoStorage.deleteObject
  https://github.com/zopefoundation/ZODB/pull/341

On Go side demo storage is needed for wendelin.core 2 because ERP5 uses
DemoStorage to run tests.
parent 954321b2
...@@ -29,11 +29,13 @@ import ( ...@@ -29,11 +29,13 @@ import (
"os" "os"
"os/exec" "os/exec"
"reflect" "reflect"
"strings"
"sync" "sync"
"testing" "testing"
"time" "time"
"lab.nexedi.com/kirr/go123/xerr" "lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/go123/xstrings"
"lab.nexedi.com/kirr/neo/go/zodb" "lab.nexedi.com/kirr/neo/go/zodb"
) )
...@@ -134,6 +136,33 @@ func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err ...@@ -134,6 +136,33 @@ func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err
return tid, nil return tid, nil
} }
// ZPyRestore restores transactions specified by zin in zodbdump format.
//
// The restore is performed via zodbtools/py.
func ZPyRestore(zurl string, zin string) (tidv []zodb.Tid, err error) {
defer xerr.Contextf(&err, "%s: zpyrestore", zurl)
// run py `zodb restore`
cmd:= exec.Command("python2", "-m", "zodbtools.zodb", "restore", zurl)
cmd.Stdin = strings.NewReader(zin)
cmd.Stderr = os.Stderr
out, err := cmd.Output()
if err != nil {
return nil, err
}
for _, line := range xstrings.SplitLines(string(out), "\n") {
tid, err := zodb.ParseTid(line)
if err != nil {
return nil, fmt.Errorf("restored, but invalid output: %s", err)
}
tidv = append(tidv, tid)
}
return tidv, nil
}
// ---- tests for storage drivers ---- // ---- tests for storage drivers ----
......
This diff is collapsed.
// Copyright (C) 2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package demo
import (
"bytes"
"context"
"io/ioutil"
"fmt"
"net/url"
"os"
"reflect"
"regexp"
"testing"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/internal/xtesting"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/zodbtools"
// for file: scheme support
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
)
// DemoData represents data for a demo: storage.
type DemoData struct {
base string // url for base.fs
δ string // ----/---- δ.fs
}
func (ddat *DemoData) URL() string {
return fmt.Sprintf("demo:(%s)/(%s)", ddat.base, ddat.δ)
}
// tOptions represents options for testing.
// TODO -> xtesting
type tOptions struct {
Preload string // preload database with data from this location
}
// withDemoData tests f with all kinds of opt.Preload data split into base + δ.
func withDemoData(t *testing.T, f func(t *testing.T, ddat *DemoData), optv ...tOptions) {
t.Helper()
X := xtesting.FatalIf(t)
opt := tOptions{}
if len(optv) > 1 {
panic("multiple tOptions not allowed")
}
if len(optv) == 1 {
opt = optv[0]
}
// retrieve zdump of Preload
zdump := ""
if opt.Preload != "" {
ctx := context.Background()
buf := &bytes.Buffer{}
stor, err := zodb.Open(ctx, opt.Preload, &zodb.OpenOptions{ReadOnly: true}); X(err)
err = zodbtools.Dump(ctx, buf, stor, 0, zodb.TidMax, /*hashonly=*/false)
stor.Close()
X(err)
zdump = buf.String()
}
// split zdump into transactions
// XXX hacky; TODO -> zodbtools.DumpReader
txnRe := regexp.MustCompile(`(?m)^txn (?P<tid>[0-9a-f]{16}) "(?P<status>.)"$`)
type zdumpTxn struct {
tid zodb.Tid
pos int // where this transaction starts in the dump
}
var txnv []zdumpTxn
for _, m := range txnRe.FindAllStringSubmatchIndex(zdump, -1) {
// [m[0]:m[1]] refers to whole txn line
__ := zdump[m[2]:m[3]]
tid, err := zodb.ParseTid(__); X(err)
txnv = append(txnv, zdumpTxn{tid, m[0]})
}
// verify f on all combinations of preload being split into base+δ
work := xtempdir(t)
defer os.RemoveAll(work)
test1 := func(δstart zodb.Tid, zdumpBase, zdumpδ string) {
t.Helper()
t.Run(fmt.Sprintf("δstart=%s", δstart), func(t *testing.T) {
t.Helper()
X := xtesting.FatalIf(t)
work1 := work + "/δ" + δstart.String()
err := os.Mkdir(work1, 0777); X(err)
base := "file://"+work1+"/base.fs"
δ := "file://"+work1+"/δ.fs"
ddat := &DemoData{base, δ}
_, err = xtesting.ZPyRestore(base, zdumpBase); X(err)
// restore δ part via `demo:(base)/(δ)` - not `file:δ`.
// The reason we do this is because restoring δ via
// just its file will fail when restoring copy data
// record with copy_from transaction being in base.
_, err = xtesting.ZPyRestore(ddat.URL(), zdumpδ); X(err)
f(t, ddat)
})
}
for i := 0; i < len(txnv); i++ {
δtail := txnv[i]
test1(δtail.tid, zdump[:δtail.pos], zdump[δtail.pos:])
}
test1(zodb.TidMax, zdump, "")
}
// withDemo tests f with demo: client connected to all kind of demo data splits.
func withDemo(t *testing.T, f func(t *testing.T, ddat *DemoData, ddrv *Storage), optv ...tOptions) {
t.Helper()
withDemoData(t, func(t *testing.T, ddat *DemoData) {
t.Helper()
X := xtesting.FatalIf(t)
ddrv, _, err := demoOpen(ddat.URL(), &zodb.DriverOptions{ReadOnly: true}); X(err)
defer func() {
err := ddrv.Close(); X(err)
}()
f(t, ddat, ddrv)
}, optv...)
}
func TestURL(t *testing.T) {
withDemo(t, func(t *testing.T, ddat *DemoData, ddrv *Storage) {
zurl := ddrv.URL()
zurlOk := ddat.URL()
if zurl != zurlOk {
t.Fatalf("bad zurl:\nhave: %s\nwant: %s", zurl, zurlOk)
}
})
}
func TestEmptyDB(t *testing.T) {
withDemo(t, func(t *testing.T, _ *DemoData, ddrv *Storage) {
xtesting.DrvTestEmptyDB(t, ddrv)
})
}
func TestLoad(t *testing.T) {
X := xtesting.FatalIf(t)
data := "../fs1/testdata/1.fs"
txnvOk, err := xtesting.LoadDBHistory(data); X(err)
withDemo(t, func(t *testing.T, _ *DemoData, ddrv *Storage) {
xtesting.DrvTestLoad(t, ddrv, txnvOk)
}, tOptions{
Preload: data,
})
}
func TestWatch(t *testing.T) {
withDemoData(t, func(t *testing.T, ddat *DemoData) {
xtesting.DrvTestWatch(t, ddat.URL(), openByURL)
})
}
// MutateBase mutates ddat.base with new commit.
func (ddat *DemoData) MutateBase() (zodb.Tid, error) {
return xtesting.ZPyCommitRaw(ddat.base, 0, xtesting.ZRawObject{
Oid: 1,
Data: []byte("ZZZ"),
})
}
// TestSync_vs_BaseMutate verifies Sync wrt base mutation.
func TestSync_vs_BaseMutate(t *testing.T) {
withDemo(t, func(t *testing.T, ddat *DemoData, ddrv *Storage) {
X := xtesting.FatalIf(t)
head, err := ddrv.Sync(context.Background())
if !(head == 0 && err == nil) {
t.Fatalf("sync0: head=%s err=%s", head, err)
}
tid, err := ddat.MutateBase(); X(err)
head, err = ddrv.Sync(context.Background())
errOk := &zodb.OpError{URL: ddrv.URL(), Op: "sync", Err: &baseMutatedError{
baseAt0: 0,
baseHead: tid,
}}
if !reflect.DeepEqual(err, errOk) {
t.Fatalf("after base mutate: sync: unexpected error:\nhave: %s\nwant: %s",
err, errOk)
}
})
}
// TestWatchLoad_vs_BaseMutate verifies Watch and Load wrt base mutation.
func TestWatchLoad_vs_BaseMutate(t *testing.T) {
withDemoData(t, func(t *testing.T, ddat *DemoData) {
X := xtesting.FatalIf(t)
watchq := make(chan zodb.Event)
ddrv, at0, err := demoOpen(ddat.URL(), &zodb.DriverOptions{
ReadOnly: true,
Watchq: watchq,
}); X(err)
defer func() {
err := ddrv.Close(); X(err)
}()
tid, err := ddat.MutateBase(); X(err)
// first wait for error from watchq
event, ok := <-watchq
if !ok {
t.Fatal("after base mutate: premature watchq close")
}
evErr, ok := event.(*zodb.EventError)
if !ok {
t.Fatalf("after base mutate: unexpected event: %T", event)
}
errBaseMutated := &baseMutatedError{
baseAt0: 0,
baseHead: tid,
}
evErrOk := &zodb.EventError{&zodb.OpError{URL: ddrv.URL(), Op: "watcher", Err: errBaseMutated}}
if !reflect.DeepEqual(evErr, evErrOk) {
t.Fatalf("after base mutate: unexpected event:\nhave: %s\nwant: %s", evErr, evErrOk)
}
// now make sure Load fails with "base mutated" error
xid := zodb.Xid{Oid: 1, At: at0}
data, serial, err := ddrv.Load(context.Background(), xid)
errOk := &zodb.OpError{URL: ddrv.URL(), Op: "load", Args: xid, Err: errBaseMutated}
if !reflect.DeepEqual(err, errOk) {
t.Fatalf("after base mutate: load: unexpected error:\nhave: %s\nwant: %s",
err, errOk)
}
if !(data == nil && serial == zodb.InvalidTid) {
t.Fatalf("after base mutate: load: unexpected data=%v serial=%v", data, serial)
}
})
}
func demoOpen(zurl string, opt *zodb.DriverOptions) (_ *Storage, at0 zodb.Tid, err error) {
defer xerr.Contextf(&err, "opendemo %s", zurl)
u, err := url.Parse(zurl)
if err != nil {
return nil, 0, err
}
d, at0, err := openByURL(context.Background(), u, opt)
if err != nil {
return nil, 0, err
}
return d.(*Storage), at0, nil
}
func xtempdir(t *testing.T) string {
t.Helper()
tmpd, err := ioutil.TempDir("", "demo")
if err != nil {
t.Fatal(err)
}
return tmpd
}
// Copyright (C) 2017 Nexedi SA and Contributors. // Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -30,4 +30,5 @@ package wks ...@@ -30,4 +30,5 @@ package wks
import ( import (
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1" _ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/zeo" _ "lab.nexedi.com/kirr/neo/go/zodb/storage/zeo"
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/demo"
) )
// Copyright (C) 2017-2019 Nexedi SA and Contributors. // Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -44,6 +44,7 @@ There are also following simpler ways: ...@@ -44,6 +44,7 @@ There are also following simpler ways:
- neo://<db>@<master> for a NEO database - neo://<db>@<master> for a NEO database
- zeo://<host>:<port> for a ZEO database - zeo://<host>:<port> for a ZEO database
- /path/to/file for a FileStorage database - /path/to/file for a FileStorage database
- demo:(zurl_base)/(zurl_δ) for a DemoStorage overlay
Please see zodburi documentation for full details: Please see zodburi documentation for full details:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment