X demo storage stub

go/zodb/storage.go

@@ -73,10 +73,10 @@ func RegisterDriver(scheme string, opener DriverOpener) {
 	driverRegistry[scheme] = opener
 }
 
-// XXX
-func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
+// XXX ...
+func OpenDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
 	// no scheme -> file://
-	if !strings.Contains(zurl, "://") {
+	if !strings.Contains(zurl, ":") {
 		zurl = "file://" + zurl
 	}
 
@@ -91,7 +91,7 @@ func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorag
 
 	opener, ok := driverRegistry[u.Scheme]
 	if !ok {
-		return nil, InvalidTid, fmt.Errorf("zodb: URL scheme \"%s://\" not supported", u.Scheme)
+		return nil, InvalidTid, fmt.Errorf("zodb: URL scheme \"%s:\" not supported", u.Scheme)
 	}
 
 	storDriver, at0, err := opener(ctx, u, opt)
@@ -116,7 +116,7 @@ func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error)
 		Watchq:   drvWatchq,
 	}
 
-	storDriver, at0, err := openDriver(ctx, zurl, drvOpt)
+	storDriver, at0, err := OpenDriver(ctx, zurl, drvOpt)
 	if err != nil {
 		return nil, err
 	}

go/zodb/storage/demo/demo.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package demo provides overlayed storage, similar to DemoStorage in ZODB/py.
+//
+// XXX
+// XXX link to DemoStorage
+package demo
+
+// TODO tests
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"regexp"
+
+	"lab.nexedi.com/kirr/go123/mem"
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/internal/task"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+// XXX
+type Storage struct {
+	base zodb.IStorageDriver
+	δ    zodb.IStorageDriver
+
+	baseAt0 zodb.Tid
+}
+
+
+// Close implements zodb.IStorageDriver .
+func (ovl *Storage) Close() error {
+	// XXX errctx?
+
+	err1 := ovl.δ.Close()
+	err2 := ovl.base.Close()
+	// XXX close base watcher
+	return xerr.Merge(err1, err2)
+}
+
+// Sync implements zodb.IStorageDriver .
+func (ovl *Storage) Sync(ctx context.Context) (head zodb.Tid, _ error) {
+	// XXX errctx?
+
+	head, err := ovl.δ.Sync(ctx)
+	if err != nil {
+		return zodb.InvalidTid, err
+	}
+
+	// δ is just created database
+	if head == 0 {
+		head = ovl.baseAt0
+	}
+
+	return head, nil
+}
+
+// Load implements zodb.IStorageDriver .
+func (ovl *Storage) Load(ctx context.Context, xid zodb.Xid) (*mem.Buf, zodb.Tid, error) {
+	// XXX errctx?
+
+	data, serial, err := ovl.δ.Load(ctx, xid)
+	if data != nil {
+		// object data is present in δ
+		return data, serial, err
+	}
+
+	useBase := false
+	var eNoData   *zodb.NoDataError
+	var eNoObject *zodb.NoObjectError
+	switch {
+	case errors.As(err, &eNoData):
+		if eNoData.DeletedAt != 0 {
+			// object deleted in δ -> whiteout
+			return data, serial, err
+		} else {
+			// object not created in δ
+			useBase = true
+		}
+
+	case errors.As(err, &eNoObject):
+		// object not created in δ
+		useBase = true
+	}
+
+	if !useBase {
+		return data, serial, err
+	}
+
+	// XXX cap .at in xid to base.At0 ? (and convert back on error return)
+	data, serial, err = ovl.base.Load(ctx, xid)
+	return data, serial, err
+}
+
+// Iterator implements zodb.IStorageDriver .
+func (ovl *Storage) Iterate(ctx context.Context, tidMin, tidMax zodb.Tid) zodb.ITxnIterator {
+	// XXX errctx?
+	panic("TODO") // XXX
+}
+
+// URL implements zodb.IStorageDriver .
+func (ovl *Storage) URL() string {
+	return "demo:(" + ovl.base.URL() + ")/(" + ovl.δ.URL() + ")"
+}
+
+var demoRe = regexp.MustCompile(`^\((.*)\)/\((.*)\)$`)
+
+func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb.IStorageDriver, _ zodb.Tid, err error) {
+	// demo:(base_zurl)/(δ_zurl)
+	defer task.Runningf(&ctx, "demo: open %s", u)(&err)
+
+	if !(u.Opaque != "" && u.RawQuery == "" && u.Fragment == "") {
+		// opaque != "" makes sure user,host,path are empty
+		return nil, zodb.InvalidTid, fmt.Errorf("invalid url")
+	}
+
+	argv := demoRe.FindStringSubmatch(u.Opaque)
+	if argv == nil {
+		return nil, zodb.InvalidTid, fmt.Errorf("invalid url")
+	}
+
+	baseZURL := argv[1]
+	δZURL    := argv[2]
+
+	// open base - always readonly
+	baseWatchq := make(chan zodb.Event)
+	base, baseAt0, err := zodb.OpenDriver(ctx, baseZURL, &zodb.DriverOptions{
+		ReadOnly: true,
+		Watchq:   baseWatchq,
+	})
+	if err != nil {
+		return nil, zodb.InvalidTid, err
+	}
+	defer func() {
+		if err != nil {
+			__ := base.Close()
+			err = xerr.Merge(err, __)
+		}
+	}()
+
+	// open δ    - as requested
+	δ, δAt0, err := zodb.OpenDriver(ctx, δZURL, opt)
+	if err != nil {
+		return nil, zodb.InvalidTid, err
+	}
+	defer func() {
+		if err != nil {
+			__ := δ.Close()
+			err = xerr.Merge(err, __)
+		}
+	}()
+
+	// verify that either
+	// - δ is all empty (just created), or
+	// - all δ transactions come strictly after base.
+	at0 := baseAt0
+	if δAt0 != 0 {
+		if δAt0 < baseAt0 {
+			return nil, zodb.InvalidTid, fmt.Errorf("base is ahead of δ: base.head=%s  δ.head=%s", baseAt0, δAt0)
+		}
+
+		iδ := δ.Iterate(ctx, 0, baseAt0)
+		for {
+			δtxni, _, err := iδ.NextTxn(ctx)
+			if err == io.EOF {
+				break // all ok - nothing in δ
+			}
+			if err != nil {
+				return nil, zodb.InvalidTid, err
+			}
+			// there is a δ transaction ∈ [δAt0, baseAt0)
+			// TODO iδ.Stop()
+			return nil, zodb.InvalidTid, fmt.Errorf("base overlaps with δ: base.head=%s  δ.tail=%s", baseAt0, δtxni.Tid)
+		}
+
+		at0 = δAt0
+	}
+
+	// XXX listen on baseWatchq and shutdown storage if base changes.
+	ovl := &Storage{
+		base: base,
+		δ   : δ,
+
+		baseAt0: baseAt0,
+	}
+
+	return ovl, at0, nil
+}
+
+func init() {
+	zodb.RegisterDriver("demo", openByURL)
+}

go/zodb/storage/demo/demo_test.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package demo
+
+// XXX stdtest Demo(data, ø)
+// XXX stdtest Demo(ø, data)
+// XXX more?

go/zodb/wks/wks.go

-// Copyright (C) 2017  Nexedi SA and Contributors.
-//                     Kirill Smelkov <kirr@nexedi.com>
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
 // it under the terms of the GNU General Public License version 3, or (at your
@@ -31,4 +31,5 @@ import (
 	_ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
 	_ "lab.nexedi.com/kirr/neo/go/zodb/storage/zeo"
 	_ "lab.nexedi.com/kirr/neo/go/neo"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/storage/demo"
 )

go/zodb/zodb.go

-// Copyright (C) 2016-2020  Nexedi SA and Contributors.
+// Copyright (C) 2016-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute

go/zodb/zodbtools/help.go

@@ -45,6 +45,8 @@ There are also following simpler ways:
 - zeo://<host>:<port>   for a ZEO database
 - /path/to/file         for a FileStorage database
 
+- demo:(zurl_base)/(zurl_δ)   XXX overlay, DemoStorage ...
+
 Please see zodburi documentation for full details:
 
 http://docs.pylonsproject.org/projects/zodburi/