X StorageBackend -> storage.Backend + registry system a-la zodb

go/neo/cmd/neo/.gitignore

+/neo

go/neo/cmd/neo/storage.go

@@ -32,19 +32,32 @@ import (
 	"lab.nexedi.com/kirr/go123/prog"
 	"lab.nexedi.com/kirr/go123/xnet"
 	"lab.nexedi.com/kirr/neo/go/neo"
-	"lab.nexedi.com/kirr/neo/go/neo/storage/fs1"
+	"lab.nexedi.com/kirr/neo/go/neo/storage"
+
+	_ "lab.nexedi.com/kirr/neo/go/neo/storage/fs1"
+	_ "lab.nexedi.com/kirr/neo/go/neo/storage/sql"
 )
 
 const storageSummary = "run storage node"
 
 func storageUsage(w io.Writer) {
 	fmt.Fprintf(w,
-`Usage: neo storage [options] <data.fs>
+`Usage: neo storage [options] <data>
 Run NEO storage node.
 
-<data.fs> is a path to FileStorage v1 file.
+<data> is backend URL for data persistence.
+
+Available backends are:
 
+`)
+	for _, back := range storage.AvailableBackends() {
+		fmt.Fprintf(w, "- %s://\n", back)
+	}
+
+	fmt.Fprintf(w,
+`
 XXX currently storage is read-only.
+
 `)
 }
 
@@ -85,15 +98,14 @@ func storageMain(argv []string) {
 	runtime.GOMAXPROCS(maxprocs*8)		// XXX *8 is enough?
 
 
-	// XXX hack to use existing zodb storage for data
-	zback, err := fs1.Open(context.Background(), argv[0])
+	back, err := storage.OpenBackend(context.Background(), argv[0])
 	if err != nil {
 		prog.Fatal(err)
 	}
 
 	net := xnet.NetPlain("tcp")	// TODO + TLS; not only "tcp" ?
 
-	storSrv := neo.NewStorage(*cluster, master, *bind, net, zback)
+	storSrv := neo.NewStorage(*cluster, master, *bind, net, back)
 
 	ctx := context.Background()
 	/*

go/neo/neo.go

@@ -21,7 +21,7 @@
 //
 // Package neo itself provides protocol definition and common infrastructure.
 // See packages neo.client and neo.server for client and server sides respectively.
-// XXX text
+// FIXME text
 package neo
 
 //go:generate gotrace gen .

go/neo/storage.go

@@ -30,37 +30,20 @@ import (
 
 	"lab.nexedi.com/kirr/neo/go/neo/neonet"
 	"lab.nexedi.com/kirr/neo/go/neo/proto"
+	"lab.nexedi.com/kirr/neo/go/neo/storage"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 	"lab.nexedi.com/kirr/neo/go/xcommon/log"
 	"lab.nexedi.com/kirr/neo/go/xcommon/task"
 	"lab.nexedi.com/kirr/neo/go/xcommon/xcontext"
 	"lab.nexedi.com/kirr/neo/go/xcommon/xio"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xerr"
 	"lab.nexedi.com/kirr/go123/xnet"
 )
 
-// StorageBackend is the interface for actual storage service that is used by Storage node.
-type StorageBackend interface {
-	// LastTid should return the id of the last committed transaction.
-	//
-	// XXX same as in zodb.IStorageDriver
-	// XXX +viewAt ?
-	LastTid(ctx context.Context) (zodb.Tid, error)
-
-	// LastOid should return the max object id stored.
-	LastOid(ctx context.Context) (zodb.Oid, error)
-
-	// Load, similarly to zodb.IStorageDriver.Load should load object data addressed by xid.
-	// FIXME kill nextSerial support after neo/py cache does not depend on next_serial
-	// XXX +viewAt ?
-	Load(ctx context.Context, xid zodb.Xid) (buf *mem.Buf, serial, nextSerial zodb.Tid, err error)
-}
-
 // Storage is NEO node that keeps data and provides read/write access to it via network.
 //
-// Storage implements only NEO protocol logic with data being persisted via provided StorageBackend.
+// Storage implements only NEO protocol logic with data being persisted via provided storage.Backend.
 type Storage struct {
 	node *NodeApp
 
@@ -70,7 +53,7 @@ type Storage struct {
 	opMu  sync.Mutex
 	opCtx context.Context
 
-	back StorageBackend
+	back storage.Backend
 
 	//nodeCome chan nodeCome	// node connected
 }
@@ -79,7 +62,7 @@ type Storage struct {
 //
 // The storage uses back as underlying backend for storing data.
 // Use Run to actually start running the node.
-func NewStorage(clusterName, masterAddr, serveAddr string, net xnet.Networker, back StorageBackend) *Storage {
+func NewStorage(clusterName, masterAddr, serveAddr string, net xnet.Networker, back storage.Backend) *Storage {
 	stor := &Storage{
 		node: NewNodeApp(net, proto.STORAGE, clusterName, masterAddr, serveAddr),
 		back: back,

go/neo/storage/fs1/fs1.go

@@ -22,10 +22,11 @@ package fs1
 
 import (
 	"context"
+	"net/url"
 
 	"lab.nexedi.com/kirr/go123/mem"
 
-	//"lab.nexedi.com/kirr/neo/go/neo"
+	"lab.nexedi.com/kirr/neo/go/neo/storage"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 	"lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
 )
@@ -48,9 +49,7 @@ type FS1Backend struct {
 	zstor *fs1.FileStorage // underlying ZODB storage
 }
 
-// XXX disabled not to create import cycle with neo(test)
-// XXX -> backend registry?
-//var _ neo.StorageBackend = (*FS1Backend)(nil)
+var _ storage.Backend = (*FS1Backend)(nil)
 
 func Open(ctx context.Context, path string) (*FS1Backend, error) {
 	zstor, err := fs1.Open(ctx, path)
@@ -73,3 +72,17 @@ func (f *FS1Backend) Load(ctx context.Context, xid zodb.Xid) (*mem.Buf, zodb.Tid
 	// FIXME kill nextSerial support after neo/py cache does not depend on next_serial
 	return f.zstor.Load_XXXWithNextSerialXXX(ctx, xid)
 }
+
+
+// ---- open by URL ----
+
+func openURL(ctx context.Context, u *url.URL) (storage.Backend, error) {
+	// TODO handle query
+	// XXX u.Path is not always raw path - recheck and fix
+	path := u.Host + u.Path
+	return Open(ctx, path)
+}
+
+func init() {
+	storage.RegisterBackend("fs1", openURL)
+}

go/neo/storage/sql/sql.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package sql provides NEO storage backend that uses SQL database for persistence.
+package sql
+
+import (
+	"context"
+	"net/url"
+
+	//"lab.nexedi.com/kirr/go123/mem"
+
+	"lab.nexedi.com/kirr/neo/go/neo/storage"
+	//"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+
+type SQLBackend struct {
+}
+
+func openURL(ctx context.Context, u *url.URL) (storage.Backend, error) {
+	return nil, nil
+}
+
+
+func init() {
+	storage.RegisterBackend("sqlite", openURL)
+}

go/neo/storage/storage.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package storage provides infrastructure common to all NEO storage backends.
+package storage
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"sort"
+	"sync"
+
+	"lab.nexedi.com/kirr/go123/mem"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+// Backend is the interface for actual storage service that is used by neo.Storage node.
+type Backend interface {
+	// LastTid should return the id of the last committed transaction.
+	//
+	// XXX same as in zodb.IStorageDriver
+	// XXX +viewAt ?
+	LastTid(ctx context.Context) (zodb.Tid, error)
+
+	// LastOid should return the max object id stored.
+	LastOid(ctx context.Context) (zodb.Oid, error)
+
+	// Load, similarly to zodb.IStorageDriver.Load should load object data addressed by xid.
+	// FIXME kill nextSerial support after neo/py cache does not depend on next_serial
+	// XXX +viewAt ?
+	Load(ctx context.Context, xid zodb.Xid) (buf *mem.Buf, serial, nextSerial zodb.Tid, err error)
+}
+
+// BackendOpener is a function to open a NEO storage backend
+type BackendOpener func (ctx context.Context, u *url.URL) (Backend, error)
+
+// {} scheme -> BackendOpener
+var backMu sync.Mutex
+var backRegistry = map[string]BackendOpener{}
+
+// RegisterBackend registers opener to be used for backend URLs with scheme.
+func RegisterBackend(scheme string, opener BackendOpener) {
+	backMu.Lock()
+	defer backMu.Unlock()
+
+	if _, already := backRegistry[scheme]; already {
+		panic(fmt.Errorf("NEO storage backend with scheme %q was already registered", scheme))
+	}
+
+	backRegistry[scheme] = opener
+}
+
+// AvailableBackends returns list of all backend schemes registered.
+//
+// the returned list is sorted.
+func AvailableBackends() []string {
+	backMu.Lock()
+	defer backMu.Unlock()
+
+	var backv []string
+	for back := range backRegistry {
+		backv = append(backv, back)
+	}
+	sort.Strings(backv)
+
+	return backv
+}
+
+
+// OpenBackend opens NEO storage backend by URL.
+func OpenBackend(ctx context.Context, backendURL string) (Backend, error) {
+	u, err := url.Parse(backendURL)
+	if err != nil {
+		return nil, err
+	}
+
+	backMu.Lock()
+	defer backMu.Unlock()
+
+	opener, ok := backRegistry[u.Scheme]
+	if !ok {
+		return nil, fmt.Errorf("neo: backend: URL scheme \"%s://\" not supported", u.Scheme)
+	}
+
+	return opener(ctx, u)
+}