Commit 4000df14 authored by Kirill Smelkov's avatar Kirill Smelkov

X neo/storage: Open backends by raw string, not URL

For both current backends (SQLite and fs1) there is currently a problem
of working with UTF-8 filenames if opening is handled via urls. For
example if backend string is

	sqlite://neo.sqlite·P1-1

after parsing as URL and further reassembled as string it will be

	neo.sqlite%C2%B7P1-1

with then errors like

	neo.sqlite%C2%B7P1-1: open: unable to open database file

showing.

Since storage backends primarily work with local files it makes sense
not to play raw url reassembly games and just pass raw strings to
backends for open.

SQLite uri processing is still possible, via e.g.

	sqlite://file:filepath?param=value,...

See

	https://www.sqlite.org/c3ref/open.html	and
	https://www.sqlite.org/uri.html

for details.
parent cf1f7c24
...@@ -22,7 +22,6 @@ package fs1 ...@@ -22,7 +22,6 @@ package fs1
import ( import (
"context" "context"
"net/url"
"lab.nexedi.com/kirr/neo/go/neo/internal/xsha1" "lab.nexedi.com/kirr/neo/go/neo/internal/xsha1"
"lab.nexedi.com/kirr/neo/go/neo/proto" "lab.nexedi.com/kirr/neo/go/neo/proto"
...@@ -93,15 +92,17 @@ func (f *Backend) Load(ctx context.Context, xid zodb.Xid) (*proto.AnswerObject, ...@@ -93,15 +92,17 @@ func (f *Backend) Load(ctx context.Context, xid zodb.Xid) (*proto.AnswerObject,
} }
// ---- open by URL ---- // ---- open ----
func openURL(ctx context.Context, u *url.URL) (storage.Backend, error) { func openBackend(ctx context.Context, path string) (storage.Backend, error) {
// TODO handle query b, err := Open(ctx, path)
// XXX u.Path is not always raw path - recheck and fix if err == nil {
path := u.Host + u.Path return b, nil
return Open(ctx, path) } else {
return nil, err // XXX don't return just b -> will be !nil interface
}
} }
func init() { func init() {
storage.RegisterBackend("fs1", openURL) storage.RegisterBackend("fs1", openBackend)
} }
...@@ -25,7 +25,6 @@ import ( ...@@ -25,7 +25,6 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"net/url"
"strings" "strings"
//"reflect" //"reflect"
...@@ -416,6 +415,7 @@ func (b *Backend) Close() error { ...@@ -416,6 +415,7 @@ func (b *Backend) Close() error {
func openConn(dburl string) (*sqlite3.Conn, error) { func openConn(dburl string) (*sqlite3.Conn, error) {
conn, err := sqlite3.Open(dburl, conn, err := sqlite3.Open(dburl,
sqlite3.OpenNoMutex, // we use connections only from 1 goroutine simultaneously sqlite3.OpenNoMutex, // we use connections only from 1 goroutine simultaneously
sqlite3.OpenUri, // handle file:... URIs
sqlite3.OpenReadWrite) //, sqlite3.OpenSharedCache) sqlite3.OpenReadWrite) //, sqlite3.OpenSharedCache)
if err != nil { if err != nil {
...@@ -540,7 +540,7 @@ func Open(dburl string) (_ *Backend, err error) { ...@@ -540,7 +540,7 @@ func Open(dburl string) (_ *Backend, err error) {
err = errv.Err() err = errv.Err()
if err != nil { if err != nil {
return nil, fmt.Errorf("NEO/go POC: not ready to handle: %s", err) return nil, fmt.Errorf("%s: NEO/go POC: not ready to handle: %s", dburl, err)
} }
// config("version") // config("version")
...@@ -562,11 +562,11 @@ func Open(dburl string) (_ *Backend, err error) { ...@@ -562,11 +562,11 @@ func Open(dburl string) (_ *Backend, err error) {
err = errv.Err() err = errv.Err()
if err != nil { if err != nil {
return nil, fmt.Errorf("NEO/go POC: checking ttrans/tobj: %s", err) return nil, fmt.Errorf("%s: NEO/go POC: checking ttrans/tobj: %s", dburl, err)
} }
if !(nttrans==0 && ntobj==0) { if !(nttrans==0 && ntobj==0) {
return nil, fmt.Errorf("NEO/go POC: not ready to handle: !empty ttrans/tobj") return nil, fmt.Errorf("%s: NEO/go POC: not ready to handle: !empty ttrans/tobj", dburl)
} }
// TODO lock db by path so other process cannot start working with it // TODO lock db by path so other process cannot start working with it
...@@ -575,10 +575,7 @@ func Open(dburl string) (_ *Backend, err error) { ...@@ -575,10 +575,7 @@ func Open(dburl string) (_ *Backend, err error) {
} }
func openURL(ctx context.Context, u *url.URL) (storage.Backend, error) { func openBackend(ctx context.Context, dburl string) (storage.Backend, error) {
url := u.String()
dburl := strings.TrimPrefix(url, u.Scheme+"://") // url with stripped sqlite://
b, err := Open(dburl) b, err := Open(dburl)
if err == nil { if err == nil {
return b, nil return b, nil
...@@ -588,5 +585,5 @@ func openURL(ctx context.Context, u *url.URL) (storage.Backend, error) { ...@@ -588,5 +585,5 @@ func openURL(ctx context.Context, u *url.URL) (storage.Backend, error) {
} }
func init() { func init() {
storage.RegisterBackend("sqlite", openURL) storage.RegisterBackend("sqlite", openBackend)
} }
...@@ -25,6 +25,7 @@ import ( ...@@ -25,6 +25,7 @@ import (
"fmt" "fmt"
"net/url" "net/url"
"sort" "sort"
"strings"
"sync" "sync"
"lab.nexedi.com/kirr/neo/go/neo/proto" "lab.nexedi.com/kirr/neo/go/neo/proto"
...@@ -49,7 +50,7 @@ type Backend interface { ...@@ -49,7 +50,7 @@ type Backend interface {
} }
// BackendOpener is a function to open a NEO storage backend // BackendOpener is a function to open a NEO storage backend
type BackendOpener func (ctx context.Context, u *url.URL) (Backend, error) type BackendOpener func (ctx context.Context, path string) (Backend, error)
// {} scheme -> BackendOpener // {} scheme -> BackendOpener
var backMu sync.Mutex var backMu sync.Mutex
...@@ -99,5 +100,7 @@ func OpenBackend(ctx context.Context, backendURL string) (Backend, error) { ...@@ -99,5 +100,7 @@ func OpenBackend(ctx context.Context, backendURL string) (Backend, error) {
return nil, fmt.Errorf("neo: backend: open %q: URL scheme \"%s://\" not supported", backendURL, u.Scheme) return nil, fmt.Errorf("neo: backend: open %q: URL scheme \"%s://\" not supported", backendURL, u.Scheme)
} }
return opener(ctx, u) // open via raw path with only scheme:// stripped
path := strings.TrimPrefix(backendURL, u.Scheme+"://")
return opener(ctx, path)
} }
...@@ -292,7 +292,7 @@ func zwrkMain(argv []string) { ...@@ -292,7 +292,7 @@ func zwrkMain(argv []string) {
// zwrk simulates database load from multiple clients. // zwrk simulates database load from multiple clients.
// //
// It first serially reads all objects and remember theirs per-object crc32 // It first serially reads all objects and remember theirs per-object crc32
// checksum. If h/check were provided this phase, similarly to zhash, also // checksum. If h/check were provided, this phase, similarly to zhash, also
// checks that the whole database content is as expected. // checks that the whole database content is as expected.
// //
// Then parallel phase starts where nwrk separate connections to database are // Then parallel phase starts where nwrk separate connections to database are
...@@ -414,7 +414,7 @@ func zwrkPreconnect(ctx context.Context, url string, at zodb.Tid, nwrk int) (_ [ ...@@ -414,7 +414,7 @@ func zwrkPreconnect(ctx context.Context, url string, at zodb.Tid, nwrk int) (_ [
// load several first objects to warm up storages connection // load several first objects to warm up storages connection
// we need to load several objects so that in case of // we need to load several objects so that in case of
// NEO cluster with several storage nodes we warm-up // NEO cluster with several storage nodes, we warm-up
// connections to them all. // connections to them all.
// //
// FIXME 16 hardcoded // FIXME 16 hardcoded
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment