Commit 6ed71cfd authored by Kirill Smelkov's avatar Kirill Smelkov

.

parent 1dd5d4c1
// Copyright (C) 2018 Nexedi SA and Contributors. // Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -29,7 +29,7 @@ import ( ...@@ -29,7 +29,7 @@ import (
sqlite3 "github.com/gwenn/gosqlite" sqlite3 "github.com/gwenn/gosqlite"
) )
// connPool is a pool of sqlite3.Conn // connPool is a pool of sqlite3.Conn .
type connPool struct { type connPool struct {
factory func() (*sqlite3.Conn, error) // =nil if pool closed factory func() (*sqlite3.Conn, error) // =nil if pool closed
......
...@@ -71,7 +71,6 @@ const schemaVersion = 3 ...@@ -71,7 +71,6 @@ const schemaVersion = 3
// table "config" stores configuration parameters which affect the persistent data. // table "config" stores configuration parameters which affect the persistent data.
// //
// XXX
// (name, nid, ptid, replicas, version, zodb=pickle...) // (name, nid, ptid, replicas, version, zodb=pickle...)
const config = ` const config = `
name TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL PRIMARY KEY,
...@@ -115,7 +114,7 @@ const obj = ` ...@@ -115,7 +114,7 @@ const obj = `
// `(partition, tid, oid)` // `(partition, tid, oid)`
// `(data_id)` // `(data_id)`
// XXX reenable for ^^^ // TODO create indices for obj
//index_dict['obj'] = ( //index_dict['obj'] = (
// "CREATE INDEX %s ON %s(partition, tid, oid)", // "CREATE INDEX %s ON %s(partition, tid, oid)",
// "CREATE INDEX %s ON %s(data_id)") // "CREATE INDEX %s ON %s(data_id)")
...@@ -127,7 +126,7 @@ const data = ` ...@@ -127,7 +126,7 @@ const data = `
compression INTEGER NOT NULL, compression INTEGER NOT NULL,
value BLOB NOT NULL value BLOB NOT NULL
` `
// XXX reenable for ^^^ // TODO reenable for ^^^
//if dedup: //if dedup:
// index_dict['data'] = ( // index_dict['data'] = (
// "CREATE UNIQUE INDEX %s ON %s(hash, compression)",) // "CREATE UNIQUE INDEX %s ON %s(hash, compression)",)
...@@ -156,9 +155,7 @@ const tobj = ` ...@@ -156,9 +155,7 @@ const tobj = `
PRIMARY KEY (tid, oid) PRIMARY KEY (tid, oid)
` `
// ----------------------------------------
type Backend struct { type Backend struct {
pool *connPool pool *connPool
...@@ -304,18 +301,11 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject ...@@ -304,18 +301,11 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject
}() }()
obj := &proto.AnswerObject{Oid: xid.Oid, DataSerial: 0} obj := &proto.AnswerObject{Oid: xid.Oid, DataSerial: 0}
// TODO reenable, but XXX we have to use Query, not QueryRow for RawBytes support
//var data sql.RawBytes
var data []byte var data []byte
// XXX recheck vvv with sqlite3 direct
// hash is variable-length BLOB - Scan refuses to put it into [20]byte
//var hash sql.RawBytes
var hash []byte var hash []byte
// obj.value_tid can be null // obj.value_tid can be null
//var valueTid sql.NullInt64 // XXX ok not to uint64 - max tid is max signed int64 var valueTid int64 // ok to be not uint64 - max tid is max signed int64
var valueTid int64 // XXX ok not to uint64 - max tid is max signed int64
// FIXME pid = getReadablePartition (= oid % Np; error if pid not readable) // FIXME pid = getReadablePartition (= oid % Np; error if pid not readable)
pid := 0 pid := 0
...@@ -323,7 +313,8 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject ...@@ -323,7 +313,8 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject
// XXX somehow detect errors in sql misuse and log them as 500 without reporting to client? // XXX somehow detect errors in sql misuse and log them as 500 without reporting to client?
// XXX such errors start with "unsupported Scan, " // XXX such errors start with "unsupported Scan, "
// XXX use conn for several query1 (see below) without intermediate returns to pool? // TODO use conn for several query1 (see below) without intermediate returns to pool?
// TODO try to use ScanRawBytes for data
err = b.query1( err = b.query1(
"SELECT tid, compression, data.hash, value, value_tid" + "SELECT tid, compression, data.hash, value, value_tid" +
...@@ -345,7 +336,7 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject ...@@ -345,7 +336,7 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject
case err == nil: case err == nil:
err = &zodb.NoDataError{ err = &zodb.NoDataError{
Oid: xid.Oid, Oid: xid.Oid,
DeletedAt: 0, // XXX hardcoded DeletedAt: 0, // FIXME hardcoded
} }
case err == errNoRows: case err == errNoRows:
...@@ -375,13 +366,13 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject ...@@ -375,13 +366,13 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject
obj.DataSerial = zodb.Tid(valueTid) obj.DataSerial = zodb.Tid(valueTid)
} }
// data -> obj.Data // data -> obj.Data
obj.Data = mem.BufAlloc(len(data)) obj.Data = mem.BufAlloc(len(data))
copy(obj.Data.Data, data) copy(obj.Data.Data, data)
// find out nextSerial // find out nextSerial
// XXX kill nextSerial support after neo/py cache does not need it // TODO kill nextSerial support after neo/py cache does not need it
// see also: https://github.com/zopefoundation/ZODB/pull/323
err = b.query1( err = b.query1(
"SELECT tid from obj" + "SELECT tid from obj" +
" WHERE partition=? AND oid=? AND tid>?" + " WHERE partition=? AND oid=? AND tid>?" +
...@@ -405,9 +396,13 @@ func (b *Backend) config(key string, pvalue *string) error { ...@@ -405,9 +396,13 @@ func (b *Backend) config(key string, pvalue *string) error {
return b.query1("SELECT value FROM config WHERE name=?", key).Scan(pvalue) return b.query1("SELECT value FROM config WHERE name=?", key).Scan(pvalue)
} }
func (b *Backend) Close() error { func (b *Backend) Close() (err error) {
err := b.pool.Close() defer func() {
return err // XXX err ctx if err != nil {
err = &zodb.OpError{URL: b.url, Op: "close", Err: err}
}
}()
return b.pool.Close()
} }
// ---- open ---- // ---- open ----
...@@ -450,12 +445,12 @@ func openConn(dburl string) (*sqlite3.Conn, error) { ...@@ -450,12 +445,12 @@ func openConn(dburl string) (*sqlite3.Conn, error) {
// primary SQLite author also confirms it should be working normally: // primary SQLite author also confirms it should be working normally:
// https://bugzilla.mozilla.org/show_bug.cgi?id=993556#c1 // https://bugzilla.mozilla.org/show_bug.cgi?id=993556#c1
// //
// XXX neo/py does not use locking_mode=EXCLUSIVE. // NOTE neo/py does not use locking_mode=EXCLUSIVE.
mode := "EXCLUSIVE" mode := "EXCLUSIVE"
mode_, err := conn.SetLockingMode("", mode) mode_, err := conn.SetLockingMode("", mode)
if err != nil { if err != nil {
conn.Close() conn.Close()
return nil, err // XXX or other error? return nil, err
} }
mode_ = strings.ToUpper(mode_) // sqlite returns "exclusive" mode_ = strings.ToUpper(mode_) // sqlite returns "exclusive"
...@@ -515,8 +510,6 @@ func Open(dburl string) (_ *Backend, err error) { ...@@ -515,8 +510,6 @@ func Open(dburl string) (_ *Backend, err error) {
value := "" value := ""
err := b.config(name, &value) err := b.config(name, &value)
// XXX prefix "b.path: config: %s:"
switch err { switch err {
case errNoRows: case errNoRows:
err = fmt.Errorf("not found") err = fmt.Errorf("not found")
...@@ -535,16 +528,13 @@ func Open(dburl string) (_ *Backend, err error) { ...@@ -535,16 +528,13 @@ func Open(dburl string) (_ *Backend, err error) {
checkConfig("version", schemaVersion) checkConfig("version", schemaVersion)
checkConfig("nid", int(proto.NID(proto.STORAGE, 1))) checkConfig("nid", int(proto.NID(proto.STORAGE, 1)))
checkConfig("replicas", 0) // XXX neo/py uses nreplicas as 1 + n(replica) checkConfig("replicas", 0) // neo/py uses nreplicas as 1 + n(replica)
err = errv.Err() err = errv.Err()
if err != nil { if err != nil {
return nil, fmt.Errorf("%s: NEO/go POC: not ready to handle: %s", dburl, err) return nil, fmt.Errorf("%s: NEO/go POC: not ready to handle: %s", dburl, err)
} }
// config("version")
// config("nid")
// config("replicas")
// config("name") // config("name")
// config("ptid") // config("ptid")
// config("backup_tid") // config("backup_tid")
......
// Copyright (C) 2018 Nexedi SA and Contributors. // Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -33,19 +33,17 @@ import ( ...@@ -33,19 +33,17 @@ import (
// TODO verify data can be read as the same // TODO verify data can be read as the same
var bg = context.Background()
func BenchmarkLoad(b *testing.B) { func BenchmarkLoad(b *testing.B) {
back, err := Open("testdata/1.sqlite") X := exc.Raiseif
exc.Raiseif(err)
back, err := Open("testdata/1.sqlite"); X(err)
defer func() { defer func() {
err := back.Close() err := back.Close(); X(err)
exc.Raiseif(err)
}() }()
lastTid, err := back.LastTid(bg) ctx := context.Background()
exc.Raiseif(err)
lastTid, err := back.LastTid(ctx); X(err)
xid := zodb.Xid{Oid: 0, At: lastTid} xid := zodb.Xid{Oid: 0, At: lastTid}
...@@ -53,7 +51,7 @@ func BenchmarkLoad(b *testing.B) { ...@@ -53,7 +51,7 @@ func BenchmarkLoad(b *testing.B) {
loop: loop:
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
obj, err := back.Load(bg, xid) obj, err := back.Load(ctx, xid)
if err != nil { if err != nil {
switch errors.Cause(err).(type) { switch errors.Cause(err).(type) {
case *zodb.NoObjectError: case *zodb.NoObjectError:
...@@ -63,7 +61,6 @@ loop: ...@@ -63,7 +61,6 @@ loop:
case *zodb.NoDataError: case *zodb.NoDataError:
panic(err) // XXX object was deleted panic(err) // XXX object was deleted
default: default:
b.Fatal(err) b.Fatal(err)
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment