.

go/neo/storage/sqlite/py/gen-testdata

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright (C) 2018  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""generate reference database for tests"""
+
+from zodbtools.util import storageFromURL
+from neo.tests.functional import NEOCluster
+import os
+from glob import glob
+
+def main():
+    zin  = storageFromURL("../../../zodb/storage/fs1/testdata/1.fs", read_only=True)
+
+    cluster = NEOCluster(db_list=["testdata/1.sqlite"], name="test", adapter="SQLite", clear_databases=True)
+    cluster.start()
+
+    zout = cluster.getZODBStorage()
+
+    zout.copyTransactionsFrom(zin)
+    zout.close()
+
+    cluster.stop()
+    cluster.waitAll()
+
+    # XXX current neo/go/sqlite limitation: it cannot reply !empty ttrans/tobj on open
+    # -> open/close storage to get t* flushed
+    cluster = NEOCluster(db_list=["testdata/1.sqlite"], name="test", adapter="SQLite", clear_databases=False)
+    cluster.start()
+    cluster.stop()
+    cluster.waitAll()
+
+    # remove sqlite journal     XXX ok?
+    for f in glob("testdata/*-journal"):
+        os.remove(f)
+
+
+
+if __name__ == '__main__':
+    main()

go/neo/storage/sqlite/sqlite.go

@@ -408,16 +408,17 @@ func (b *Backend) Close() error {
 	return err	// XXX err ctx
 }
 
-// ---- open by URL ----
+// ---- open ----
 
-func openURL(ctx context.Context, u *url.URL) (_ storage.Backend, err error) {
-	url := u.String()
-	dburl := strings.TrimPrefix(url, u.Scheme+"://") // url with stripped sqlite://
+// Open opens Backend connected to SQLite3 database @ dburl.
+//
+// dburl can be just filesystem path or SQLite3 database URI.
+func Open(dburl string) (_ *Backend, err error) {
 	connFactory := func() (*sqlite3.Conn, error) {
 		return sqlite3.Open(dburl)
 	}
 
-	b := &Backend{pool: newConnPool(connFactory), url: url}
+	b := &Backend{pool: newConnPool(connFactory), url: "sqlite://" + dburl}
 
 	defer func() {
 		if err != nil {
@@ -502,6 +503,18 @@ func openURL(ctx context.Context, u *url.URL) (_ storage.Backend, err error) {
 }
 
 
+func openURL(ctx context.Context, u *url.URL) (storage.Backend, error) {
+	url := u.String()
+	dburl := strings.TrimPrefix(url, u.Scheme+"://") // url with stripped sqlite://
+
+	b, err := Open(dburl)
+	if err == nil {
+		return b, nil
+	} else {
+		return nil, err	// XXX don't return just b -> will be !nil interface
+	}
+}
+
 func init() {
 	storage.RegisterBackend("sqlite", openURL)
 }

go/neo/storage/sqlite/sqlite_test.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package sqlite
+
+//go:generate ./py/gen-testdata
+
+import (
+	"context"
+	"testing"
+
+	"github.com/pkg/errors"
+
+	"lab.nexedi.com/kirr/go123/exc"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+// TODO verify data can be read as the same
+
+var bg = context.Background()
+
+func BenchmarkLoad(b *testing.B) {
+	back, err := Open("testdata/1.sqlite")
+	exc.Raiseif(err)
+
+	defer func() {
+		err := back.Close()
+		exc.Raiseif(err)
+	}()
+
+	lastTid, err := back.LastTid(bg)
+	exc.Raiseif(err)
+
+	xid := zodb.Xid{Oid: 0, At: lastTid}
+
+	b.ResetTimer()
+
+loop:
+	for i := 0; i < b.N; i++ {
+		obj, err := back.Load(bg, xid)
+		if err != nil {
+			switch errors.Cause(err).(type) {
+			case *zodb.NoObjectError:
+				xid.Oid = 0	// last object was read -> cycle back to beginning
+				continue loop
+
+			case *zodb.NoDataError:
+				panic(err)	// XXX object was deleted
+
+
+			default:
+				b.Fatal(err)
+			}
+		}
+
+		buf := obj.Data
+
+		if obj.Compression {
+			b.Fatal("compression was used")
+		}
+
+		buf.XRelease()
+	}
+}