.

go/zodb/btree/btree.go

+// Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+// All Rights Reserved.
+//
+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This software is subject to the provisions of the Zope Public License,
+// Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+// THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+// WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+// FOR A PARTICULAR PURPOSE
+
+package main
+// ZODB BTree handling			XXX -> zodb
+
+import (
+	"context"
+	"sort"
+
+	"lab.nexedi.com/kirr/go123/xerr"
+	//"lab.nexedi.com/kirr/neo/go/zodb"
+	pickle "github.com/kisielk/og-rek"
+)
+
+// XXX -> template
+type KEY int64
+
+
+// ZBucket mimics ?OBucket from btree/py, with ? being any integer.
+//
+// py description:
+//
+// A Bucket wraps contiguous vectors of keys and values.  Keys are unique,
+// and stored in sorted order.  The 'values' pointer may be NULL if the
+// Bucket is used to implement a set.  Buckets serving as leafs of BTrees
+// are chained together via 'next', so that the entire BTree contents
+// can be traversed in sorted order quickly and easily.
+type ZBucket struct {
+	*PyPersistent
+
+	next   *ZBucket		// the bucket with the next-larger keys
+	keys   []KEY		// 'len' keys, in increasing order
+	values []interface{}	// 'len' corresponding values
+}
+
+// zBTreeItem mimics BTreeItem from btree/py.
+type zBTreeItem struct {
+	key	KEY
+	child	interface{}	// ZBTree or ZBucket
+}
+
+// ZBTree mimics ?OBTree from btree/py, with ? being any integer.
+//
+// See https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/Development.txt#L198
+// for details.
+type ZBTree struct {
+	*PyPersistent
+
+	// firstbucket points to the bucket containing the smallest key in
+	// the BTree.  This is found by traversing leftmost child pointers
+	// (data[0].child) until reaching a Bucket.
+	firstbucket *ZBucket
+
+	// The BTree points to 'len' children, via the "child" fields of the data
+	// array.  There are len-1 keys in the 'key' fields, stored in increasing
+	// order.  data[0].key is unused.  For i in 0 .. len-1, all keys reachable
+	// from data[i].child are >= data[i].key and < data[i+1].key, at the
+	// endpoints pretending that data[0].key is minus infinity and
+	// data[len].key is positive infinity.
+	data	[]zBTreeItem
+}
+
+// Get searches BTree by key.
+//
+// It loads intermediate BTree nodes from database on demand as needed.
+func (t *ZBTree) Get(ctx context.Context, key KEY) (_ interface{}, _ bool, err error) {
+	defer xerr.Contextf(&err, "btree(%s): get %s", t.POid(), key)	// XXX + url?
+	err = t.PActivate(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+
+	if len(t.data) == 0 {
+		// empty btree
+		t.PDeactivate()
+		return nil, false, nil
+	}
+
+	for {
+		// search i: K(i) ≤ k < K(i+1)	; K(0) = -∞, K(len) = +∞
+		i := sort.Search(len(t.data), func(i int) bool {
+			j := i + 1
+			if j == len(t.data) {
+				return true	// [len].key = +∞
+			}
+			return key < t.data[j].key
+		})
+
+		switch child := t.data[i].child.(type) {
+		case *ZBTree:
+			t.PDeactivate()
+			t = child
+			err = t.PActivate(ctx)
+			if err != nil {
+				return nil, false, err
+			}
+
+		case *ZBucket:
+			t.PDeactivate()
+			return child.Get(ctx, key)
+		}
+	}
+}
+
+// Get searches Bucket by key.
+func (b *ZBucket) Get(ctx context.Context, key KEY) (_ interface{}, _ bool, err error) {
+	defer xerr.Contextf(&err, "bucket(%s): get %s", b.POid(), key)	// XXX + url?
+	err = b.PActivate(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+
+	v, ok := b.get(key)
+	b.PDeactivate()
+	return v, ok, nil
+}
+
+// get searches Bucket by key.
+//
+// no loading from database is done. The bucket must be already activated.
+func (b *ZBucket) get(key KEY) (interface{}, bool) {
+	// search i: K(i-1) < k ≤ K(i)		; K(-1) = -∞, K(len) = +∞
+	i := sort.Search(len(b.keys), func(i int) bool {
+		return key <= b.keys[i]
+	})
+
+	if i == len(b.keys) || b.keys[i] != key {
+		return nil, false // not found
+	}
+	return b.values[i], true
+}
+
+// XXX ZBucket.MinKey ?
+// XXX ZBucket.MaxKey ?
+
+
+// ---- serialization ----
+
+// from https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/BTreeTemplate.c#L1087:
+//
+// For an empty BTree (self->len == 0), None.
+//
+// For a BTree with one child (self->len == 1), and that child is a bucket,
+// and that bucket has a NULL oid, a one-tuple containing a one-tuple
+// containing the bucket's state:
+//
+//     (
+//         (
+//              child[0].__getstate__(),
+//         ),
+//     )
+//
+// Else a two-tuple.  The first element is a tuple interleaving the BTree's
+// keys and direct children, of size 2*self->len - 1 (key[0] is unused and
+// is not saved).  The second element is the firstbucket:
+//
+//     (
+//          (child[0], key[1], child[1], key[2], child[2], ...,
+//                                       key[len-1], child[len-1]),
+//          self->firstbucket
+//     )
+//
+// In the above, key[i] means self->data[i].key, and similarly for child[i].
+
+// DropState implements Stateful.
+func (t *ZBTree) DropState() {
+	t.firstbucket = nil
+	t.data = nil
+}
+
+// PySetState implements PyStateful to set btree data from pystate.
+func (bt *ZBTree) PySetState(pystate interface{}) error {
+	// empty btree
+	if _, ok := pystate.(pickle.None); ok {
+		bt.firstbucket = nil
+		bt.data = nil
+		return nil
+	}
+
+	t, ok := pystate.(pickle.Tuple)
+	if !ok || !(1 <= len(t) && len(t) <= 2) {
+		// XXX
+	}
+
+	// btree with 1 child bucket without oid
+	if len(t) == 1 {
+		bucket := &ZBucket{PyPersistent: nil /* FIXME */}
+		err := bucket.PySetState(t[0])
+		if err != nil {
+			// XXX
+		}
+
+		bt.firstbucket = bucket
+		bt.data = []zBTreeItem{{key: 0, child: bucket}}
+		return nil
+	}
+
+	// regular btree
+	t, ok = t[0].(pickle.Tuple)
+	if !(ok && len(t) % 2 == 0) {
+		// XXX
+	}
+
+	bt.firstbucket, ok = t[1].(*ZBucket)
+	if !ok {
+		// XXX
+	}
+
+	n := len(t) / 2
+	bt.data = make([]zBTreeItem, 0, n)
+	for i, idx := 0, 0; i < n; i++ {
+		key := int64(0)
+		if i > 0 {
+			// key[0] is unused and not saved
+			key, ok = t[idx].(int64)	// XXX Xint
+			if ! ok {
+				// XXX
+			}
+			idx++
+		}
+		child := t[idx]
+		idx++
+
+		switch child.(type) {
+		default:
+			// XXX
+
+		case *ZBTree:  // ok
+		case *ZBucket: // ok
+		}
+
+		bt.data = append(bt.data, zBTreeItem{key: KEY(key), child: child})
+	}
+
+	return nil
+}
+
+
+// from https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/BucketTemplate.c#L1195:
+//
+// For a mapping bucket (self->values is not NULL), a one-tuple or two-tuple.
+// The first element is a tuple interleaving keys and values, of length
+// 2 * self->len.  The second element is the next bucket, present iff next is
+// non-NULL:
+//
+//	(
+//	     (keys[0], values[0], keys[1], values[1], ...,
+//	                          keys[len-1], values[len-1]),
+//	     <self->next iff non-NULL>
+//	)
+
+// DropState implements Stateful to discard bucket state.
+func (b *ZBucket) DropState() {
+	b.next = nil
+	b.keys = nil
+	b.values = nil
+}
+
+// PySetState implements PyStateful to set bucket data from pystate.
+func (b *ZBucket) PySetState(pystate interface{}) error {
+	t, ok := pystate.(pickle.Tuple)
+	if !ok || !(1 <= len(t) && len(t) <= 2) {
+		// XXX complain
+	}
+
+	// .next present
+	if len(t) == 2 {
+		next, ok := t[1].(*ZBucket)
+		if !ok {
+			// XXX
+		}
+		b.next = next
+	}
+
+	// main part
+	t, ok = t[0].(pickle.Tuple)
+	// XXX if !ok || (len(t) % 2 != 0)
+
+	// reset arrays just in case
+	n := len(t) / 2
+	b.keys = make([]KEY, 0, n)
+	b.values = make([]interface{}, 0, n)
+
+	for i := 0; i < n; i++ {
+		xk := t[2*i]
+		v := t[2*i+1]
+
+		k, ok := xk.(int64)	// XXX use KEY	XXX -> Xint64
+		if !ok {
+			// XXX
+		}
+
+		// XXX check keys are sorted?
+		b.keys = append(b.keys, KEY(k))		// XXX cast
+		b.values = append(b.values, v)
+	}
+
+	return nil
+}
+
+
+// ---- register classes to ZODB ----
+
+func bucketNew(base *PyPersistent) IPyPersistent	{ return &ZBucket{PyPersistent: base}	}
+func btreeNew(base *PyPersistent) IPyPersistent		{ return &ZBTree{PyPersistent: base}	}
+
+func init() {
+	registerPyClass("zodb.BTree.LOBucket", bucketNew)
+	registerPyClass("zodb.BTree.LOBtree",  btreeNew)
+}

go/zodb/btree/btree_test.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+
+import (
+	//"testing"
+)
+
+
+// TODO

go/zodb/internal/weak/weak.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// based on:
+// https://groups.google.com/d/msg/golang-nuts/PYWxjT2v6ps/dL71oJk1mXEJ
+// https://play.golang.org/p/f9HY6-z8Pp
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package weak provides weak references for Go.
+//
+// XXX
+package weak
+
+import (
+	"runtime"
+	"sync"
+	"unsafe"
+)
+
+// iface is how Go runtime represents an interface.
+//
+// NOTE layout must be synchronized to Go runtime representation.
+type iface struct {
+	typ  uintptr // type
+	data uintptr // data
+}
+
+// weakRefState represents current state of an object WeakRef points to.
+type weakRefState int32
+
+const (
+	objGot      weakRefState = +1 // WeakRef.Get returned !nil
+	objLive     weakRefState =  0 // object is alive, Get did not run yet in this GC cycle
+	objReleased weakRefState = -1 // the finalizer marked object as released
+)
+
+
+// WeakRef is a weak reference.
+//
+// Create one with NewWeakRef and retrieve referenced object with Get.
+//
+// There must be no more than 1 weak reference to any object.
+// Weak references must not be attached to an object on which runtime.SetFinalizer is also used.
+// Weak references must not be copied.
+type WeakRef struct {
+	iface
+
+	// XXX try to do without mutex and only with atomics
+	mu    sync.Mutex
+	state weakRefState
+}
+
+// NewWeakRef creates new weak reference pointing to obj.
+//
+// XXX + onrelease callback?
+func NewWeakRef(obj interface{}) *WeakRef {
+	// since starting from ~ Go1.4 the GC is precise, we can save interface
+	// pointers to uintptr and that won't prevent GC from garbage
+	// collecting the object.
+	w := &WeakRef{
+		iface: *(*iface)(unsafe.Pointer(&obj)),
+		state: objLive,
+	}
+
+	var release func(interface{})
+	release = func(obj interface{}) {
+		// GC decided that the object is no longer reachable and
+		// scheduled us to run as finalizer. During the time till we
+		// actually run, WeakRef.Get might have been come to run and
+		// "rematerializing" the object for use. Check if we do not
+		// race with any Get in progress, and reschedule us to retry at
+		// next GC if we do.
+		w.mu.Lock()
+		if w.state == objGot {
+			w.state = objLive
+			runtime.SetFinalizer(obj, release)
+		} else {
+			w.state = objReleased
+		}
+		w.mu.Unlock()
+	}
+
+	runtime.SetFinalizer(obj, release)
+	return w
+}
+
+// Get returns object pointed to by this weak reference.
+//
+// If original object is still alive - it is returned.
+// If not - nil is returned.
+func (w *WeakRef) Get() (obj interface{}) {
+	w.mu.Lock()
+	if w.state != objReleased {
+		w.state = objGot
+
+		// recreate interface{} from saved words.
+		// XXX do writes as pointers so that compiler emits write barriers to notify GC?
+		i := (*iface)(unsafe.Pointer(&obj))
+		*i = w.iface
+	}
+	w.mu.Unlock()
+	return obj
+}

go/zodb/internal/weak/weak_test.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+
+import (
+	"runtime"
+	"testing"
+	"time"
+	"unsafe"
+)
+
+// verify that interface <-> iface works ok.
+func TestIface(t *testing.T) {
+	var i interface{}
+	var fi *iface
+
+	isize := unsafe.Sizeof(i)
+	fsize := unsafe.Sizeof(*fi)
+
+	if isize != fsize {
+		t.Fatalf("sizeof(interface{}) (%d)  !=   sizeof(iface) (%d)", isize, fsize)
+	}
+
+	i = 3
+	var j interface{}
+	if !(j == nil && i != j) {
+		t.Fatalf("i == j ?  (i: %#v,  j: %#v}", i, j)
+	}
+
+	fi = (*iface)(unsafe.Pointer(&i))
+	fj := (*iface)(unsafe.Pointer(&j))
+	*fj = *fi
+
+	if i != j {
+		t.Fatalf("i (%#v)  !=  j (%#v)", i, j)
+	}
+}
+
+
+func TestWeakRef(t *testing.T) {
+	type T struct { _ [8]int64 }	// large enough not to go into tinyalloc
+
+	p := new(T)
+	w := NewWeakRef(p)
+	pptr := uintptr(unsafe.Pointer(p))
+
+	assertEq := func(a, b interface{}) {
+		t.Helper()
+		if a != b {
+			t.Fatalf("not equal: %#v  !=  %#v", a, b)
+		}
+	}
+
+	// perform GC + give finalizers a chancet to run.
+	GC := func() {
+		runtime.GC()
+
+		// GC only queues finalizers, not runs them directly. Give it
+		// some time so that finalizers could have been run.
+		time.Sleep(10*time.Millisecond) // XXX hack
+	}
+
+
+	assertEq(w.state, objLive)
+	assertEq(w.Get(), p)
+	assertEq(w.state, objGot)
+	GC()
+	assertEq(w.state, objGot) // fin has not been run at all (p is live)
+	assertEq(w.Get(), p)
+	assertEq(w.state, objGot)
+
+	p = nil
+	GC()
+	assertEq(w.state, objLive) // fin ran and downgraded got -> live
+	switch p_ := w.Get().(type) {
+	default:
+		t.Fatalf("Get after objGot -> objLive: %#v", p_)
+	case *T:
+		if uintptr(unsafe.Pointer(p_)) != pptr {
+			t.Fatal("Get after objGot -> objLive: T, but ptr is not the same")
+		}
+	}
+	assertEq(w.state, objGot)
+
+	GC()
+	assertEq(w.state, objLive) // fin ran again and again downgraded got -> live
+
+	GC()
+	assertEq(w.state, objReleased) // fin ran again and released the object
+	assertEq(w.Get(), nil)
+}

go/zodb/persistent.go

+// Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+// All Rights Reserved.
+//
+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This software is subject to the provisions of the Zope Public License,
+// Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+// THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+// WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+// FOR A PARTICULAR PURPOSE
+
+package main
+// Bits that should be in ZODB		XXX -> zodb
+
+import (
+	"context"
+	"sync"
+
+	"lab.nexedi.com/kirr/go123/mem"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+// IPersistent is the interface that every in-RAM object representing any database object implements.
+//
+// It is based on IPersistent from ZODB/py:
+//
+//	https://github.com/zopefoundation/ZODB/blob/3.10.7-4-gb8d7a8567/src/persistent/interfaces.py#L22
+//
+// but is not exactly equal to it.
+type IPersistent interface {
+	PJar()    *Connection	// Connection this in-RAM object is part of.
+	POid()    zodb.Oid	// object ID in the database.
+
+	// object serial in the database as of particular Connection (PJar) view.
+	// 0 (invalid tid) if not yet loaded (XXX ok?)
+	PSerial() zodb.Tid
+
+
+	// PActivate brings object to live state.
+	//
+	// It requests to persistency layer that in-RAM object data to be present.
+	// If object state was not in RAM - it is loaded from the database.
+	//
+	// On successful return the object data is either the same as in the
+	// database or, if this data was previously modified by user of
+	// object's jar, that modified data.
+	//
+	// Object data must be accessed only after corresponding PActivate
+	// call, which marks that object's data as being in use.
+	PActivate(ctx context.Context) error
+
+	// PDeactivate indicates that corresponding PActivate caller finished access to object's data.
+	//
+	// As PActivate makes sure object's data is present in-RAM, PDeactivate
+	// tells persistency layer that this data is no longer used by
+	// corresponding PActivate caller.
+	//
+	// Note that it is valid to have several concurrent uses of object
+	// data, each protected with corresponding PActivate/PDeactivate pair:
+	// as long as there is still any PActivate not yet compensated with
+	// corresponding PDeactivate, object data will assuredly stay alive in RAM.
+	//
+	// Besides exotic cases, the caller thus must not use object's data
+	// after PDeactivate call.
+	PDeactivate()
+
+	// PInvalidate requests in-RAM object data to be discarded.
+	//
+	// Irregardless of whether in-RAM object data is the same as in the
+	// database, or it was modified, that in-RAM data must be forgotten.
+	//
+	// PInvalidate must not be called while there is any in-progress
+	// object's data use (PActivate till PDeactivate).
+	//
+	// In practice this means that:
+	//
+	//	- application must make sure to finish all objects accesses
+	//	  before transaction boundary: at transaction boundary - either
+	//	  at abort or commit, the persistency layer will sync to
+	//	  database and process invalidations.
+	//
+	//	- if PInvalidate is explicitly called by application, the
+	//	  application must care to make sure it does not access the
+	//	  object data simultaneously.
+	PInvalidate()
+
+	// PModify marks in-RAM object state as modified.
+	//
+	// It informs persistency layer that object's data was changed and so
+	// its state needs to be either saved back into database on transaction
+	// commit, or discarded on transaction abort.
+	//
+	// The object must be already activated.
+	//PModify()	TODO
+
+	// XXX probably don't need this.
+	//PState()  ObjectState	// in-RAM object state.
+
+
+	// Object must be stateful for persistency to work.
+	// XXX try to move out of IPersistent?
+	Stateful
+}
+
+// ObjectState describes state of in-RAM object.
+type ObjectState int
+
+const (
+	GHOST ObjectState = -1
+	UPTODATE          = 0
+	CHANGED           = 1
+	// no STICKY - we pin objects in RAM with PActivate
+)
+
+// Persistent is common base implementation for in-RAM representation of database objects.
+type Persistent struct {
+	jar	*Connection
+	oid	zodb.Oid
+	serial	zodb.Tid
+
+	mu	 sync.Mutex
+	state	 ObjectState
+	refcnt	 int32
+	instance IPersistent	// Persistent should be the base for the instance
+	loading  *loadState
+}
+
+func (obj *Persistent) PJar() *Connection	{ return obj.jar	}
+func (obj *Persistent) POid() zodb.Oid		{ return obj.oid	}
+func (obj *Persistent) PSerial() zodb.Tid	{ return obj.serial	}
+
+// loadState indicates object's load state/result.
+//
+// when !ready the loading is in progress.
+// when ready the loading has been completed.
+type loadState struct {
+	ready chan struct{} // closed when loading finishes
+
+	// error from the load.
+	// if there was no error, loaded data goes to object state.
+	err   error
+}
+
+// Stateful is the interface describing in-RAM object whose data state can be
+// exchanged as raw bytes.
+type Stateful interface {
+	// DropState should discard in-RAM object state.
+	// XXX move out of Stateful? -> Ghostable?
+	DropState()
+
+	// SetState should set state of the in-RAM object from raw data.
+	//
+	// state ownership is not passed to SetState, so if state needs to be
+	// retained after SetState returns it needs to be incref'ed.
+	SetState(state *mem.Buf) error
+
+	// GetState should return state of the in-RAM object as raw data.
+	//GetState() *mem.Buf	TODO
+}
+
+
+// Connection represents a view of ZODB database.
+//
+// The view is representing state of ZODB objects as of `at` transaction.
+//
+// Connection changes are private and are isolated from changes in other Connections.
+//
+// XXX Connection, and I{Py}Persistent methods that relate to it, are not safe for
+// modifications from multiple goroutines simultaneously.
+//
+// XXX ^^^ better must be safe - use case: e.g. prefetch.
+type Connection struct {
+	stor	zodb.IStorage	// underlying storage
+	at	zodb.Tid	// current view of database
+
+	// {} oid -> obj
+	//
+	// rationale:
+	//
+	// on invalidations: we need to go oid -> obj and invalidate it.
+	// -> Connection need to keep {} oid -> obj.
+	// -> we can use that {} when loading a persistent Ref twice to get to the same object.
+	//
+	// however: if Connection keeps strong link to obj, just
+	// obj.PDeactivate will not fully release obj if there are no
+	// references to it from other objects:
+	//
+	//	- deactivate will release obj state (ok)
+	//	- but there will be still reference from connection `oid -> obj` map to this object.
+	//
+	// -> we can solve it by using "weak" pointers in the map.
+	//
+	// NOTE we cannot use regular map and arbitrarily manually "gc" entries
+	// there periodically: since for an obj we don't know whether other
+	// objects are referencing it, we can't just remove obj's oid from
+	// the map - if we do so and there are other live objects that
+	// reference obj, user code can still reach obj via those
+	// references. On the other hand, if another, not yet loaded, object
+	// also references obj and gets loaded, traversing reference from
+	// that loaded object will load second copy of obj, thus breaking 1
+	// object in db <-> 1 live object invariant:
+	//
+	//	A  →  B  →  C
+	//	↓           |
+	//      D <--------- - - -> D2 (wrong)
+	//
+	// - A activate
+	// - D activate
+	// - B activate
+	// - D gc, A still keeps link on D
+	// - C activate -> it needs to get to D, but D was removed from objtab
+	//   -> new D2 is wrongly created
+	//
+	// that's why we have to depend on Go's GC to know whether there are
+	// still live references left or not. And that in turn means finalizers
+	// and thus weak references.
+	//
+	// some link on the subject:
+	// https://groups.google.com/forum/#!topic/golang-nuts/PYWxjT2v6ps
+	//
+	// NOTE2 finalizers don't run on when they are attached to an object in cycle.
+	// Hopefully we don't have cycles with ZBTree/ZBucket	XXX verify this
+	objmu  sync.Mutex
+	objtab map[zodb.Oid]*WeakRef	// oid -> WeakRef(IPersistent)
+
+	// hooks for application to influence live caching decisions.
+	cacheControl LiveCacheControl
+}
+
+// LiveCacheControl is the interface that allows applications to influence
+// Connection's decisions with respect to Connection's live cache.
+type LiveCacheControl interface {
+	// WantEvict is called when object is going to be evicted from live
+	// cache on deactivation and made ghost.
+	//
+	// If !ok the object will remain live.
+	//
+	// NOTE on invalidation invalidated objects are evicted from live cache
+	// unconditionally.
+	WantEvict(obj IPersistent) (ok bool)
+}
+
+
+// ---- activate/deactivate/invalidate ----
+
+// PActivate implements IPersistent.
+func (obj *Persistent) PActivate(ctx context.Context) (err error) {
+	obj.mu.Lock()
+	obj.refcnt++
+	doload := (obj.refcnt == 1 && obj.state == GHOST)
+	defer func() {
+		if err != nil {
+			obj.PDeactivate()
+		}
+	}()
+	if !doload {
+		// someone else is already activated/activating the object.
+		// wait for its loading to complete and we are done.
+		loading := obj.loading
+		obj.mu.Unlock()
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()	// XXX err ctx
+		case <-loading.ready:
+			return loading.err	// XXX err ctx?
+		}
+	}
+
+	// we become responsible for loading the object
+	loading := &loadState{ready: make(chan struct{})}
+	obj.loading = loading	// XXX assert before it was = nil ?
+	obj.mu.Unlock()
+
+	// do the loading outside of obj lock
+	state, serial, err := obj.jar.load(ctx, obj.oid)
+
+	// relock the object
+	obj.mu.Lock()
+
+	// XXX assert obj.loading == loading
+	// XXX assert obj.state   == GHOST
+
+	obj.serial  = serial
+
+	// try to pass loaded state to object
+	if err == nil {
+		err = obj.instance.SetState(state)	// XXX err ctx
+		state.Release()
+		if err == nil {
+			obj.state = UPTODATE
+		}
+	}
+
+	loading.err = err
+
+	obj.mu.Unlock()
+	close(loading.ready)
+
+	return err	// XXX err ctx
+}
+
+// PDeactivate implements IPersistent.
+func (obj *Persistent) PDeactivate() {
+	obj.mu.Lock()
+	defer obj.mu.Unlock()
+
+	obj.refcnt--
+	if obj.refcnt < 0 {
+		panic("deactivate: refcnt < 0")
+	}
+	if obj.refcnt > 0 {
+		return // users still left
+	}
+
+	// no users left. Let's see whether we should transition this object to ghost.
+	if obj.state >= CHANGED {
+		return
+	}
+
+	if cc := obj.jar.cacheControl; cc != nil {
+		if !cc.WantEvict(obj.instance) {
+			return
+		}
+	}
+
+	obj.serial = 0
+	obj.instance.DropState()
+	obj.state = GHOST
+	obj.loading = nil
+}
+
+// PInvalidate() implements IPersistent.
+func (obj *Persistent) PInvalidate() {
+	obj.mu.Lock()
+	defer obj.mu.Unlock()
+
+	if obj.refcnt != 0 {
+		// object is currently in use
+		panic("invalidate: refcnt != 0")
+	}
+
+	obj.serial = 0
+	obj.instance.DropState()
+	obj.state = GHOST
+	obj.loading = nil
+}
+
+
+// ----------------------------------------
+
+// XXX Connection.{Get,get} without py dependency?
+// but then how to create a ghost of correct class? -> reflect.Type?
+
+// load loads object specified by oid.
+//
+// XXX must be called ... (XXX e.g. outside transaction boundary) so that there is no race on .at .
+func (conn *Connection) load(ctx context.Context, oid zodb.Oid) (_ *mem.Buf, serial zodb.Tid, _ error) {
+	return conn.stor.Load(ctx, zodb.Xid{Oid: oid, At: conn.at})
+}

go/zodb/zodbpy.go

+// Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+// All Rights Reserved.
+//
+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This software is subject to the provisions of the Zope Public License,
+// Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+// THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+// WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+// FOR A PARTICULAR PURPOSE
+
+package main
+// Bits that should be in ZODB		XXX -> zodb
+
+import (
+	"context"
+	"fmt"
+
+	"lab.nexedi.com/kirr/go123/mem"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	pickle "github.com/kisielk/og-rek"
+)
+
+// IPyPersistent is the interface that every in-RAM object representing Python ZODB object implements.
+type IPyPersistent interface {
+	IPersistent
+
+	PyClass() pickle.Class // python class of this object
+//	PyState() interface{}  // object state. python passes this to pyclass.__new__().__setstate__()
+
+	// IPyPersistent must be stateful for persistency to work
+	// XXX try to move out of IPyPersistent? Rationale: we do not want e.g. PySetState to
+	// be available to user who holds IPyPersistent interface: it is confusing to have
+	// both PActivate and PySetState at the same time.
+	PyStateful
+}
+
+// PyPersistent is common base implementation for in-RAM representation of ZODB Python objects.
+type PyPersistent struct {
+	Persistent
+	pyclass pickle.Class
+}
+
+func (pyobj *PyPersistent) PyClass() pickle.Class	{ return pyobj.pyclass	}
+//func (pyobj *PyPersistent) PyState() interface{}	{ return pyobj.pystate	}
+
+// PyStateful is the interface describing in-RAM object whose data state can be
+// exchanged as Python data.
+type PyStateful interface {
+	// PySetState should set state of the in-RAM object from Python data.
+	// Analog of __setstate__() in Python.
+	PySetState(pystate interface{}) error
+
+	// PyGetState should return state of the in-RAM object as Python data.
+	// Analog of __getstate__() in Python.
+	//PyGetState() interface{}	TODO
+}
+
+// ---- PyPersistent <-> Persistent state exchange ----
+
+// pyinstance returns .instance upcasted to IPyPersistent.
+//
+// this should be always safe because we always create pyObjects via
+// newGhost which passes IPyPersistent as instance to IPersistent.
+func (pyobj *PyPersistent) pyinstance() IPyPersistent {
+	return pyobj.instance.(IPyPersistent)
+}
+
+func (pyobj *PyPersistent) SetState(state *mem.Buf) error {
+	pyclass, pystate, err := zodb.PyData(state.Data).Decode()
+	if err != nil {
+		return err	// XXX err ctx
+	}
+
+	if pyclass != pyobj.pyclass {
+		// complain that pyclass changed
+		// (both ref and object data use pyclass so it indeed can be different)
+		return &wrongClassError{want: pyobj.pyclass, have: pyclass} // XXX + err ctx
+	}
+
+	return pyobj.pyinstance().PySetState(pystate)	// XXX err ctx = ok?
+}
+
+// TODO PyPersistent.GetState
+
+// ---- pyclass -> new ghost ----
+
+// function representing new of a class.
+type pyClassNewFunc func(base *PyPersistent) IPyPersistent
+
+// path(pyclass) -> new(pyobj)
+var pyClassTab = make(map[string]pyClassNewFunc)
+
+// registerPyClass registers python class to be transformed to Go instance
+// created via classNew.
+//
+// must be called from global init().
+func registerPyClass(pyClassPath string, classNew pyClassNewFunc) {
+	pyClassTab[pyClassPath] = classNew
+	// XXX + register so that zodb.PyData decode handles pyClassPath
+}
+
+// newGhost creates new ghost object corresponding to pyclass and oid.
+func (conn *Connection) newGhost(pyclass pickle.Class, oid zodb.Oid) IPyPersistent {
+	pyobj := &PyPersistent{
+		Persistent:  Persistent{jar: conn, oid: oid, serial: 0, state: GHOST},
+		pyclass: pyclass,
+	}
+
+	// switch on pyclass and transform e.g. "zodb.BTree.Bucket" -> *ZBucket
+	classNew := pyClassTab[pyclass.Module + "." + pyclass.Name]
+	var instance IPyPersistent
+	if classNew != nil {
+		instance = classNew(pyobj)
+	} else {
+		instance = &dummyPyInstance{PyPersistent: pyobj}
+	}
+
+	pyobj.instance = instance
+	return instance
+}
+
+// dummyPyInstance is used for python classes that were not registered.
+type dummyPyInstance struct {
+	*PyPersistent
+	pystate interface{}
+}
+
+func (d *dummyPyInstance) DropState() {
+	d.pystate = nil
+}
+
+func (d *dummyPyInstance) PySetState(pystate interface{}) error	{
+	d.pystate = pystate
+	return nil
+}
+
+
+// ----------------------------------------
+
+
+// Get returns in-RAM object corresponding to specified ZODB object according to current database view.
+//
+// If there is already in-RAM object that corresponds to oid, that in-RAM object is returned.
+// Otherwise new in-RAM object is created and filled with object's class loaded from the database.
+//
+// The scope of the object returned is the Connection.	XXX ok?
+//
+// The object's data is not neccessarily loaded after Get returns. Use
+// PActivate to make sure the object ifs fully loaded.
+func (conn *Connection) Get(ctx context.Context, oid zodb.Oid) (IPyPersistent, error) {
+	conn.objmu.Lock()		// XXX -> rlock
+	wobj := conn.objtab[oid]
+	var xobj interface{}
+	if wobj != nil {
+		xobj = wobj.Get()
+	}
+	conn.objmu.Unlock()
+
+	// object was already there in objtab.
+	if xobj != nil {
+		return xobj.(IPyPersistent), nil
+	}
+
+	// object is not there in objtab - raw load it, get its class -> get(pyclass, oid)
+	pyclass, pystate, serial, err := conn.loadpy(ctx, oid)
+	if err != nil {
+		return nil, err		// XXX errctx
+	}
+
+	obj, err := conn.get(pyclass, oid)
+	if err != nil {
+		return nil, err
+	}
+
+	// XXX we are dropping just loaded pystate. Usually Get should be used
+	// to only load root object, so maybe that is ok.
+	//
+	// TODO -> use (pystate, serial) to activate.
+	_, _ = pystate, serial
+	return obj, nil
+}
+
+// wrongClassError is the error cause returned when python object's class is not what was expected.
+type wrongClassError struct {
+	want, have pickle.Class
+}
+
+func (e *wrongClassError) Error() string {
+	return fmt.Sprintf("wrong class: want %q; have %q", e.want, e.have)
+}
+
+// get returns in-RAM object corresponding to specified ZODB object according to current database view.
+//
+// If there is already in-RAM object that corresponds to oid, that in-RAM object is returned.
+// Otherwise new in-RAM object is created according to specified class.
+//
+// The object's data is not neccessarily loaded after get returns. Use
+// PActivate to make sure the object is fully loaded.
+//
+// XXX object scope.
+//
+// Use-case: in ZODB references are (pyclass, oid), so new ghost is created
+// without further loading anything.
+func (conn *Connection) get(pyclass pickle.Class, oid zodb.Oid) (IPyPersistent, error) {
+	conn.objmu.Lock()		// XXX -> rlock
+	wobj := conn.objtab[oid]
+	var pyobj IPyPersistent
+	checkClass := false
+	if wobj != nil {
+		if xobj := wobj.Get(); xobj != nil {
+			pyobj = xobj.(IPyPersistent)
+		}
+	}
+	if pyobj == nil {
+		pyobj = conn.newGhost(pyclass, oid)
+		conn.objtab[oid] = NewWeakRef(pyobj)
+	} else {
+		checkClass = true
+	}
+	conn.objmu.Unlock()
+
+	if checkClass {
+		if cls := pyobj.PyClass(); pyclass != cls {
+			return nil, &zodb.OpError{
+				URL:  conn.stor.URL(),
+				Op:   fmt.Sprintf("@%s: get", conn.at), // XXX abuse
+				Args: oid,
+				Err:  &wrongClassError{pyclass, cls},
+			}
+		}
+	}
+
+	return pyobj, nil
+}
+
+// loadpy loads object specified by oid and decodes it as a ZODB Python object.
+//
+// loadpy does not create any in-RAM object associated with Connection.
+// It only returns decoded database data.
+func (conn *Connection) loadpy(ctx context.Context, oid zodb.Oid) (pyclass pickle.Class, pystate interface{}, serial zodb.Tid, _ error) {
+	buf, serial, err := conn.stor.Load(ctx, zodb.Xid{Oid: oid, At: conn.at})
+	if err != nil {
+		return pickle.Class{}, nil, 0, err
+	}
+
+	defer buf.Release()
+
+	pyclass, pystate, err = zodb.PyData(buf.Data).Decode()
+	if err != nil {
+		return pickle.Class{}, nil, 0, err	// XXX err ctx
+	}
+
+	return pyclass, pystate, serial, nil
+}