go/zodb: Implement Connection

Connection represents an application-level view of a ZODB database.
It has groups of in-RAM application-level objects associated with it.
The objects are isolated from both changes in further database
transactions and from changes to in-RAM objects in other connections.

Connection, as objects group manager, is responsible for handling
object -> object database references. For this to work it keeps

	{} oid -> obj

dict and uses it to find already loaded object when another object
persistently references particular oid. Since it related pydata handling
of persistent references is correspondingly implemented in this patch.

The dict must keep weak references on objects. The following text
explains the rationale:

	if Connection keeps strong link to obj, just
	obj.PDeactivate will not fully release obj if there are no
	references to it from other objects:

	     - deactivate will release obj state (ok)
	     - but there will be still reference from connection `oid -> obj` map to this object,
	       which means the object won't be garbage-collected.

	-> we can solve it by using "weak" pointers in the map.

	NOTE we cannot use regular map and arbitrarily manually "gc" entries
	there periodically: since for an obj we don't know whether other
	objects are referencing it, we can't just remove obj's oid from
	the map - if we do so and there are other live objects that
	reference obj, user code can still reach obj via those
	references. On the other hand, if another, not yet loaded, object
	also references obj and gets loaded, traversing reference from
	that loaded object will load second copy of obj, thus breaking 1
	object in db <-> 1 live object invariant:

	     A  →  B  →  C
	     ↓           |
	     D <--------- - - -> D2 (wrong)

	- A activate
	- D activate
	- B activate
	- D gc, A still keeps link on D
	- C activate -> it needs to get to D, but D was removed from objtab
	  -> new D2 is wrongly created

	that's why we have to depend on Go's GC to know whether there are
	still live references left or not. And that in turn means finalizers
	and thus weak references.

	some link on the subject:
	https://groups.google.com/forum/#!topic/golang-nuts/PYWxjT2v6ps

go/zodb/connection.go

@@ -18,21 +18,85 @@
 // See https://www.nexedi.com/licensing for rationale and options.
 
 package zodb
-// database connection.
+// application-level database connection.
 
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	"lab.nexedi.com/kirr/go123/mem"
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/zodb/internal/weak"
 )
 
-// Connection represents a view of a ZODB database.
+// Connection represents application-level view of a ZODB database.
+//
+// The view is represented by IPersistent objects associated with the connection.
+// Connection changes are private and are isolated from changes in other Connections.
+// Connection's view corresponds to particular database state and is thus
+// isolated from further database transactions.
+//
+// Connection is safe to access from multiple goroutines simultaneously.
 type Connection struct {
 	stor IStorage                // underlying storage
 	at   Tid                     // current view of database; stable inside a transaction.
+
+	// {} oid -> obj
+	//
+	// rationale:
+	//
+	// on invalidations: we need to go oid -> obj and invalidate it.
+	// -> Connection need to keep {} oid -> obj.
+	// -> we can use that {} when loading a persistent Ref twice to get to the same object.
+	//
+	// however: if Connection keeps strong link to obj, just
+	// obj.PDeactivate will not fully release obj if there are no
+	// references to it from other objects:
+	//
+	//	- deactivate will release obj state (ok)
+	//	- but there will be still reference from connection `oid -> obj` map to this object,
+	//	  which means the object won't be garbage-collected.
+	//
+	// -> we can solve it by using "weak" pointers in the map.
+	//
+	// NOTE we cannot use regular map and arbitrarily manually "gc" entries
+	// there periodically: since for an obj we don't know whether other
+	// objects are referencing it, we can't just remove obj's oid from
+	// the map - if we do so and there are other live objects that
+	// reference obj, user code can still reach obj via those
+	// references. On the other hand, if another, not yet loaded, object
+	// also references obj and gets loaded, traversing reference from
+	// that loaded object will load second copy of obj, thus breaking 1
+	// object in db <-> 1 live object invariant:
+	//
+	//	A  →  B  →  C
+	//	↓           |
+	//      D <--------- - - -> D2 (wrong)
+	//
+	// - A activate
+	// - D activate
+	// - B activate
+	// - D gc, A still keeps link on D
+	// - C activate -> it needs to get to D, but D was removed from objtab
+	//   -> new D2 is wrongly created
+	//
+	// that's why we have to depend on Go's GC to know whether there are
+	// still live references left or not. And that in turn means finalizers
+	// and thus weak references.
+	//
+	// some link on the subject:
+	// https://groups.google.com/forum/#!topic/golang-nuts/PYWxjT2v6ps
+	//
+	// NOTE2 finalizers don't run on when they are attached to an object in cycle.
+	// Hopefully we don't have cycles with BTree/Bucket.
+	objmu  sync.Mutex
+	objtab map[Oid]*weak.Ref // oid -> weak.Ref(IPersistent)
 }
 
+// ----------------------------------------
+
+
 // wrongClassError is the error cause returned when ZODB object's class is not what was expected.
 type wrongClassError struct {
 	want, have string
@@ -42,6 +106,85 @@ func (e *wrongClassError) Error() string {
 	return fmt.Sprintf("wrong class: want %q; have %q", e.want, e.have)
 }
 
+// get is like Get, but used when we already know object class.
+//
+// Use-case: in ZODB references are (pyclass, oid), so new ghost is created
+// without further loading anything.
+func (conn *Connection) get(class string, oid Oid) (IPersistent, error) {
+	conn.objmu.Lock() // XXX -> rlock?
+	wobj := conn.objtab[oid]
+	var obj IPersistent
+	checkClass := false
+	if wobj != nil {
+		if xobj := wobj.Get(); xobj != nil {
+			obj = xobj.(IPersistent)
+		}
+	}
+	if obj == nil {
+		obj = newGhost(class, oid, conn)
+		conn.objtab[oid] = weak.NewRef(obj)
+	} else {
+		checkClass = true
+	}
+	conn.objmu.Unlock()
+
+	if checkClass {
+		if cls := zclassOf(obj); class != cls {
+			var err error = &wrongClassError{class, cls}
+			xerr.Contextf(&err, "get %s", Xid{conn.at, oid})
+			return nil, err
+		}
+	}
+
+	return obj, nil
+}
+
+// Get returns in-RAM object corresponding to specified ZODB object according to current database view.
+//
+// If there is already in-RAM object that corresponds to oid, that in-RAM object is returned.
+// Otherwise new in-RAM object is created and filled with object's class loaded from the database.
+//
+// The scope of the object returned is the Connection.
+//
+// The object's data is not necessarily loaded after Get returns. Use
+// PActivate to make sure the object is fully loaded.
+func (conn *Connection) Get(ctx context.Context, oid Oid) (_ IPersistent, err error) {
+	defer xerr.Contextf(&err, "Get %s", oid)
+
+	conn.objmu.Lock() // XXX -> rlock?
+	wobj := conn.objtab[oid]
+	var xobj interface{}
+	if wobj != nil {
+		xobj = wobj.Get()
+	}
+	conn.objmu.Unlock()
+
+	// object was already there in objtab.
+	if xobj != nil {
+		return xobj.(IPersistent), nil
+	}
+
+	// object is not there in objtab - raw load it, get its class -> get(pyclass, oid)
+	// XXX "py always" hardcoded
+	class, pystate, serial, err := conn.loadpy(ctx, oid)
+	if err != nil {
+		xerr.Contextf(&err, "Get %s", Xid{conn.at, oid})
+		return nil, err
+	}
+
+	obj, err := conn.get(class, oid)
+	if err != nil {
+		return nil, err
+	}
+
+	// XXX we are dropping just loaded pystate. Usually Get should be used
+	// to only load root object, so maybe that is ok.
+	//
+	// TODO -> use (pystate, serial) to activate.
+	_, _ = pystate, serial
+	return obj, nil
+}
+
 // load loads object specified by oid.
 func (conn *Connection) load(ctx context.Context, oid Oid) (_ *mem.Buf, serial Tid, _ error) {
 	return conn.stor.Load(ctx, Xid{Oid: oid, At: conn.at})

go/zodb/persistent.go

@@ -179,7 +179,7 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
 			xerr.Context(&err, "setstate")
 
 		case PyStateful:
-			err = pySetState(istate, obj.zclass.class, state)
+			err = pySetState(istate, obj.zclass.class, state, obj.jar)
 			xerr.Context(&err, "pysetstate")
 
 		default:

go/zodb/pydata.go

@@ -22,6 +22,7 @@ package zodb
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 
 	pickle "github.com/kisielk/og-rek"
@@ -58,13 +59,17 @@ func (d PyData) ClassName() string {
 	return pyclassPath(klass)
 }
 
+// TODO PyData.referencesf
+
 // decode decodes raw ZODB python data into Python class and state.
-func (d PyData) decode() (pyclass pickle.Class, pystate interface{}, err error) {
+//
+// jar is used to resolve persistent references.
+func (d PyData) decode(jar *Connection) (pyclass pickle.Class, pystate interface{}, err error) {
 	defer xerr.Context(&err, "pydata: decode")
 
 	p := pickle.NewDecoderWithConfig(
 		bytes.NewReader([]byte(d)),
-		&pickle.DecoderConfig{/* TODO: handle persistent references */},
+		&pickle.DecoderConfig{PersistentLoad: jar.loadref},
 	)
 
 	xklass, err := p.Decode()
@@ -85,6 +90,38 @@ func (d PyData) decode() (pyclass pickle.Class, pystate interface{}, err error)
 	return klass, state, nil
 }
 
+// loadref loads persistent references resolving them through jar.
+//
+// https://github.com/zopefoundation/ZODB/blob/a89485c1/src/ZODB/serialize.py#L80
+func (jar *Connection) loadref(ref pickle.Ref) (_ interface{}, err error) {
+	defer xerr.Context(&err, "loadref")
+
+	// ref = (oid, class)
+	// TODO add support for ref formats besides (oid, class)
+
+	t, ok := ref.Pid.(pickle.Tuple)
+	if !ok {
+		return nil, fmt.Errorf("expect (); got %T", ref.Pid)
+	}
+	if len(t) != 2 {
+		return nil, fmt.Errorf("expect (oid, class); got [%d]()", len(t))
+	}
+
+	oid, err := xoid(t[0])
+	if err != nil {
+		return nil, err
+	}
+
+	pyclass, err := xpyclass(t[1])
+	if err != nil {
+		return nil, err
+	}
+
+	class := pyclassPath(pyclass)
+
+	return jar.get(class, oid)
+}
+
 // xpyclass verifies and extracts py class from unpickled value.
 //
 // it normalizes py class that has just been decoded from a serialized ZODB
@@ -127,6 +164,23 @@ func xpyclass(xklass interface{}) (_ pickle.Class, err error) {
 	return pickle.Class{}, fmt.Errorf("expect type; got %T", xklass)
 }
 
+// xoid verifies and extracts oid from unpickled value.
+//
+// TODO +zobdpickle.binary support
+func xoid(x interface{}) (_ Oid, err error) {
+	defer xerr.Context(&err, "oid")
+
+	s, ok := x.(string)
+	if !ok {
+		return InvalidOid, fmt.Errorf("expect str; got %T", x)
+	}
+	if len(s) != 8 {
+		return InvalidOid, fmt.Errorf("expect [8]str; got [%d]str", len(s))
+	}
+
+	return Oid(binary.BigEndian.Uint64([]byte(s))), nil
+}
+
 // pyclassPath returns full path for a python class.
 //
 // for example class "ABC" in module "wendelin.lib" has its full path as "wendelin.lib.ABC".

go/zodb/zodb.go

@@ -95,10 +95,34 @@
 // For MyObject to implement IPersistent it must embed Persistent type.
 // MyObject also has to register itself to persistency machinery with RegisterClass.
 //
-// Object activation protocol is safe to access from
+// In-RAM application objects are handled in groups.
+// A group corresponds to particular
+// view of the database (at) and has isolation guarantee from further database
+// transactions, and from in-progress changes to in-RAM objects in other
+// groups.
+//
+// If object₁ references object₂ in the database, the database reference will
+// be represented with corresponding reference between in-RAM application
+// objects. If there are multiple database references to one object, it will be
+// represented by the same number of references to only one in-RAM application object.
+// An in-RAM application object can have reference to another in-RAM
+// application object only from the same group(+).
+// Reference cycles are also allowed. In general objects graph in the database
+// is isomorphly mapped to application objects graph in RAM.
+//
+// A particular view of the database together with corresponding group of
+// application objects isolated for modifications is represented by Connection.
+// Connection is also sometimes called a "jar" in ZODB terminology.
+//
+//
+// Both Connection and object activation protocol is safe to access from
 // multiple goroutines simultaneously.
 //
 //
+// --------
+//
+// (+) if both objects are from the same database.
+//
 // Python data
 //
 // To maintain database data compatibility with ZODB/py, ZODB/go provides

go/zodb/zodbpy.go

@@ -21,6 +21,8 @@ package zodb
 // Support for python objects/data in ZODB.
 
 import (
+	"context"
+
 	"lab.nexedi.com/kirr/go123/mem"
 )
 
@@ -44,8 +46,9 @@ type PyStateful interface {
 // state on PyStateful obj.
 //
 // It is an error if decoded state has python class not as specified.
-func pySetState(obj PyStateful, objClass string, state *mem.Buf) error {
-	pyclass, pystate, err := PyData(state.Data).decode()
+// jar is used to resolve persistent references.
+func pySetState(obj PyStateful, objClass string, state *mem.Buf, jar *Connection) error {
+	pyclass, pystate, err := PyData(state.Data).decode(jar)
 	if err != nil {
 		return err
 	}
@@ -62,3 +65,32 @@ func pySetState(obj PyStateful, objClass string, state *mem.Buf) error {
 }
 
 // TODO pyGetState
+
+
+
+// loadpy loads object specified by oid and decodes it as a ZODB Python object.
+//
+// loadpy does not create any in-RAM object associated with Connection.
+// It only returns decoded database data.
+func (conn *Connection) loadpy(ctx context.Context, oid Oid) (class string, pystate interface{}, serial Tid, _ error) {
+	xid := Xid{Oid: oid, At: conn.at}
+	buf, serial, err := conn.stor.Load(ctx, xid)
+	if err != nil {
+		return "", nil, 0, err
+	}
+
+	defer buf.Release()
+
+	pyclass, pystate, err := PyData(buf.Data).decode(conn)
+	if err != nil {
+		err = &OpError{
+			URL:  conn.stor.URL(),
+			Op:   "loadpy",
+			Args: xid,
+			Err:  err,
+		}
+		return "", nil, 0, err
+	}
+
+	return pyclassPath(pyclass), pystate, serial, nil
+}