go/zodb/internal/weak: New package to handle weak references

We will need weak references to handle {} oid -> obj inside zodb.Connection .

In Go world they often say that weak references are not needed at all.
Please see however the next patch for detailed rationale for why weak
references (finalizers and cooperation from Go's GC in other words) are
_required_ in that case.

go/zodb/internal/weak/weak.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// based on:
+// https://groups.google.com/d/msg/golang-nuts/PYWxjT2v6ps/dL71oJk1mXEJ
+// https://play.golang.org/p/f9HY6-z8Pp
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package weak provides weak references for Go.
+package weak
+
+import (
+	"runtime"
+	"sync"
+	"unsafe"
+)
+
+// iface is how Go runtime represents an interface.
+//
+// NOTE layout must be synchronized to Go runtime representation.
+type iface struct {
+	typ  uintptr // type
+	data uintptr // data
+}
+
+// weakRefState represents current state of an object Ref points to.
+type weakRefState int32
+
+const (
+	objGot      weakRefState = +1 // Ref.Get returned !nil
+	objLive     weakRefState =  0 // object is alive, Get did not run yet in this GC cycle
+	objReleased weakRefState = -1 // the finalizer marked object as released
+)
+
+
+// Ref is a weak reference.
+//
+// Create one with NewRef and retrieve referenced object with Get.
+//
+// There must be no more than 1 weak reference to any object.
+// Weak references must not be attached to an object on which runtime.SetFinalizer is also used.
+// Weak references must not be copied.
+type Ref struct {
+	iface
+
+	// XXX try to do without mutex and only with atomics
+	mu    sync.Mutex
+	state weakRefState
+}
+
+// NewRef creates new weak reference pointing to obj.
+//
+// TODO + onrelease callback?
+func NewRef(obj interface{}) *Ref {
+	// since starting from ~ Go1.4 the GC is precise, we can save interface
+	// pointers to uintptr and that won't prevent GC from garbage
+	// collecting the object.
+	w := &Ref{
+		iface: *(*iface)(unsafe.Pointer(&obj)),
+		state: objLive,
+	}
+
+	var release func(interface{})
+	release = func(obj interface{}) {
+		// GC decided that the object is no longer reachable and
+		// scheduled us to run as finalizer. During the time till we
+		// actually run, Ref.Get might have been come to run and
+		// "rematerializing" the object for use. Check if we do not
+		// race with any Get in progress, and reschedule us to retry at
+		// next GC if we do.
+		w.mu.Lock()
+		if w.state == objGot {
+			w.state = objLive
+			runtime.SetFinalizer(obj, release)
+		} else {
+			w.state = objReleased
+		}
+		w.mu.Unlock()
+	}
+
+	runtime.SetFinalizer(obj, release)
+	return w
+}
+
+// Get returns object pointed to by this weak reference.
+//
+// If original object is still alive - it is returned.
+// If not - nil is returned.
+func (w *Ref) Get() (obj interface{}) {
+	w.mu.Lock()
+	if w.state != objReleased {
+		w.state = objGot
+
+		// recreate interface{} from saved words.
+		// XXX do writes as pointers so that compiler emits write barriers to notify GC?
+		i := (*iface)(unsafe.Pointer(&obj))
+		*i = w.iface
+	}
+	w.mu.Unlock()
+	return obj
+}

go/zodb/internal/weak/weak_test.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package weak
+
+import (
+	"runtime"
+	"testing"
+	"time"
+	"unsafe"
+)
+
+// verify that interface <-> iface works ok.
+func TestIface(t *testing.T) {
+	var i interface{}
+	var fi *iface
+
+	isize := unsafe.Sizeof(i)
+	fsize := unsafe.Sizeof(*fi)
+
+	if isize != fsize {
+		t.Fatalf("sizeof(interface{}) (%d)  !=   sizeof(iface) (%d)", isize, fsize)
+	}
+
+	i = 3
+	var j interface{}
+	if !(j == nil && i != j) {
+		t.Fatalf("i == j ?  (i: %#v,  j: %#v}", i, j)
+	}
+
+	fi = (*iface)(unsafe.Pointer(&i))
+	fj := (*iface)(unsafe.Pointer(&j))
+	*fj = *fi
+
+	if i != j {
+		t.Fatalf("i (%#v)  !=  j (%#v)", i, j)
+	}
+}
+
+func TestWeakRef(t *testing.T) {
+	type T struct{ _ [8]int64 } // large enough not to go into tinyalloc
+
+	p := new(T)
+	w := NewRef(p)
+	pptr := uintptr(unsafe.Pointer(p))
+
+	assertEq := func(a, b interface{}) {
+		t.Helper()
+		if a != b {
+			t.Fatalf("not equal: %#v  !=  %#v", a, b)
+		}
+	}
+
+	// perform GC + give finalizers a chance to run.
+	GC := func() {
+		runtime.GC()
+
+		// GC only queues finalizers, not runs them directly. Give it
+		// some time so that finalizers could have been run.
+		time.Sleep(10 * time.Millisecond) // XXX hack
+	}
+
+	assertEq(w.state, objLive)
+	assertEq(w.Get(), p)
+	assertEq(w.state, objGot)
+	GC()
+	assertEq(w.state, objGot) // fin has not been run at all (p is live)
+	assertEq(w.Get(), p)
+	assertEq(w.state, objGot)
+
+	p = nil
+	GC()
+	assertEq(w.state, objLive) // fin ran and downgraded got -> live
+	switch p_ := w.Get().(type) {
+	default:
+		t.Fatalf("Get after objGot -> objLive: %#v", p_)
+	case *T:
+		if uintptr(unsafe.Pointer(p_)) != pptr {
+			t.Fatal("Get after objGot -> objLive: T, but ptr is not the same")
+		}
+	}
+	assertEq(w.state, objGot)
+
+	GC()
+	assertEq(w.state, objLive) // fin ran again and again downgraded got -> live
+
+	GC()
+	assertEq(w.state, objReleased) // fin ran again and released the object
+	assertEq(w.Get(), nil)
+}