bigfile/zodb: Note that even LivePersistent goes to GHOST state on invalidation

LivePersistent can go to ghost state, because invalidation cannot be
ignored, i.e. they indicate the object has been changed externally.

This does not break our logic for ZBigFile and ZBigArray as
invalidations can happen only at transaction boundary, so during the
course of transaction those classes are guaranteed to stay uptodate and
thus not loose ._v_file and ._v_fileh (which is the reason they inherit
from LivePersistent).

it is ok to loose ._v_file and ._v_fileh at transaction boundary and
become ghost - those objects will be recreated upon going back uptodate
and will stay alive again during the whole transaction window.

We care only not to loose e.g. ._v_fileh inside transaction, because
loosing that data manager and thus data it manages inside transaction
can break synchronization logic and forget changed-through-mmap data.

bigfile/file_zodb.py

@@ -162,6 +162,16 @@ class _ZBigFile(BigFile):
 
 
 # Persistent that never goes to ghost state, if it was ever uptodate.
+#
+# NOTE
+#
+# On invalidation LivePersistent still goes to ghost state, because
+# invalidation cannot be ignored, i.e. they indicate the object has been
+# changed externally.
+#
+# Invalidation can happen only at transaction boundary, so during the course of
+# transaction LivePersistent is guaranteed to stay uptodate.
+#
 # XXX move to common place?
 class LivePersistent(Persistent):
     # don't allow us to go to ghost
@@ -175,6 +185,8 @@ class LivePersistent(Persistent):
         return
 
 
+    # NOTE _p_invalidate() is triggered on invalidations. We do not override it.
+
 
 # NOTE Can't inherit from Persistent and BigFile at the same time - both are C
 # types and their layout conflict. Persistent must be here for object to be

bigfile/tests/test_filezodb.py

@@ -19,7 +19,7 @@ from wendelin.bigfile.file_zodb import LivePersistent, ZBigFile
 from wendelin.bigfile import ram_reclaim
 from wendelin.lib.zodb import dbclose
 from wendelin.lib.testing import getTestDB
-from persistent import UPTODATE, GHOST
+from persistent import UPTODATE, GHOST, CHANGED
 import transaction
 from transaction import TransactionManager
 from numpy import ndarray, array_equal, uint8, zeros
@@ -121,6 +121,75 @@ def test_livepersistent():
 
     # ok
     dbclose(root)
+    del root, db, lp
+
+
+    # demo that upon cache invalidation LivePersistent can go back to ghost
+    root = dbopen()
+    conn = root._p_jar
+    db   = conn.db()
+    conn.close()
+    del root, conn
+
+    tm1 = TransactionManager()
+    tm2 = TransactionManager()
+
+    conn1 = db.open(transaction_manager=tm1)
+    root1 = conn1.root()
+    lp1 = root1['live']
+
+    conn2 = db.open(transaction_manager=tm2)
+    root2 = conn2.root()
+    lp2 = root2['live']
+
+    # 2 connections are setup running in parallel with initial obj state as ghost
+    assert lp1._p_jar   is conn1
+    assert lp2._p_jar   is conn2
+
+    assert lp1._p_state is GHOST
+    assert lp2._p_state is GHOST
+
+    # conn1: modify  ghost -> changed
+    lp1.attr = 1
+
+    assert lp1._p_state is CHANGED
+    assert lp2._p_state is GHOST
+
+    # conn2: read    ghost -> uptodate
+    assert getattr(lp1, 'attr', None) == 1
+    assert getattr(lp2, 'attr', None) is None
+
+    assert lp1._p_state is CHANGED
+    assert lp2._p_state is UPTODATE
+
+    # conn1: commit  changed -> uptodate; conn2 untouched
+    tm1.commit()
+
+    assert lp1._p_state is UPTODATE
+    assert lp2._p_state is UPTODATE
+
+    assert getattr(lp1, 'attr', None) == 1
+    assert getattr(lp2, 'attr', None) is None
+
+    # conn2: commit  (nothing changed - just transaction boundary)
+    #                 uptodate -> ghost (invalidation)
+    tm2.commit()
+
+    assert lp1._p_state is UPTODATE
+    assert lp2._p_state is GHOST
+
+    assert getattr(lp1, 'attr', None) == 1
+
+    # conn2: after reading, the state is again uptodate + changes from conn1 are here
+    a = getattr(lp2, 'attr', None)
+    assert lp2._p_state is UPTODATE
+    assert a == 1
+
+
+    conn2.close()
+    del conn2, root2
+    dbclose(root1)
+
 
 
 # i'th memory block as u8 ndarray