WIP: Switch from ZODB3 (3.10.7) to ZODB (4.4.3)

component/egg-patch/ZODB-4.4.3.patch

+From b74eef767952daf6b972511e9bba389be00955be Mon Sep 17 00:00:00 2001
+From: Julien Muchembled <jm@nexedi.com>
+Date: Sat, 20 Aug 2016 22:51:26 +0200
+Subject: [PATCH] Call _p_resolveConflict() even if a conflicting change
+ doesn't change the state
+
+This reverts to the behaviour of 3.10.3 and older.
+
+diff --git a/src/ZODB/ConflictResolution.py b/src/ZODB/ConflictResolution.py
+index 2caa87b..be7ef99 100644
+--- a/src/ZODB/ConflictResolution.py
++++ b/src/ZODB/ConflictResolution.py
+@@ -271,13 +271,6 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
+         if not committedData:
+             committedData  = self.loadSerial(oid, committedSerial)
+ 
+-        if newpickle == oldData:
+-            # old -> new diff is empty, so merge is trivial
+-            return committedData
+-        if committedData == oldData:
+-            # old -> committed diff is empty, so merge is trivial
+-            return newpickle
+-
+         newstate = unpickler.load()
+         old       = state(self, oid, oldSerial, prfactory, oldData)
+         committed = state(self, oid, committedSerial, prfactory, committedData)
+diff --git a/src/ZODB/tests/testconflictresolution.py b/src/ZODB/tests/testconflictresolution.py
+index d863790..895eab7 100644
+--- a/src/ZODB/tests/testconflictresolution.py
++++ b/src/ZODB/tests/testconflictresolution.py
+@@ -40,6 +40,15 @@ def tearDown(test):
+ class ResolveableWhenStateDoesNotChange(persistent.Persistent):
+ 
+     def _p_resolveConflict(self, old, committed, new):
++        if new == old:
++            # old -> new diff is empty, so merge is trivial
++            committed['resolved'] = 'committed'
++            return committed
++        elif committed == old:
++            # old -> committed diff is empty, so merge is trivial
++            new['resolved'] = 'new'
++            return new
++        # 3-way merge
+         raise ZODB.POSException.ConflictError
+ 
+ class Unresolvable(persistent.Persistent):
+@@ -47,8 +56,17 @@ class Unresolvable(persistent.Persistent):
+ 
+ def succeed_with_resolution_when_state_is_unchanged():
+     """
+-    If a conflicting change doesn't change the state, then don't even
+-    bother calling _p_resolveConflict
++    If a conflicting change doesn't change the state, then we must still call
++    _p_resolveConflict, even if in most cases the result would be either
++    committed or new (as shown above in ResolveableWhenStateDoesNotChange).
++    One use case is to implement an "asynchronous" cache:
++    - Initially, a cache value is not filled (e.g. None is used to describe
++      this state).
++    - A transaction fills the cache (actually done by a background application)
++      (None -> "foo").
++    - A concurrent transaction invalidates the cache due to some user action
++      (None -> None), and pushes a new background task to fill the cache.
++    Then the expected resolved value is None, and not "foo".
+ 
+     >>> db = ZODB.DB('t.fs') # FileStorage!
+     >>> storage = db.storage
+@@ -63,23 +81,23 @@ def succeed_with_resolution_when_state_is_unchanged():
+     >>> oid = conn.root.x._p_oid
+ 
+ So, let's try resolving when the old and committed states are the same
+-bit the new state (pickle) is different:
++but the new state (pickle) is different:
+ 
+     >>> p = storage.tryToResolveConflict(
+     ...         oid, serial1, serial1, storage.loadSerial(oid, serial2))
+ 
+-    >>> p == storage.loadSerial(oid, serial2)
+-    True
++    >>> conn._reader.getState(p)['resolved']
++    'new'
+ 
+ 
+-And when the old and new states are the same bit the committed state
++And when the old and new states are the same but the committed state
+ is different:
+ 
+     >>> p = storage.tryToResolveConflict(
+     ...         oid, serial2, serial1, storage.loadSerial(oid, serial1))
+ 
+-    >>> p == storage.loadSerial(oid, serial2)
+-    True
++    >>> conn._reader.getState(p)['resolved']
++    'committed'
+ 
+ But we still conflict if both the committed and new are different than
+ the original:
+@@ -92,7 +110,9 @@ def succeed_with_resolution_when_state_is_unchanged():
+     ConflictError: database conflict error (oid 0x01, ...
+ 
+ 
+-Of course, none of this applies if content doesn't support conflict resolution.
++Of course, there's also no automatic trivial merge if content doesn't support
++conflict resolution. Touching an object without change is a common locking
++mechanism.
+ 
+     >>> conn.root.y = Unresolvable()
+     >>> conn.root.y.v = 1

software/neoppod/software-common.cfg

@@ -17,7 +17,7 @@ extends =
     ../../component/pycurl/buildout.cfg
 
 parts =
-# keep neoppod first so that ZODB3 is built correctly,
+# keep neoppod first so that ZODB is built correctly,
 # before any other section that would depend on it
     neoppod-develop
     neoppod
@@ -40,9 +40,16 @@ environment = neoppod-setup-env
 
 [neoppod]
 recipe = zc.recipe.egg
-eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
+eggs = neoppod[admin, ctl, master, storage-mysqldb]
   ${python-mysqlclient:egg}
-  ZODB3
+  psutil
+  BTrees
+  ZODB
+  zope.testing
+patch-binary = ${patch:location}/bin/patch
+ZODB-patches =
+  ${:_profile_base_location_}/../../component/egg-patch/ZODB-4.4.3.patch#d9c13b2873e770e97f19d0323392e839
+ZODB-patch-options = -p1
 
 [slapos-deps-eggs]
 recipe = zc.recipe.egg
@@ -102,7 +109,12 @@ md5sum = 81ab5e842ecf8385b12d735585497cc8
 
 [versions]
 slapos.recipe.template = 2.9
-ZODB3 = 3.10.7
+ZODB = 4.4.3+SlapOSPatched001
+BTrees = 4.3.1
+persistent = 4.2.1
+transaction = 1.7.0
+zdaemon = 4.1.0
+zodbpickle = 0.6.0
 # Required by slapos.toolbox = 0.61
 slapos.toolbox = 0.61
 PyRSS2Gen = 1.1

software/neoppod/software-zodb3.cfg

+[buildout]
+extends = software.cfg
+
+[neoppod]
+eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
+  ${python-mysqlclient:egg}
+  ZODB3
+ZODB3-patch-options = -p1
+ZODB3-patches =
+  ${neoppod-repository:location}/ZODB3.patch
+
+[versions]
+ZODB3 = 3.10.7+SlapOSPatched001
+transaction = 1.1.1
+zdaemon = 2.0.7

software/neoppod/software-zodb4.cfg

-[buildout]
-extends = software.cfg
-
-[neoppod]
-eggs = neoppod
-  ${python-mysqlclient:egg}
-  psutil
-  ZODB
-  zope.testing
-
-[versions]
-ZODB = 4.4.3
-transaction =
-zdaemon =

software/neoppod/software.cfg

@@ -33,19 +33,12 @@ recipe = zc.recipe.egg
 eggs = erp5.util
 interpreter = ${:_buildout_section_name_}
 
-[neoppod]
-patch-binary = ${patch:location}/bin/patch
-ZODB3-patch-options = -p1
-ZODB3-patches +=
-  ${neoppod-repository:location}/ZODB3.patch
-
 [versions]
-ZODB3 = 3.10.7+SlapOSPatched001
 erp5.util = 0.4.46
 # To match ERP5
-transaction = 1.1.1
 ZConfig = 2.9.3
 zc.lockfile = 1.0.2
-zdaemon = 2.0.7
 zope.event = 3.5.2
+zope.exceptions = 3.6.2
+zope.testing = 3.9.7
 ##

stack/erp5/buildout.cfg

@@ -466,10 +466,9 @@ initialization =
 
 [eggs]
 <= neoppod
-eggs =
+eggs = ${neoppod:eggs}
   ${numpy:egg}
   ${matplotlib:egg}
-  ${python-mysqlclient:egg}
   ${lxml-python:egg}
   ${pandas:egg}
   ${pillow-python:egg}
@@ -524,9 +523,7 @@ eggs =
   pylint
   pytracemalloc
 
-  neoppod[client]
 # Zope
-  ZODB3
   Zope2
 # Zope acquisition patch
   Acquisition
@@ -683,6 +680,8 @@ PyXML = 0.8.5
 Pympler = 0.4.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
+ZEO = 4.2.1
+ZODB3 = 3.11.0
 Zope2 = 2.13.24
 astor = 0.5
 # astroid 1.4.1 breaks testDynamicClassGeneration