Drop support for ZODB3

MANIFEST.in

 graft tools
-include neo.conf CHANGELOG.rst TODO ZODB3.patch
+include neo.conf CHANGELOG.rst TODO

README.rst

@@ -52,7 +52,7 @@ Requirements
 
   - MySQLdb: https://github.com/PyMySQL/mysqlclient-python
 
-- For client nodes: ZODB 3.10.x or later
+- For client nodes: ZODB 4.4.5 or later
 
 Installation
 ============

neo/client/__init__.py

@@ -25,51 +25,7 @@ def patch():
 
     H = lambda f: md5(f.func_code.co_code).hexdigest()
 
-    # Allow serial to be returned as late as tpc_finish
-    #
-    # This makes possible for storage to allocate serial inside tpc_finish,
-    # removing the requirement to serialise second commit phase (tpc_vote
-    # to tpc_finish/tpc_abort).
-
-    h = H(Connection.tpc_finish)
-
-    def tpc_finish(self, transaction):
-        """Indicate confirmation that the transaction is done."""
-
-        def callback(tid):
-            if self._mvcc_storage:
-                # Inter-connection invalidation is not needed when the
-                # storage provides MVCC.
-                return
-            d = dict.fromkeys(self._modified)
-            self._db.invalidate(tid, d, self)
-#       It's important that the storage calls the passed function
-#       while it still has its lock.  We don't want another thread
-#       to be able to read any updated data until we've had a chance
-#       to send an invalidation message to all of the other
-#       connections!
-        # <patch>
-        serial = self._storage.tpc_finish(transaction, callback)
-        if serial is not None:
-            assert isinstance(serial, bytes), repr(serial)
-            for oid_iterator in (self._modified, self._creating):
-                for oid in oid_iterator:
-                    obj = self._cache.get(oid, None)
-                    # Ignore missing objects and don't update ghosts.
-                    if obj is not None and obj._p_changed is not None:
-                        obj._p_changed = 0
-                        obj._p_serial = serial
-        # </patch>
-        self._tpc_cleanup()
-
-    global OLD_ZODB
-    OLD_ZODB = h in (
-        'ab9b1b8d82c40e5fffa84f7bc4ea3a8b', # Python 2.7
-        )
-
-    if OLD_ZODB:
-        Connection.tpc_finish = tpc_finish
-    elif hasattr(Connection, '_handle_serial'): # merged upstream ?
+    if hasattr(Connection, '_handle_serial'): # merged upstream ?
         assert hasattr(Connection, '_warn_about_returned_serial')
 
     # sync() is used to provide a "network barrier", which is required for

neo/client/app.py

@@ -25,9 +25,6 @@ except ImportError:
     from cPickle import dumps, loads
     _protocol = 1
 from ZODB.POSException import UndoError, ConflictError, ReadConflictError
-from . import OLD_ZODB
-if OLD_ZODB:
-  from ZODB.ConflictResolution import ResolvedSerial
 from persistent.TimeStamp import TimeStamp
 
 from neo.lib import logging
@@ -641,9 +638,6 @@ class Application(ThreadedApplication):
         #       - If possible, recover from master failure.
         if txn_context.error:
             raise NEOStorageError(txn_context.error)
-        if OLD_ZODB:
-            return [(oid, ResolvedSerial)
-                for oid in txn_context.resolved_dict]
         return txn_context.resolved_dict
 
     def tpc_abort(self, transaction):

neo/storage/database/importer.py

@@ -609,6 +609,10 @@ class ImporterDatabaseManager(DatabaseManager):
         except TypeError: # loadBefore returned None
             return False
         except POSKeyError:
+            # loadBefore does not distinguish between an oid:
+            # - that does not exist at any serial
+            # - that was deleted
+            # - whose creation was undone
             assert not o or o[3] is None, o
             return o
         if serial != tid:
@@ -617,22 +621,15 @@ class ImporterDatabaseManager(DatabaseManager):
             u_tid = u64(serial)
         if u_tid <= self.zodb_tid and o:
             return o
-        if value:
-            value = zodb.repickle(value)
-            checksum = util.makeChecksum(value)
-        else:
-            # CAVEAT: Although we think loadBefore should not return an empty
-            #         value for a deleted object (BBB: fixed in ZODB4),
-            #         there's no need to distinguish this case in the above
-            #         except clause because it would be crazy to import a
-            #         NEO DB using this backend.
-            checksum = None
+        value = zodb.repickle(value)
         if not next_serial:
             next_serial = db._getNextTID(db._getPartition(u_oid), u_oid, u_tid)
             if next_serial:
                 next_serial = p64(next_serial)
-        return (serial, next_serial,
-            0, checksum, value, zodb.getDataTid(z_oid, u_tid))
+        return (serial, next_serial, 0,
+                util.makeChecksum(value),
+                value,
+                zodb.getDataTid(z_oid, u_tid))
 
     def getTransaction(self, tid, all=False):
         u64 = util.u64

neo/tests/__init__.py

@@ -517,6 +517,15 @@ class TransactionalResource(object):
             return lambda *_: None
         return self.__getattribute__(attr)
 
+try:
+    from ZODB.Connection import TransactionMetaData
+except ImportError: # BBB: ZODB < 5
+    def getTransactionMetaData(txn, conn):
+        return txn
+else:
+    def getTransactionMetaData(txn, conn):
+        return txn.data(conn)
+
 
 class Patch(object):
     """

neo/tests/threaded/__init__.py

@@ -330,6 +330,15 @@ class Node(object):
     def filterConnection(self, *peers):
         return ConnectionFilter(self.getConnectionList(*peers))
 
+    @contextmanager
+    def patchDeferred(self, method):
+        deferred = []
+        with Patch(method.__self__, **{method.__name__:
+                lambda orig, *args, **kw: deferred.append(
+                      partial(orig, *args, **kw))}) as p:
+            yield p
+        self.em.wakeup(*deferred)
+
 class ServerNode(Node):
 
     _server_class_dict = {}

neo/tests/threaded/test.py

@@ -36,7 +36,7 @@ from neo.lib.handler import DelayEvent, EventHandler
 from neo.lib import logging
 from neo.lib.protocol import (CellStates, ClusterStates, NodeStates, NodeTypes,
     Packets, Packet, uuid_str, ZERO_OID, ZERO_TID, MAX_TID)
-from .. import Patch, TransactionalResource
+from .. import Patch, TransactionalResource, getTransactionMetaData
 from . import ClientApplication, ConnectionFilter, LockLock, NEOCluster, \
     NEOThreadedTest, RandomConflictDict, Serialized, ThreadId, with_cluster
 from neo.lib.util import add64, makeChecksum, p64, u64
@@ -1312,10 +1312,7 @@ class Test(NEOThreadedTest):
                 # Check that the storage hasn't answered to the store,
                 # which means that a lock is still taken for r['x'] by t2.
                 self.tic()
-                try:
-                    txn = txn.data(c1)
-                except (AttributeError, KeyError): # BBB: ZODB < 5
-                    pass
+                txn = getTransactionMetaData(txn, c1)
                 txn_context = cluster.client._txn_container.get(txn)
                 empty = txn_context.queue.empty()
                 ll()

neo/tests/threaded/testImporter.py

@@ -19,18 +19,22 @@ from cStringIO import StringIO
 from itertools import izip_longest
 import os, random, shutil, threading, time, unittest
 import transaction, ZODB
+from persistent import Persistent
 from neo.client.exception import NEOPrimaryMasterLost
 from neo.lib import logging
+from neo.lib.protocol import MAX_TID
 from neo.lib.util import cached_property, p64, u64
 from neo.master.transactions import TransactionManager
 from neo.storage.database import getAdapterKlass, importer, manager
 from neo.storage.database.importer import \
     Repickler, TransactionRecord, WriteBack
-from .. import expectedFailure, getTempDirectory, random_tree, Patch
+from .. import expectedFailure, getTempDirectory, random_tree, \
+    Patch, TransactionalResource, getTransactionMetaData
 from . import NEOCluster, NEOThreadedTest
 from ZODB import serialize
 from ZODB.DB import TransactionalUndo
 from ZODB.FileStorage import FileStorage
+from ZODB.POSException import POSKeyError
 
 class Equal:
 
@@ -371,6 +375,45 @@ class ImporterTests(NEOThreadedTest):
             t.begin()
             finalCheck(r)
 
+    def testDeleteAndUndo(self):
+        fs_path, cfg = self.getFS()
+        c = ZODB.DB(FileStorage(fs_path)).open()
+        s = c.db().storage
+        r = c.root()
+        tid = r._p_serial
+        r[''] = delete = Persistent()
+        transaction.commit()
+        self.assertEqual(delete._p_oid, p64(1))
+        del r['']
+        TransactionalResource(transaction, 0, commit=lambda txn:
+            s.deleteObject(delete._p_oid, delete._p_serial,
+                           getTransactionMetaData(txn, c)))
+        transaction.commit()
+        r[''] = undo = Persistent()
+        transaction.commit()
+        c.db().undo(s.undoLog(last=1)[0]['id'])
+        transaction.commit()
+        self.assertEqual(undo._p_oid, p64(2))
+        def check():
+            self.assertIsNone(s.loadBefore(delete._p_oid, tid))
+            for oid in delete._p_oid, undo._p_oid, p64(3):
+                self.assertRaises(POSKeyError, s.loadBefore, oid, MAX_TID)
+        check() # FileStorage
+        c.db().close()
+        importer = {'zodb': [('root', cfg)]}
+        with NEOCluster(importer=importer) as cluster:
+            storage = cluster.storage
+            dm = storage.dm
+            with storage.patchDeferred(dm._finished):
+                with storage.patchDeferred(dm.doOperation):
+                    cluster.start()
+                    s = cluster.getZODBStorage()
+                    check() # before import
+                self.tic()
+                check() # imported, Importer getObject
+            self.tic()
+            check() # imported, direct getObject
+
 
 if __name__ == "__main__":
     unittest.main()

setup.py

@@ -44,7 +44,7 @@ get3rdParty(x, '3rdparty/' + x, 'https://lab.nexedi.com/nexedi/erp5'
     '/raw/14b0fcdcc31c5791646f9590678ca028f5d221f5/product/ERP5Type/' + x,
     'abb7970856540fd02150edd1fa9a3a3e8d0074ec526ab189684ef7ea9b41825f')
 
-zodb_require = ['ZODB3>=3.10dev']
+zodb_require = ['ZODB>=4.4.5']
 
 extras_require = {
     'admin': [],