Merge pull request #92 from zopefoundation/prefetch

Added a connection prefetch method

CHANGES.rst

@@ -2,6 +2,21 @@
  Change History
 ================
 
+5.0.0a6 (unreleased)
+====================
+
+- Added a connection ``prefetch`` method that can be used to request
+  that a storage prefect data an application will need::
+
+    conn.prefetch(obj, ...)
+
+  Where arguments can be objects, object ids, or iterables of objects
+  or object ids.
+
+  Added optional ``prefetch`` methods to the storage APIs. If a
+  storage doesn't support prefetch, then the connection prefetch
+  method is a noop.
+
 5.0.0a5 (2016-07-06)
 ====================
 

src/ZODB/Connection.py

@@ -1080,6 +1080,27 @@ class Connection(ExportImport, object):
     # Savepoint support
     #####################################################################
 
+    def prefetch(self, *args):
+        try:
+            self._storage.prefetch(self._prefetch_flatten(args))
+        except AttributeError:
+            if not hasattr(self._storage, 'prefetch'):
+                self.prefetch = lambda *a: None
+            else:
+                raise
+
+    def _prefetch_flatten(self, args):
+        for arg in args:
+            if isinstance(arg, bytes):
+                yield arg
+            elif hasattr(arg, '_p_oid'):
+                yield arg._p_oid
+            else:
+                for ob in arg:
+                    if isinstance(ob, bytes):
+                        yield ob
+                    else:
+                        yield ob._p_oid
 
 @implementer(IDataManagerSavepoint)
 class Savepoint:

src/ZODB/interfaces.py

@@ -754,6 +754,16 @@ class IStorage(Interface):
         """
 
 
+class IPrefetchStorage(IStorage):
+
+    def prefetch(oids, tid):
+        """Prefetch data for the given object ids before the given tid
+
+        The oids argument is an iterable that should be iterated no
+        more than once.
+        """
+
+
 class IMultiCommitStorage(IStorage):
     """A multi-commit storage can commit multiple transactions at once.
 
@@ -1112,6 +1122,14 @@ class IMVCCStorage(IStorage):
         A POSKeyError is raised if there is no record for the object id.
         """
 
+class IMVCCPrefetchStorage(IMVCCStorage):
+
+    def prefetch(oids):
+        """Prefetch data for the given object ids
+
+        The oids argument is an iterable that should be iterated no
+        more than once.
+        """
 
 class IStorageCurrentRecordIteration(IStorage):
 

src/ZODB/mvccadapter.py

@@ -146,6 +146,15 @@ class MVCCAdapterInstance(Base):
             raise POSException.ReadConflictError(repr(oid))
         return r[:2]
 
+    def prefetch(self, oids):
+        try:
+            self._storage.prefetch(oids, self._start)
+        except AttributeError:
+            if not hasattr(self._storage, 'prefetch'):
+                self.prefetch = lambda *a: None
+            else:
+                raise
+
     _modified = None # Used to keep track of oids modified within a
                      # transaction, so we can invalidate them later.
 

src/ZODB/tests/test_prefetch.py

+import unittest
+
+from ZODB.utils import z64, u64
+import ZODB
+
+from .MVCCMappingStorage import MVCCMappingStorage
+
+class PrefetchTests(unittest.TestCase):
+
+    def test_prefetch(self):
+        db = ZODB.DB(None)
+
+        fetched = []
+        def prefetch(oids, tid):
+            fetched.append((list(map(u64, oids)), tid))
+
+        db.storage.prefetch = prefetch
+
+        with db.transaction() as conn:
+            for i in range(10):
+                conn.root()[i] = conn.root().__class__()
+
+        conn = db.open()
+        conn.prefetch(z64)
+        conn.prefetch([z64])
+        conn.prefetch(conn.root())
+
+        conn.prefetch(z64, (conn.root()[i] for i in range(3)), conn.root()[3])
+
+        self.assertEqual(fetched,
+                         [([0], conn._storage._start),
+                          ([0], conn._storage._start),
+                          ([0], conn._storage._start),
+                          ([0, 1, 2, 3, 4], conn._storage._start),
+                          ])
+
+        db.close()
+
+    def test_prefetch_optional(self):
+        conn = ZODB.connection(None)
+        conn.prefetch(z64)
+        conn.prefetch([z64])
+        conn.prefetch(conn.root())
+        conn.prefetch(z64, [z64])
+        conn.prefetch(z64, [z64], conn.root())
+        conn.close()
+
+    def test_prefetch_optional_imvcc(self):
+        conn = ZODB.connection(MVCCMappingStorage())
+        conn.prefetch(z64)
+        conn.prefetch([z64])
+        conn.prefetch(conn.root())
+        conn.prefetch(z64, [z64])
+        conn.prefetch(z64, [z64], conn.root())
+        conn.close()
+
+
+def test_suite():
+    return unittest.makeSuite(PrefetchTests)