Merge branch 'master' into python38-and-39

.gitignore

@@ -5,6 +5,7 @@ __pycache__
 .installed.cfg
 .tox
 bin
+default.profraw
 develop-eggs
 dist
 eggs

CHANGES.rst

@@ -4,6 +4,21 @@ Changelog
 5.3.0 (unreleased)
 ------------------
 
+- Add support for Python 3.8 and Python 3.9.
+
+- Add more accurate error handling for ``asyncio.CancelledError``.
+  See `issue 165 <https://github.com/zopefoundation/ZEO/issues/165>`_.
+
+
+5.2.3 (2021-08-09)
+------------------
+
+- Ensure ``ZEO`` satisfies the ``ZODB >= 5.6`` requirement that
+  ``lastTransaction()`` changes only after invalidation processing.
+  Violating this requirement can lead to race conditions and
+  associated data corruption
+  `#166 <https://github.com/zopefoundation/ZEO/issues/166>`_.
+
 - Add automated tests against the ZODB ``master`` branch
   see `issue 177 <https://github.com/zopefoundation/ZEO/issues/177>`_.
 
@@ -13,12 +28,6 @@ Changelog
 - Improve log message when client cache is out of sync with server.
   See `issue 142 <https://github.com/zopefoundation/ZEO/issues/142>`_.
 
-- Add support for Python 3.8 and Python 3.9.
-
-- Add more accurate error handling for ``asyncio.CancelledError``.
-  See `issue 165 <https://github.com/zopefoundation/ZEO/issues/165>`_.
-
-
 5.2.2 (2020-08-11)
 ------------------
 
@@ -48,6 +57,7 @@ Changelog
   <https://github.com/zopefoundation/ZODB/issues/155>`_. See `issue
   134 <https://github.com/zopefoundation/ZEO/issues/134>`_.
 
+
 5.2.0 (2018-03-28)
 ------------------
 

doc/clients.rst

@@ -140,7 +140,7 @@ An application that used ZODB might configure it's database using a
 string like::
 
   <zodb>
-     cache-size-bytes 1000MB
+     cache-size 1000MB
 
      <filestorage>
        path /var/lib/Data.fs
@@ -158,7 +158,7 @@ but first you have to import it's definition, because ZEO isn't built
 into ZODB.  Here's an example::
 
   <zodb>
-     cache-size-bytes 1000MB
+     cache-size 1000MB
 
      %import ZEO
 
@@ -219,7 +219,7 @@ read-only-fallback
 server-sync
    Sets the ``server_sync`` option described above.
 
-wait_timeout
+wait-timeout
    How long to wait for an initial connection, defaulting to 30
    seconds.  If an initial connection can't be made within this time
    limit, then creation of the client storage will fail with a
@@ -234,13 +234,10 @@ wait_timeout
      connection to be established before failing with a
      ``ZEO.Exceptions.ClientDisconnected`` exception.
 
-client_label
+client-label
    A short string to display in *server* logs for an event relating to
    this client. This can be helpful when debugging.
 
-disconnect_poll
-   The delay in seconds between attempts to connect to the
-   server, in seconds.  Defaults to 1 second.
 
 Client SSL configuration
 ------------------------

src/ZEO/asyncio/client.py

@@ -659,11 +659,31 @@ class Client(object):
             try:
                 tid = yield self.protocol.fut('tpc_finish', tid)
                 cache = self.cache
+                # The cache invalidation here and that in
+                # ``invalidateTransaction`` are both performed
+                # in the IO thread. Thus there is no interference.
+                # Other threads might observe a partially invalidated
+                # cache. However, regular loads will access
+                # object state before ``tid``; therefore,
+                # partial invalidation for ``tid`` should not harm.
                 for oid, data, resolved in updates:
                     cache.invalidate(oid, tid)
                     if data and not resolved:
                         cache.store(oid, tid, None, data)
+                # ZODB >= 5.6 requires that ``lastTransaction`` changes
+                # only after invalidation processing (performed in
+                # the ``f`` call below) (for ``ZEO``, ``lastTransaction``
+                # is implemented as ``cache.getLastTid()``).
+                # Some tests involve ``f`` in the verification that
+                # ``tpc_finish`` modifies ``lastTransaction`` and require
+                # that ``cache.setLastTid`` is called before ``f``.
+                # We use locking below to ensure that the
+                # effect of ``setLastTid`` is observable by other
+                # threads only after ``f`` has been called.
+                with cache._lock:
                     cache.setLastTid(tid)
+                    f(tid)
+                future.set_result(tid)
             except Exception as exc:
                 future.set_exception(exc)
 
@@ -672,9 +692,6 @@ class Client(object):
                 # recovering to a consistent state.
                 self.protocol.close()
                 self.disconnected(self.protocol)
-            else:
-                f(tid)
-                future.set_result(tid)
         else:
             future.set_exception(ClientDisconnected())
 
@@ -684,6 +701,8 @@ class Client(object):
 
     def invalidateTransaction(self, tid, oids):
         if self.ready:
+            # see the cache related comment in ``tpc_finish_threadsafe``
+            # why we think that locking is not necessary at this place
             for oid in oids:
                 self.cache.invalidate(oid, tid)
             self.client.invalidateTransaction(tid, oids)

src/ZEO/asyncio/tests.py

@@ -9,7 +9,7 @@ from zope.testing import setupstack
 from concurrent.futures import Future
 import mock
 from ZODB.POSException import ReadOnlyError
-from ZODB.utils import maxtid
+from ZODB.utils import maxtid, RLock
 
 import collections
 import logging
@@ -699,6 +699,7 @@ class MemoryCache(object):
         # { oid -> [(start, end, data)] }
         self.data = collections.defaultdict(list)
         self.last_tid = None
+        self._lock = RLock()
 
     clear = __init__