Test auto-purging of the available-connection deque.

This triggered some code changes.  When closing a connection,
the meaning of "the pool is too big" is debatable:  it may mean
that the number of all connections the pool knows about exceeds
pool_size, or it may mean that the stack of available
connections (a subset of all the connections) exceeds pool_size.

The code was changed to mean the latter, primarily because it's
predictable.  The total number of connections the pool knows
about can change at any time, because it depends on which weak
references DB holds that cyclic gc hasn't yet cleared.

But it doesn't matter much either way, since clients are never
"supposed to" exceed pool_size anyway.  I think it degrades
more gracefully this way.

branches/tim-simpler_connection/src/ZODB/DB.py

@@ -49,7 +49,7 @@ class _ConnectionPool(object):
 
     When a connection is explicitly closed, tell the pool via repush().
     That adds the connection to a stack of connections available for
-    reuse, and throws away the oldest stack entries if the pool is too large.
+    reuse, and throws away the oldest stack entries if the stack is too large.
     pop() pops this stack.
 
     When a connection is obtained via pop(), the pool holds only a weak
@@ -72,7 +72,8 @@ class _ConnectionPool(object):
         # A stack of connections available to hand out.  This is a subset
         # of self.all.  push() and repush() add to this, and may remove
         # the oldest available connections if the pool is too large.
-        # pop() pops this stack.
+        # pop() pops this stack.  There are never more than pool_size entries
+        # in this stack.
         # In Python 2.4, a collections.deque would make more sense than
         # a list (we push only "on the right", but may pop from both ends).
         self.available = []
@@ -81,15 +82,16 @@ class _ConnectionPool(object):
     # If the pool_size is smaller than the current value, this may discard
     # the oldest available connections.
     def set_pool_size(self, pool_size):
-        self.pool_size = pool_size + 1  # _reduce_size shoots for < pool_size
-        self._reduce_size()
         self.pool_size = pool_size
+        self._reduce_size()
 
     # Register a new available connection.  We must not know about c already.
+    # c will be pushed onto the available stack even if we're over the
+    # pool size limit.
     def push(self, c):
         assert c not in self.all
         assert c not in self.available
-        self._reduce_size()
+        self._reduce_size(strictly_less=True)
         self.all.add(c)
         self.available.append(c)
         n, limit = len(self.all), self.pool_size
@@ -100,17 +102,21 @@ class _ConnectionPool(object):
             reporter("DB.open() has %s open connections with a pool_size "
                      "of %s", n, limit)
 
-    # Reregister an available connection formerly obtained via pop().
+    # Reregister an available connection formerly obtained via pop().  This
+    # pushes it on the stack of available connections, and may discard
+    # older available connections.
     def repush(self, c):
         assert c in self.all
         assert c not in self.available
-        self._reduce_size()
+        self._reduce_size(strictly_less=True)
         self.available.append(c)
 
     # Throw away the oldest available connections until we're under our
-    # target size.  It may not be possible to achieve this.
-    def _reduce_size(self):
-        while self.available and len(self.all) >= self.pool_size:
+    # target size (strictly_less=False) or no more than that (strictly_less=
+    # True, the default).  It may not be possible to achieve this.
+    def _reduce_size(self, strictly_less=False):
+        target = self.pool_size - bool(strictly_less)
+        while len(self.available) > target:
             c = self.available.pop(0)
             self.all.remove(c)
 

branches/tim-simpler_connection/src/ZODB/tests/dbopen.txt

@@ -17,7 +17,7 @@ Here we exercise the connection management done by the DB class.
 >>> from ZODB import DB
 >>> from ZODB.MappingStorage import MappingStorage as Storage
 
-Capturing log messages from DB is important for part of the examples:
+Capturing log messages from DB is important for some of the examples:
 
 >>> from zope.testing.loggingsupport import InstalledHandler
 >>> handler = InstalledHandler('ZODB.DB')
@@ -71,7 +71,7 @@ hold if the default pool size is overridden.
 >>> handler.clear()
 >>> st.close()
 >>> st = Storage()
->>> PS = 2
+>>> PS = 2 # smaller pool size
 >>> db = DB(st, pool_size=PS)
 >>> conns = [db.open() for dummy in range(PS)]
 >>> handler.records
@@ -97,7 +97,7 @@ More warnings through 4 connections:
 >>> print msg.name, msg.levelname, msg.getMessage()
 ZODB.DB WARNING DB.open() has 4 open connections with a pool_size of 2
 
-And a critical for going beyond that:
+And critical for going beyond that:
 
 >>> conns.append(db.open())
 >>> len(conns)
@@ -112,12 +112,9 @@ We can change the pool size on the fly:
 
 >>> handler.clear()
 >>> db.setPoolSize(6)
->>> conns.append(db.open)
+>>> conns.append(db.open())
 >>> handler.records  # no log msg -- the pool is bigger now
 []
->>> conns.append(db.open()) # likewise
->>> handler.records
-[]
 >>> conns.append(db.open()) # but one more and there's a warning again
 >>> len(handler.records)
 1
@@ -194,7 +191,7 @@ weak set allows DB's statistics methods to return info about connections
 that are still alive.
 
 
->>> len(db.cacheDetailSize())
+>>> len(db.cacheDetailSize())  # one result for each connection's cache
 3
 
 If a connection object is abandoned (it becomes unreachable), then it
@@ -207,7 +204,7 @@ isn't predictable.  It can be forced by running gc.collect():
 >>> len(pool.all)
 3
 >>> c3 = None
->>> dummy = gc.collect()
+>>> dummy = gc.collect()  # removes c3 from pool.all
 >>> len(pool.all)
 2
 
@@ -217,5 +214,74 @@ the stack of available connections by magic:
 >>> len(pool.available)
 0
 
+Nothing in that last block should have logged any msgs:
+
+>>> handler.records
+[]
+
+If "too many" connections are open, then closing one may kick an older
+closed one out of the available connection stack.
+
+>>> st.close()
+>>> st = Storage()
+>>> db = DB(st, pool_size=3)
+>>> conns = [db.open() for dummy in range(6)]
+>>> len(handler.records)  # 3 warnings for the "excess" connections
+3
+>>> pool = db._pools['']
+>>> len(pool.available), len(pool.all)
+(0, 6)
+
+Let's mark them:
+
+>>> for i, c in enumerate(conns):
+...     c.MARKER = i
+
+Closing connections adds them to the stack:
+
+>>> for i in range(3):
+...     conns[i].close()
+>>> len(pool.available), len(pool.all)
+(3, 6)
+>>> del conns[:3]  # leave the ones with MARKERs 3, 4 and 5
+
+Closing another one will purge the one with MARKER 0 from the stack
+(since it was the first added to the stack):
+
+>>> [c.MARKER for c in pool.available]
+[0, 1, 2]
+>>> conns[0].close()  # MARKER 3
+>>> len(pool.available), len(pool.all)
+(3, 5)
+>>> [c.MARKER for c in pool.available]
+[1, 2, 3]
+
+Similarly for the other two:
+
+>>> conns[1].close(); conns[2].close()
+>>> len(pool.available), len(pool.all)
+(3, 3)
+>>> [c.MARKER for c in pool.available]
+[3, 4, 5]
+
+Reducing the pool size may also purge the oldest closed connections:
+
+>>> db.setPoolSize(2)  # gets rid of MARKER 3
+>>> len(pool.available), len(pool.all)
+(2, 2)
+>>> [c.MARKER for c in pool.available]
+[4, 5]
+
+Since MARKER 5 is still the last one added to the stack, it will be the
+first popped:
+
+>>> c1 = db.open(); c2 = db.open()
+>>> c1.MARKER, c2.MARKER
+(5, 4)
+>>> len(pool.available), len(pool.all)
+(0, 2)
+
+Clean up.
+
 >>> st.close()
 >>> handler.uninstall()