Commit 6149d5ac authored by Jim Fulton's avatar Jim Fulton

Merged Dieter's branch that ads an option to drop a zeo cache rather

than verifying it.
parent 807e2cca
......@@ -8,6 +8,12 @@ Change History
New Features
------------
- New `ClientStorage` configuration option `drop_cache_rather_verify`.
If this option is true then the ZEO client cache is dropped instead of
the long (unoptimized) verification. For large caches, setting this
option can avoid effective downtimes in the order of hours when
the connection to the ZEO server was interrupted for a longer time.
- The connection now estimates the object size based on its pickle size
and informs the cache about size changes.
......
......@@ -122,6 +122,7 @@ class ClientStorage(object):
wait_for_server_on_startup=None, # deprecated alias for wait
wait=None, wait_timeout=None,
read_only=0, read_only_fallback=0,
drop_cache_rather_verify=False,
username='', password='', realm=None,
blob_dir=None, shared_blob_dir=False):
"""ClientStorage constructor.
......@@ -196,6 +197,9 @@ class ClientStorage(object):
realm -- not documented.
drop_cache_rather_verify -- a flag indicating that the cache
should be dropped rather than expensively verified.
blob_dir -- directory path for blob data. 'blob data' is data that
is retrieved via the loadBlob API.
......@@ -218,6 +222,14 @@ class ClientStorage(object):
if debug:
log2("ClientStorage(): debug argument is no longer used")
# Remember some parameters for "_setupCache"
self._var_ = var
self._storage_ = storage
self._client_ = client
self._cache_size_ = cache_size
self._drop_cache_rather_verify = drop_cache_rather_verify
# wait defaults to True, but wait_for_server_on_startup overrides
# if not None
if wait_for_server_on_startup is not None:
......@@ -342,13 +354,7 @@ class ClientStorage(object):
else:
self.fshelper = None
# Decide whether to use non-temporary files
if client is not None:
dir = var or os.getcwd()
cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
else:
cache_path = None
self._cache = self.ClientCacheClass(cache_path, size=cache_size)
self._setupCache()
self._rpc_mgr = self.ConnectionManagerClass(addr, self,
tmin=min_disconnect_poll,
......@@ -363,6 +369,19 @@ class ClientStorage(object):
if not self._rpc_mgr.attempt_connect():
self._rpc_mgr.connect()
def _setupCache(self):
'''create and open the cache.'''
# Decide whether to use non-temporary files
storage = self._storage_
client = self._client_
cache_size = self._cache_size_
if client is not None:
dir = self._var_ or os.getcwd()
cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
else:
cache_path = None
self._cache = self.ClientCacheClass(cache_path, size=cache_size)
def _wait(self, timeout=None):
if timeout is not None:
deadline = time.time() + timeout
......@@ -1225,6 +1244,23 @@ class ClientStorage(object):
elif ltid and ltid != utils.z64:
self._cache.setLastTid(ltid)
# From this point on, we do not have complete information about
# the missed transactions. The reason is that cache
# verification only checks objects in the client cache and
# there may be objects in the object caches that aren't in the
# client cach that would need verification too. We avoid that
# problem by just invalidating the objects in the object caches.
if self._db is not None:
self._db.invalidateCache()
if self._cache and self._drop_cache_rather_verify:
log2("dropping cache")
self._cache.close()
self._setupCache() # creates a new cache
self._server = server
self._ready.set()
return "cache dropped"
log2("Verifying cache")
for oid, tid in self._cache.contents():
server.verify(oid, tid)
......@@ -1381,7 +1417,8 @@ class TransactionIterator(object):
class ClientStorageTransactionInformation(ZODB.BaseStorage.TransactionRecord):
def __init__(self, storage, txiter, tid, status, user, description, extension):
def __init__(self, storage, txiter, tid, status, user, description,
extension):
self._storage = storage
self._txiter = txiter
self._completed = False
......@@ -1394,7 +1431,8 @@ class ClientStorageTransactionInformation(ZODB.BaseStorage.TransactionRecord):
self.extension = extension
def __iter__(self):
riid = self._storage._server.iterator_record_start(self._txiter._iid, self.tid)
riid = self._storage._server.iterator_record_start(self._txiter._iid,
self.tid)
return self._storage._setup_iterator(RecordIterator, riid)
......
......@@ -102,6 +102,16 @@
<metadefault>$INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid)</metadefault>
</key>
<!-- DM 2006-06-12: added option -->
<key name="drop-cache-rather-verify" datatype="boolean"
required="no" default="false">
<description>
indicates that the cache should be dropped rather than
verified when the verification optimization is not
available (e.g. when the ZEO server restarted).
</description>
</key>
</sectiontype>
</component>
......@@ -60,6 +60,8 @@ logger = logging.getLogger('ZEO.tests.testZEO')
class DummyDB:
def invalidate(self, *args):
pass
def invalidateCache(*unused):
pass
class OneTimeTests(unittest.TestCase):
......@@ -145,6 +147,58 @@ class MiscZEOTests:
self.assertNotEquals(ZODB.utils.z64, storage3.lastTransaction())
storage3.close()
def checkDropCacheRatherVerifyImplementation(self):
# As it is quite difficult to set things up such that the verification
# optimizations do not step in, we emulate both the cache
# as well as the server.
from ZODB.TimeStamp import TimeStamp
class CacheEmulator(object):
# the settings below would be inconsitent for a normal cache
# but they are sufficient for our test setup
def __len__(self): return 1 # claim not to be empty
def contents(self): return () # do not invalidate anything
def getLastTid(self): return
def close(self): pass
class ServerEmulator(object):
def verify(*unused): pass
def endZeoVerify(*unused): pass
def lastTransaction(*unused): pass
storage = self._storage
storage._cache = cache = CacheEmulator()
server = ServerEmulator()
# test the standard behaviour
self.assertEqual(storage.verify_cache(server), "full verification")
# test the "drop cache rather verify" behaviour
storage._drop_cache_rather_verify = True
self.assertEqual(storage.verify_cache(server), "cache dropped")
# verify that we got a new cache
self.assert_(cache != storage._cache)
class ConfigurationTests(unittest.TestCase):
def checkDropCacheRatherVerifyConfiguration(self):
from ZODB.config import storageFromString
# the default is to do verification and not drop the cache
cs = storageFromString('''
<zeoclient>
server localhost:9090
wait false
</zeoclient>
''')
self.assertEqual(cs._drop_cache_rather_verify, False)
cs.close()
# now for dropping
cs = storageFromString('''
<zeoclient>
server localhost:9090
wait false
drop-cache-rather-verify true
</zeoclient>
''')
self.assertEqual(cs._drop_cache_rather_verify, True)
cs.close()
class GenericTests(
# Base class for all ZODB tests
StorageTestBase.StorageTestBase,
......@@ -955,7 +1009,9 @@ transaction, we'll get a result:
test_classes = [FileStorageTests, FileStorageRecoveryTests,
MappingStorageTests, DemoStorageTests,
BlobAdaptedFileStorageTests, BlobWritableCacheTests]
BlobAdaptedFileStorageTests, BlobWritableCacheTests,
ConfigurationTests,
]
def test_suite():
suite = unittest.TestSuite()
......
......@@ -163,6 +163,13 @@
that are accepted by this server.
</description>
</key>
<!-- DM 2008-05-15: added -->
<key name="drop-cache-rather-verify" datatype="boolean" default="off">
<description>
A flag indicating whether the client cache should be dropped
instead of an expensive verification.
</description>
</key>
</sectiontype>
<sectiontype name="demostorage" datatype=".DemoStorage"
......
......@@ -167,6 +167,7 @@ class ZEOClient(BaseConfig):
wait=self.config.wait,
read_only=self.config.read_only,
read_only_fallback=self.config.read_only_fallback,
drop_cache_rather_verify=self.config.drop_cache_rather_verify,
username=self.config.username,
password=self.config.password,
realm=self.config.realm)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment