Commit 2c105365 authored by Thomas Lotze's avatar Thomas Lotze

- Fixed bug in ClientCache that occurred with objects larger than the total

  cache size.
parent 668dc2af
...@@ -10,7 +10,7 @@ New Features ...@@ -10,7 +10,7 @@ New Features
- Versions are no-longer supported. - Versions are no-longer supported.
- ZEO cache files can be larger than 4G. Note that lder ZEO cache - ZEO cache files can be larger than 4G. Note that older ZEO cache
files are not supported. files are not supported.
- Document conflict resolution (see ZODB/ConflictResolution.txt). - Document conflict resolution (see ZODB/ConflictResolution.txt).
...@@ -62,3 +62,6 @@ Bugs Fixed ...@@ -62,3 +62,6 @@ Bugs Fixed
deal with garbage files deal with garbage files
- Fixed bug in which MVCC would not work for blobs. - Fixed bug in which MVCC would not work for blobs.
- Fixed bug in ClientCache that occurred with objects larger than the total
cache size.
...@@ -234,6 +234,8 @@ class ClientCache(object): ...@@ -234,6 +234,8 @@ class ClientCache(object):
"already have current data for oid") "already have current data for oid")
else: else:
return return
if not self.fc.add(o):
return # too large
self.current[oid] = start_tid self.current[oid] = start_tid
self._trace(0x52, oid, start_tid, dlen=len(data)) self._trace(0x52, oid, start_tid, dlen=len(data))
else: else:
...@@ -241,9 +243,10 @@ class ClientCache(object): ...@@ -241,9 +243,10 @@ class ClientCache(object):
p = start_tid, end_tid p = start_tid, end_tid
if p in L: if p in L:
return # duplicate store return # duplicate store
if not self.fc.add(o):
return # too large
bisect.insort_left(L, p) bisect.insort_left(L, p)
self._trace(0x54, oid, start_tid, end_tid, dlen=len(data)) self._trace(0x54, oid, start_tid, end_tid, dlen=len(data))
self.fc.add(o)
## ##
# Remove all knowledge of noncurrent revisions of oid, both in # Remove all knowledge of noncurrent revisions of oid, both in
...@@ -883,7 +886,8 @@ class FileCache(object): ...@@ -883,7 +886,8 @@ class FileCache(object):
## ##
# Add Object object to the cache. This may evict existing objects, to # Add Object object to the cache. This may evict existing objects, to
# make room (and almost certainly will, in steady state once the cache # make room (and almost certainly will, in steady state once the cache
# is first full). The object must not already be in the cache. # is first full). The object must not already be in the cache. If the
# object is too large for the cache, False is returned, otherwise True.
def add(self, object): def add(self, object):
size = OBJECT_HEADER_SIZE + object.size size = OBJECT_HEADER_SIZE + object.size
# A number of cache simulation experiments all concluded that the # A number of cache simulation experiments all concluded that the
...@@ -891,7 +895,7 @@ class FileCache(object): ...@@ -891,7 +895,7 @@ class FileCache(object):
# objects simply weren't cached. For now, we ignore the request # objects simply weren't cached. For now, we ignore the request
# only if the entire cache file is too small to hold the object. # only if the entire cache file is too small to hold the object.
if size > self.maxsize - ZEC4_HEADER_SIZE: if size > self.maxsize - ZEC4_HEADER_SIZE:
return return False
assert object.key not in self.key2entry assert object.key not in self.key2entry
assert len(object.key[0]) == 8 assert len(object.key[0]) == 8
...@@ -902,6 +906,7 @@ class FileCache(object): ...@@ -902,6 +906,7 @@ class FileCache(object):
available = self._makeroom(size) available = self._makeroom(size)
self._writeobj(object, available) self._writeobj(object, available)
return True
## ##
# Return Object for key, or None if not in cache. # Return Object for key, or None if not in cache.
......
...@@ -52,6 +52,7 @@ Initially the object is not in the cache: ...@@ -52,6 +52,7 @@ Initially the object is not in the cache:
We can add it to the cache: We can add it to the cache:
>>> fc.add(obj1_1) >>> fc.add(obj1_1)
True
And now it's in the cache: And now it's in the cache:
...@@ -146,6 +147,7 @@ Case 1: Allocating a new block that fits after the last used one ...@@ -146,6 +147,7 @@ Case 1: Allocating a new block that fits after the last used one
>>> obj2_1 = Object(key=(oid(2), tid(1)), data='******', >>> obj2_1 = Object(key=(oid(2), tid(1)), data='******',
... start_tid=tid(1), end_tid=None) ... start_tid=tid(1), end_tid=None)
>>> fc.add(obj2_1) >>> fc.add(obj2_1)
True
The new block fits exactly in the remaining 47 bytes (41 bytes header + 6 The new block fits exactly in the remaining 47 bytes (41 bytes header + 6
bytes payload) so the beginning of the data is the same except for the last 47 bytes payload) so the beginning of the data is the same except for the last 47
...@@ -171,6 +173,7 @@ Case 2: Allocating a block that wraps around and frees *exactly* one block ...@@ -171,6 +173,7 @@ Case 2: Allocating a block that wraps around and frees *exactly* one block
>>> obj3_1 = Object(key=(oid(3), tid(1)), data='@'*100, >>> obj3_1 = Object(key=(oid(3), tid(1)), data='@'*100,
... start_tid=tid(1), end_tid=None) ... start_tid=tid(1), end_tid=None)
>>> fc.add(obj3_1) >>> fc.add(obj3_1)
True
>>> hexprint(fc.f) >>> hexprint(fc.f)
00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...| 00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...|
...@@ -192,6 +195,7 @@ Case 3: Allocating a block that requires 1 byte less than the next block ...@@ -192,6 +195,7 @@ Case 3: Allocating a block that requires 1 byte less than the next block
>>> obj4_1 = Object(key=(oid(4), tid(1)), data='~~~~~', >>> obj4_1 = Object(key=(oid(4), tid(1)), data='~~~~~',
... start_tid=tid(1), end_tid=None) ... start_tid=tid(1), end_tid=None)
>>> fc.add(obj4_1) >>> fc.add(obj4_1)
True
>>> hexprint(fc.f) >>> hexprint(fc.f)
00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...| 00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...|
...@@ -213,6 +217,7 @@ Case 4: Allocating a block that requires 2 bytes less than the next block ...@@ -213,6 +217,7 @@ Case 4: Allocating a block that requires 2 bytes less than the next block
>>> obj4_1 = Object(key=(oid(5), tid(1)), data='^'*98, >>> obj4_1 = Object(key=(oid(5), tid(1)), data='^'*98,
... start_tid=tid(1), end_tid=None) ... start_tid=tid(1), end_tid=None)
>>> fc.add(obj4_1) >>> fc.add(obj4_1)
True
>>> hexprint(fc.f) >>> hexprint(fc.f)
00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...| 00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...|
...@@ -239,6 +244,7 @@ it. ...@@ -239,6 +244,7 @@ it.
>>> obj4_1 = Object(key=(oid(6), tid(1)), data='+'*95, >>> obj4_1 = Object(key=(oid(6), tid(1)), data='+'*95,
... start_tid=tid(1), end_tid=None) ... start_tid=tid(1), end_tid=None)
>>> fc.add(obj4_1) >>> fc.add(obj4_1)
True
>>> hexprint(fc.f) >>> hexprint(fc.f)
00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...| 00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...|
...@@ -264,6 +270,7 @@ file. ...@@ -264,6 +270,7 @@ file.
>>> obj4_1 = Object(key=(oid(7), tid(1)), data='-'*89, >>> obj4_1 = Object(key=(oid(7), tid(1)), data='-'*89,
... start_tid=tid(1), end_tid=None) ... start_tid=tid(1), end_tid=None)
>>> fc.add(obj4_1) >>> fc.add(obj4_1)
True
>>> hexprint(fc.f) >>> hexprint(fc.f)
00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...| 00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...|
...@@ -287,6 +294,7 @@ Again, we replace the block at the beginning of the cache. ...@@ -287,6 +294,7 @@ Again, we replace the block at the beginning of the cache.
>>> obj4_1 = Object(key=(oid(8), tid(1)), data='='*80, >>> obj4_1 = Object(key=(oid(8), tid(1)), data='='*80,
... start_tid=tid(1), end_tid=None) ... start_tid=tid(1), end_tid=None)
>>> fc.add(obj4_1) >>> fc.add(obj4_1)
True
>>> hexprint(fc.f) >>> hexprint(fc.f)
00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...| 00000000 5a 45 43 34 00 00 00 00 00 00 00 00 61 00 00 00 |ZEC4........a...|
......
...@@ -241,6 +241,36 @@ class CacheTests(unittest.TestCase): ...@@ -241,6 +241,36 @@ class CacheTests(unittest.TestCase):
eq(copy.current, self.cache.current) eq(copy.current, self.cache.current)
eq(copy.noncurrent, self.cache.noncurrent) eq(copy.noncurrent, self.cache.noncurrent)
def testCurrentObjectLargerThanCache(self):
if self.cache.path:
os.remove(self.cache.path)
self.cache = ZEO.cache.ClientCache(size=50)
self.cache.open()
# We store an object that is a bit larger than the cache can handle.
self.cache.store(n1, n2, None, "x"*64)
# We can see that it was not stored.
self.assertEquals(None, self.cache.load(n1))
# If an object cannot be stored in the cache, it must not be
# recorded as current.
self.assert_(n1 not in self.cache.current)
# Regression test: invalidation must still work.
self.cache.invalidate(n1, n2)
def testOldObjectLargerThanCache(self):
if self.cache.path:
os.remove(self.cache.path)
self.cache = ZEO.cache.ClientCache(size=50)
self.cache.open()
# We store an object that is a bit larger than the cache can handle.
self.cache.store(n1, n2, n3, "x"*64)
# We can see that it was not stored.
self.assertEquals(None, self.cache.load(n1))
# If an object cannot be stored in the cache, it must not be
# recorded as non-current.
self.assert_((n2, n3) not in self.cache.noncurrent[n1])
def test_suite(): def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment