Skip to content

Z
Zope
Commit ba2387f8 authored by Barry Warsaw's avatar Barry Warsaw
Browse files
Options

_collect_objs(), _mark(): Fix some problems that cropped up in the 

cycle deletion tests.
parent 111ff445
Hide whitespace changes
Inline Side-by-side
Showing with 116 additions and 88 deletions
lib/python/BDBStorage/BDBFullStorage.py
View file @ ba2387f8
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
"""Berkeley storage with full undo and versioning support. """Berkeley storage with full undo and versioning support.
""" """
__version__ = '$Revision: 1.50 $'.split()[-2:][0] __version__ = '$Revision: 1.51 $'.split()[-2:][0]
import sys import sys
import time import time
...@@ -1504,7 +1504,10 @@ class Full(BerkeleyBase, ConflictResolvingStorage): ...@@ -1504,7 +1504,10 @@ class Full(BerkeleyBase, ConflictResolvingStorage):
# Delete the object from the serials table # Delete the object from the serials table
c = self._serials.cursor(txn) c = self._serials.cursor(txn)
try: try:
rec = c.set(oid) try:
rec = c.set(oid)
except db.DBNotFoundError:
rec = None
while rec and rec[0] == oid: while rec and rec[0] == oid:
c.delete() c.delete()
rec = c.next_dup() rec = c.next_dup()
...@@ -1520,7 +1523,10 @@ class Full(BerkeleyBase, ConflictResolvingStorage): ...@@ -1520,7 +1523,10 @@ class Full(BerkeleyBase, ConflictResolvingStorage):
# Collect all metadata records for this object # Collect all metadata records for this object
c = self._metadata.cursor(txn) c = self._metadata.cursor(txn)
try: try:
rec = c.set_range(oid) try:
rec = c.set_range(oid)
except db.DBNotFoundError:
rec = None
while rec and rec[0][:8] == oid: while rec and rec[0][:8] == oid:
revid, metadata = rec revid, metadata = rec
tid = revid[8:] tid = revid[8:]
...@@ -1593,25 +1599,26 @@ class Full(BerkeleyBase, ConflictResolvingStorage): ...@@ -1593,25 +1599,26 @@ class Full(BerkeleyBase, ConflictResolvingStorage):
# root references, and then for each of those, find all the objects it # root references, and then for each of those, find all the objects it
# references, and so on until we've traversed the entire object graph. # references, and so on until we've traversed the entire object graph.
while oid: while oid:
if self._packmark.has_key(oid): if not self._packmark.has_key(oid):
# We've already seen this object # We haven't seen this object yet
continue self._packmark.put(oid, PRESENT, txn=txn)
self._packmark.put(oid, PRESENT, txn=txn) # Get the pickle data for the most current revision of this
# Get the pickle data for the most current revision of this object # object as of the pack time.
# as of the pack time. tid = self._findrev(oid, packtid, txn)
tid = self._findrev(oid, packtid, txn) # Say there's no root object (as is the case in some of the
# Say there's no root object (as is the case in some of the unit # unit tests), and we're looking up oid ZERO. Then serial
# tests), and we're looking up oid ZERO. Then serial will be None. # will be None.
if tid is not None: if tid is not None:
lrevid = self._metadata[oid+tid][16:24] lrevid = self._metadata[oid+tid][16:24]
data = self._pickles[oid+lrevid] data = self._pickles[oid+lrevid]
# Now get the oids of all the objects referenced by this pickle # Now get the oids of all the objects referenced by this
refdoids = [] # pickle
referencesf(data, refdoids) refdoids = []
# And append them to the queue for later referencesf(data, refdoids)
for oid in refdoids: # And append them to the queue for later
self._oidqueue.append(oid, txn) for oid in refdoids:
# Pop the next oid off the queue and do it all again self._oidqueue.append(oid, txn)
# Pop the next oid off the queue and do it all again
rec = self._oidqueue.consume() rec = self._oidqueue.consume()
oid = rec and rec[1] oid = rec and rec[1]
assert len(self._oidqueue) == 0 assert len(self._oidqueue) == 0
......
lib/python/BDBStorage/BDBMinimalStorage.py
View file @ ba2387f8
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
"""Berkeley storage without undo or versioning. """Berkeley storage without undo or versioning.
""" """
__version__ = '$Revision: 1.14 $'[-2:][0] __version__ = '$Revision: 1.15 $'[-2:][0]
import time import time
import threading import threading
...@@ -440,23 +440,24 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage): ...@@ -440,23 +440,24 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage):
# root references, and then for each of those, find all the objects it # root references, and then for each of those, find all the objects it
# references, and so on until we've traversed the entire object graph. # references, and so on until we've traversed the entire object graph.
while oid: while oid:
if self._packmark.has_key(oid): if not self._packmark.has_key(oid):
# We've already seen this object # We've haven't yet seen this object
continue self._packmark.put(oid, PRESENT, txn=txn)
self._packmark.put(oid, PRESENT, txn=txn) # Get the pickle data for this object
# Get the pickle data for this object tid = self._getCurrentSerial(oid)
tid = self._getCurrentSerial(oid) # Say there's no root object (as is the case in some of the
# Say there's no root object (as is the case in some of the unit # unit tests), and we're looking up oid ZERO. Then serial
# tests), and we're looking up oid ZERO. Then serial will be None. # will be None.
if tid is not None: if tid is not None:
data = self._pickles[oid+tid] data = self._pickles[oid+tid]
# Now get the oids of all the objects referenced by this pickle # Now get the oids of all the objects referenced by this
refdoids = [] # pickle
referencesf(data, refdoids) refdoids = []
# And append them to the queue for later referencesf(data, refdoids)
for oid in refdoids: # And append them to the queue for later
self._oidqueue.append(oid, txn) for oid in refdoids:
# Pop the next oid off the queue and do it all again self._oidqueue.append(oid, txn)
# Pop the next oid off the queue and do it all again
rec = self._oidqueue.consume() rec = self._oidqueue.consume()
oid = rec and rec[1] oid = rec and rec[1]
assert len(self._oidqueue) == 0 assert len(self._oidqueue) == 0
...@@ -482,10 +483,14 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage): ...@@ -482,10 +483,14 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage):
orec = self._oidqueue.consume() orec = self._oidqueue.consume()
while orec: while orec:
oid = orec[1] oid = orec[1]
serial = self._getCurrentSerial(oid)
# Delete the object from the serials table # Delete the object from the serials table
c = self._serials.cursor(txn) c = self._serials.cursor(txn)
try: try:
rec = c.set(oid) try:
rec = c.set(oid)
except db.DBNotFoundError:
rec = None
while rec and rec[0] == oid: while rec and rec[0] == oid:
c.delete() c.delete()
rec = c.next_dup() rec = c.next_dup()
...@@ -501,7 +506,10 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage): ...@@ -501,7 +506,10 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage):
# Now collect the pickle data and do reference counting # Now collect the pickle data and do reference counting
c = self._pickles.cursor(txn) c = self._pickles.cursor(txn)
try: try:
rec = c.set_range(oid) try:
rec = c.set_range(oid)
except db.DBNotFoundError:
rec = None
while rec and rec[0][:8] == oid: while rec and rec[0][:8] == oid:
data = rec[1] data = rec[1]
c.delete() c.delete()
...@@ -510,8 +518,7 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage): ...@@ -510,8 +518,7 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage):
self._update(deltas, data, -1) self._update(deltas, data, -1)
for oid, delta in deltas.items(): for oid, delta in deltas.items():
refcount = U64(self._refcounts.get(oid, ZERO)) + delta refcount = U64(self._refcounts.get(oid, ZERO)) + delta
assert refcount >= 0 if refcount <= 0:
if refcount == 0:
self._oidqueue.append(oid, txn) self._oidqueue.append(oid, txn)
else: else:
self._refcounts.put(oid, p64(refcount), txn=txn) self._refcounts.put(oid, p64(refcount), txn=txn)
......
lib/python/BDBStorage/Full.py
View file @ ba2387f8
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
"""Berkeley storage with full undo and versioning support. """Berkeley storage with full undo and versioning support.
""" """
__version__ = '$Revision: 1.50 $'.split()[-2:][0] __version__ = '$Revision: 1.51 $'.split()[-2:][0]
import sys import sys
import time import time
...@@ -1504,7 +1504,10 @@ class Full(BerkeleyBase, ConflictResolvingStorage): ...@@ -1504,7 +1504,10 @@ class Full(BerkeleyBase, ConflictResolvingStorage):
# Delete the object from the serials table # Delete the object from the serials table
c = self._serials.cursor(txn) c = self._serials.cursor(txn)
try: try:
rec = c.set(oid) try:
rec = c.set(oid)
except db.DBNotFoundError:
rec = None
while rec and rec[0] == oid: while rec and rec[0] == oid:
c.delete() c.delete()
rec = c.next_dup() rec = c.next_dup()
...@@ -1520,7 +1523,10 @@ class Full(BerkeleyBase, ConflictResolvingStorage): ...@@ -1520,7 +1523,10 @@ class Full(BerkeleyBase, ConflictResolvingStorage):
# Collect all metadata records for this object # Collect all metadata records for this object
c = self._metadata.cursor(txn) c = self._metadata.cursor(txn)
try: try:
rec = c.set_range(oid) try:
rec = c.set_range(oid)
except db.DBNotFoundError:
rec = None
while rec and rec[0][:8] == oid: while rec and rec[0][:8] == oid:
revid, metadata = rec revid, metadata = rec
tid = revid[8:] tid = revid[8:]
...@@ -1593,25 +1599,26 @@ class Full(BerkeleyBase, ConflictResolvingStorage): ...@@ -1593,25 +1599,26 @@ class Full(BerkeleyBase, ConflictResolvingStorage):
# root references, and then for each of those, find all the objects it # root references, and then for each of those, find all the objects it
# references, and so on until we've traversed the entire object graph. # references, and so on until we've traversed the entire object graph.
while oid: while oid:
if self._packmark.has_key(oid): if not self._packmark.has_key(oid):
# We've already seen this object # We haven't seen this object yet
continue self._packmark.put(oid, PRESENT, txn=txn)
self._packmark.put(oid, PRESENT, txn=txn) # Get the pickle data for the most current revision of this
# Get the pickle data for the most current revision of this object # object as of the pack time.
# as of the pack time. tid = self._findrev(oid, packtid, txn)
tid = self._findrev(oid, packtid, txn) # Say there's no root object (as is the case in some of the
# Say there's no root object (as is the case in some of the unit # unit tests), and we're looking up oid ZERO. Then serial
# tests), and we're looking up oid ZERO. Then serial will be None. # will be None.
if tid is not None: if tid is not None:
lrevid = self._metadata[oid+tid][16:24] lrevid = self._metadata[oid+tid][16:24]
data = self._pickles[oid+lrevid] data = self._pickles[oid+lrevid]
# Now get the oids of all the objects referenced by this pickle # Now get the oids of all the objects referenced by this
refdoids = [] # pickle
referencesf(data, refdoids) refdoids = []
# And append them to the queue for later referencesf(data, refdoids)
for oid in refdoids: # And append them to the queue for later
self._oidqueue.append(oid, txn) for oid in refdoids:
# Pop the next oid off the queue and do it all again self._oidqueue.append(oid, txn)
# Pop the next oid off the queue and do it all again
rec = self._oidqueue.consume() rec = self._oidqueue.consume()
oid = rec and rec[1] oid = rec and rec[1]
assert len(self._oidqueue) == 0 assert len(self._oidqueue) == 0
......
lib/python/BDBStorage/Minimal.py
View file @ ba2387f8
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
"""Berkeley storage without undo or versioning. """Berkeley storage without undo or versioning.
""" """
__version__ = '$Revision: 1.14 $'[-2:][0] __version__ = '$Revision: 1.15 $'[-2:][0]
import time import time
import threading import threading
...@@ -440,23 +440,24 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage): ...@@ -440,23 +440,24 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage):
# root references, and then for each of those, find all the objects it # root references, and then for each of those, find all the objects it
# references, and so on until we've traversed the entire object graph. # references, and so on until we've traversed the entire object graph.
while oid: while oid:
if self._packmark.has_key(oid): if not self._packmark.has_key(oid):
# We've already seen this object # We've haven't yet seen this object
continue self._packmark.put(oid, PRESENT, txn=txn)
self._packmark.put(oid, PRESENT, txn=txn) # Get the pickle data for this object
# Get the pickle data for this object tid = self._getCurrentSerial(oid)
tid = self._getCurrentSerial(oid) # Say there's no root object (as is the case in some of the
# Say there's no root object (as is the case in some of the unit # unit tests), and we're looking up oid ZERO. Then serial
# tests), and we're looking up oid ZERO. Then serial will be None. # will be None.
if tid is not None: if tid is not None:
data = self._pickles[oid+tid] data = self._pickles[oid+tid]
# Now get the oids of all the objects referenced by this pickle # Now get the oids of all the objects referenced by this
refdoids = [] # pickle
referencesf(data, refdoids) refdoids = []
# And append them to the queue for later referencesf(data, refdoids)
for oid in refdoids: # And append them to the queue for later
self._oidqueue.append(oid, txn) for oid in refdoids:
# Pop the next oid off the queue and do it all again self._oidqueue.append(oid, txn)
# Pop the next oid off the queue and do it all again
rec = self._oidqueue.consume() rec = self._oidqueue.consume()
oid = rec and rec[1] oid = rec and rec[1]
assert len(self._oidqueue) == 0 assert len(self._oidqueue) == 0
...@@ -482,10 +483,14 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage): ...@@ -482,10 +483,14 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage):
orec = self._oidqueue.consume() orec = self._oidqueue.consume()
while orec: while orec:
oid = orec[1] oid = orec[1]
serial = self._getCurrentSerial(oid)
# Delete the object from the serials table # Delete the object from the serials table
c = self._serials.cursor(txn) c = self._serials.cursor(txn)
try: try:
rec = c.set(oid) try:
rec = c.set(oid)
except db.DBNotFoundError:
rec = None
while rec and rec[0] == oid: while rec and rec[0] == oid:
c.delete() c.delete()
rec = c.next_dup() rec = c.next_dup()
...@@ -501,7 +506,10 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage): ...@@ -501,7 +506,10 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage):
# Now collect the pickle data and do reference counting # Now collect the pickle data and do reference counting
c = self._pickles.cursor(txn) c = self._pickles.cursor(txn)
try: try:
rec = c.set_range(oid) try:
rec = c.set_range(oid)
except db.DBNotFoundError:
rec = None
while rec and rec[0][:8] == oid: while rec and rec[0][:8] == oid:
data = rec[1] data = rec[1]
c.delete() c.delete()
...@@ -510,8 +518,7 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage): ...@@ -510,8 +518,7 @@ class Minimal(BerkeleyBase, ConflictResolvingStorage):
self._update(deltas, data, -1) self._update(deltas, data, -1)
for oid, delta in deltas.items(): for oid, delta in deltas.items():
refcount = U64(self._refcounts.get(oid, ZERO)) + delta refcount = U64(self._refcounts.get(oid, ZERO)) + delta
assert refcount >= 0 if refcount <= 0:
if refcount == 0:
self._oidqueue.append(oid, txn) self._oidqueue.append(oid, txn)
else: else:
self._refcounts.put(oid, p64(refcount), txn=txn) self._refcounts.put(oid, p64(refcount), txn=txn)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7