Commit 0c21f503 authored by Tim Peters's avatar Tim Peters

Forward port from Zope 2.7 branch.

Many changes to fsrefs.py, based on problems I hit using it in real life.
parent 37c58b4a
......@@ -12,6 +12,20 @@ oid 17 was displayed as 0000000000000011. As a Python integer, that's
octal 9. Or was it meant to be decimal 11? Or was it meant to be hex?
Now it displays as 0x11.
fsrefs.py:
When run with -v, produced tracebacks for objects whose creation was
merely undone. This was confusing. Tracebacks are now produced only
if there's "a real" problem loading an oid.
If the current revision of object O refers to an object P whose
creation has been undone, this is now identified as a distinct case.
Captured and ignored most attempts to stop it via Ctrl+C. Repaired.
Now makes two passes, so that an accurate report can be given of all
invalid references.
What's new in ZODB3 3.3 beta 2
==============================
......
......@@ -13,13 +13,14 @@ import types
def get_pickle_metadata(data):
# ZODB's data records contain two pickles. The first is the class
# of the object, the second is the object.
if data.startswith('(c'):
# of the object, the second is the object. We're only trying to
# pick apart the first here, to extract the module and class names.
if data.startswith('(c'): # pickle MARK GLOBAL sequence
# Don't actually unpickle a class, because it will attempt to
# load the class. Just break open the pickle and get the
# module and class from it.
modname, classname, rest = data.split('\n', 2)
modname = modname[2:]
modname = modname[2:] # strip leading '(c'
return modname, classname
f = StringIO(data)
u = Unpickler(f)
......
......@@ -39,21 +39,19 @@ oid of the object is given in a message saying so, and if -v was specified
then the traceback corresponding to the load failure is also displayed
(this is the only effect of the -v flag).
Two other kinds of errors are also detected, one strongly related to
"failed to load", when an object O loads OK, and directly refers to a
persistent object P but there's a problem with P:
Three other kinds of errors are also detected, when an object O loads OK,
and directly refers to a persistent object P but there's a problem with P:
- If P doesn't exist in the database, a message saying so is displayed.
The unsatisifiable reference to P is often called a "dangling
reference"; P is called "missing" in the error output.
- If it was earlier determined that P could not be loaded (but does exist
in the database), a message saying that O refers to an object that can't
be loaded is displayed. Note that fsrefs only makes one pass over the
database, so if an object O refers to an unloadable object P, and O is
seen by fsrefs before P, an "O refers to the unloadable P" message will
not be produced; a message saying that P can't be loaded will be
produced when fsrefs later tries to load P, though.
- If the current state of the database is such that P's creation has
been undone, then P can't be loaded either. This is also a kind of
dangling reference, but is identified as "object creation was undone".
- If P can't be loaded (but does exist in the database), a message saying
that O refers to an object that can't be loaded is displayed.
fsrefs also (indirectly) checks that the .index file is sane, because
fsrefs uses the index to get its idea of what constitutes "all the objects
......@@ -65,28 +63,52 @@ revisions of objects; therefore fsrefs cannot find problems in versions or
in non-current revisions.
"""
from ZODB.FileStorage import FileStorage
from ZODB.TimeStamp import TimeStamp
from ZODB.utils import u64
from ZODB.FileStorage.fsdump import get_pickle_metadata
import cPickle
import cStringIO
import traceback
import types
from ZODB.FileStorage import FileStorage
from ZODB.TimeStamp import TimeStamp
from ZODB.utils import u64, oid_repr
from ZODB.FileStorage.fsdump import get_pickle_metadata
from ZODB.POSException import POSKeyError
VERBOSE = 0
# So full of undocumented magic it's hard to fathom.
# The existence of cPickle.noload() isn't documented, and what it
# does isn't documented either. In general it unpickles, but doesn't
# actually build any objects of user-defined classes. Despite that
# persistent_load is documented to be a callable, there's an
# undocumented gimmick where if it's actually a list, for a PERSID or
# BINPERSID opcode cPickle just appends "the persistent id" to that list.
# Also despite that "a persistent id" is documented to be a string,
# ZODB persistent ids are actually (often? always?) tuples, most often
# of the form
# (oid, (module_name, class_name))
# So the effect of the following is to dig into the object pickle, and
# return a list of the persistent ids found (which are usually nested
# tuples), without actually loading any modules or classes.
# Note that pickle.py doesn't support any of this, it's undocumented code
# only in cPickle.c.
def get_refs(pickle):
refs = []
# The pickle is in two parts. First there's the class of the object,
# needed to build a ghost, See get_pickle_metadata for how complicated
# this can get. The second part is the state of the object. We want
# to find all the persistent references within both parts (although I
# expect they can only appear in the second part).
f = cStringIO.StringIO(pickle)
u = cPickle.Unpickler(f)
u.persistent_load = refs
u.noload()
u.noload()
u.persistent_load = refs = []
u.noload() # class info
u.noload() # instance state info
return refs
def report(oid, data, serial, fs, missing):
# There's a problem with oid. 'data' is its pickle, and 'serial' its
# serial number. 'missing' is a list of (oid, class, reason) triples,
# explaining what the problem(s) is(are).
def report(oid, data, serial, missing):
from_mod, from_class = get_pickle_metadata(data)
if len(missing) > 1:
plural = "s"
......@@ -101,28 +123,41 @@ def report(oid, data, serial, fs, missing):
description = "%s.%s" % info
else:
description = str(info)
print "\toid %s %s: %r" % (hex(u64(oid)), reason, description)
print "\toid %s %s: %r" % (oid_repr(oid), reason, description)
print
def main(path):
fs = FileStorage(path, read_only=1)
# Set of oids in the index that failed to load due to POSKeyError.
# This is what happens if undo is applied to the transaction creating
# the object (the oid is still in the index, but its current data
# record has a backpointer of 0, and POSKeyError is raised then
# because of that backpointer).
undone = {}
# Set of oids that were present in the index but failed to load.
# This does not include oids in undone.
noload = {}
for oid in fs._index.keys():
try:
data, serial = fs.load(oid, "")
except (KeyboardInterrupt, SystemExit):
raise
except POSKeyError:
undone[oid] = 1
except:
print "oid %s failed to load" % hex(u64(oid))
if VERBOSE:
traceback.print_exc()
noload[oid] = 1
# If we get here after we've already loaded objects
# that refer to this one, we will not have gotten error reports
# from the latter about the current object being unloadable.
# We could fix this by making two passes over the storage, but
# that seems like overkill.
inactive = noload.copy()
inactive.update(undone)
for oid in fs._index.keys():
if oid in inactive:
continue
data, serial = fs.load(oid, "")
refs = get_refs(data)
missing = [] # contains 3-tuples of oid, klass-metadata, reason
for info in refs:
......@@ -132,12 +167,14 @@ def main(path):
# failed to unpack
ref = info
klass = '<unknown>'
if not fs._index.has_key(ref):
if ref not in fs._index:
missing.append((ref, klass, "missing"))
if noload.has_key(ref):
if ref in noload:
missing.append((ref, klass, "failed to load"))
if ref in undone:
missing.append((ref, klass, "object creation was undone"))
if missing:
report(oid, data, serial, fs, missing)
report(oid, data, serial, missing)
if __name__ == "__main__":
import sys
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment