Commit b28ca4f2 authored by Dmitry Vasiliev's avatar Dmitry Vasiliev

No more "<unknown>" class names. :-)

Most frequently "<unknown>" classes appears for objects changed
later in the same transaction with changes of an interesting objects
so we save data records for all objects changed in the transaction
and then check them for references.
parent 7aa7c699
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
import ZODB.FileStorage import ZODB.FileStorage
from ZODB.utils import get_pickle_metadata from ZODB.utils import get_pickle_metadata
from ZODB.utils import U64, p64, oid_repr, tid_repr, get_refs from ZODB.utils import p64, oid_repr, tid_repr, get_refs
from ZODB.TimeStamp import TimeStamp from ZODB.TimeStamp import TimeStamp
# Extract module.class string from pickle. # Extract module.class string from pickle.
...@@ -125,6 +125,8 @@ class Tracer(object): ...@@ -125,6 +125,8 @@ class Tracer(object):
def run(self): def run(self):
"""Find all occurrences of the registered oids in the database.""" """Find all occurrences of the registered oids in the database."""
# Maps oid of a reference to its module.class name.
self._ref2name = {}
for txn in ZODB.FileStorage.FileIterator(self.path): for txn in ZODB.FileStorage.FileIterator(self.path):
self._check_trec(txn) self._check_trec(txn)
...@@ -133,43 +135,66 @@ class Tracer(object): ...@@ -133,43 +135,66 @@ class Tracer(object):
# txn has members tid, status, user, description, # txn has members tid, status, user, description,
# _extension, _pos, _tend, _file, _tpos # _extension, _pos, _tend, _file, _tpos
self._produced_msg = False self._produced_msg = False
# Map and list for save data records for current transaction.
self._records_map = {}
self._records = []
for drec in txn: for drec in txn:
self._save_references(drec)
for drec in self._records:
self._check_drec(drec) self._check_drec(drec)
if self._produced_msg: if self._produced_msg:
# Copy txn info for later output. # Copy txn info for later output.
self.tid2info[txn.tid] = (txn.status, txn.user, txn.description, self.tid2info[txn.tid] = (txn.status, txn.user, txn.description,
txn._tpos) txn._tpos)
# Process next data record. If a message is produced, self._produced_msg def _save_references(self, drec):
# will be set True.
def _check_drec(self, drec):
# drec has members oid, tid, version, data, data_txn # drec has members oid, tid, version, data, data_txn
tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
if pick: if pick:
oidclass = None
if oid in self.oids: if oid in self.oids:
oidclass = get_class(pick) klass = get_class(pick)
self._msg(oid, tid, "new revision", oidclass, self._msg(oid, tid, "new revision", klass, "at", pos)
"at", drec.pos)
self.oids[oid] += 1 self.oids[oid] += 1
self.oid2name[oid] = oidclass self.oid2name[oid] = self._ref2name[oid] = klass
self._records_map[oid] = drec
self._records.append(drec)
elif oid in self.oids:
# Or maybe it's a version abort.
self._msg(oid, tid, "creation undo at", pos)
# Process next data record. If a message is produced, self._produced_msg
# will be set True.
def _check_drec(self, drec):
# drec has members oid, tid, version, data, data_txn
tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
ref2name = self._ref2name
ref2name_get = ref2name.get
records_map_get = self._records_map.get
if pick:
oid_in_oids = oid in self.oids
for ref, klass in get_refs(pick): for ref, klass in get_refs(pick):
if klass is None:
klass = '<unknown>'
elif isinstance(klass, tuple):
klass = "%s.%s" % klass
if ref in self.oids: if ref in self.oids:
oidclass = ref2name_get(oid, None)
if oidclass is None: if oidclass is None:
oidclass = get_class(pick) ref2name[oid] = oidclass = get_class(pick)
self._msg(ref, tid, "referenced by", oid_repr(oid), self._msg(ref, tid, "referenced by", oid_repr(oid),
oidclass, "at", pos) oidclass, "at", pos)
if oid in self.oids: if oid_in_oids:
if klass is None:
klass = ref2name_get(ref, None)
if klass is None:
r = records_map_get(ref, None)
# For save memory we only save references
# seen in one transaction with interesting
# objects changes. So in some circumstances
# we may still got "<unknown>" class name.
if r is None:
klass = "<unknown>"
else:
ref2name[ref] = klass = get_class(r.data)
elif isinstance(klass, tuple):
ref2name[ref] = klass = "%s.%s" % klass
self._msg(oid, tid, "references", oid_repr(ref), klass, self._msg(oid, tid, "references", oid_repr(ref), klass,
"at", pos) "at", pos)
elif oid in self.oids:
# Or maybe it's a version abort.
self._msg(oid, tid, "creation undo at", pos)
...@@ -94,7 +94,7 @@ oid 0x00 persistent.mapping.PersistentMapping 2 revisions ...@@ -94,7 +94,7 @@ oid 0x00 persistent.mapping.PersistentMapping 2 revisions
tid user='' tid user=''
tid description='added an OOBTree' tid description='added an OOBTree'
new revision persistent.mapping.PersistentMapping at 207 new revision persistent.mapping.PersistentMapping at 207
references 0x01 <unknown> at 207 references 0x01 BTrees._OOBTree.OOBTree at 207
oid 0x01 BTrees._OOBTree.OOBTree 1 revision oid 0x01 BTrees._OOBTree.OOBTree 1 revision
tid 0x... offset=168 ... tid 0x... offset=168 ...
tid user='' tid user=''
...@@ -103,18 +103,6 @@ oid 0x01 BTrees._OOBTree.OOBTree 1 revision ...@@ -103,18 +103,6 @@ oid 0x01 BTrees._OOBTree.OOBTree 1 revision
referenced by 0x00 persistent.mapping.PersistentMapping at 207 referenced by 0x00 persistent.mapping.PersistentMapping at 207
So there are two revisions of oid 0 now, and the second references oid 1. So there are two revisions of oid 0 now, and the second references oid 1.
It's peculiar that the class shows as <unknown> in:
references 0x01 <unknown> at 207
The code that does this takes long tours through undocumented code in
cPickle.c (using cPickle features that aren't in pickle.py, and aren't even
documented as existing). Whatever the reason, ZODB/util.py's get_refs()
function returns (oid_0x01, None) for the reference to oid 1, instead of the
usual (oid, (module_name, class_name)) form. Before I wrote this test,
I never saw a case of that before! "references" lines usually identify
the class of the object. Anyway, the correct class is given in the new
output for oid 1.
One more, storing a reference in the BTree back to the root object: One more, storing a reference in the BTree back to the root object:
...@@ -123,7 +111,7 @@ One more, storing a reference in the BTree back to the root object: ...@@ -123,7 +111,7 @@ One more, storing a reference in the BTree back to the root object:
>>> txn.get().note('circling back to the root') >>> txn.get().note('circling back to the root')
>>> txn.get().commit() >>> txn.get().commit()
>>> t = Tracer(path) >>> t = Tracer(path)
>>> t.register_oids(*range(3)) >>> t.register_oids(0, 1, 2)
>>> t.run(); t.report() #doctest: +ELLIPSIS >>> t.run(); t.report() #doctest: +ELLIPSIS
oid 0x00 persistent.mapping.PersistentMapping 2 revisions oid 0x00 persistent.mapping.PersistentMapping 2 revisions
tid 0x... offset=4 ... tid 0x... offset=4 ...
...@@ -134,7 +122,7 @@ oid 0x00 persistent.mapping.PersistentMapping 2 revisions ...@@ -134,7 +122,7 @@ oid 0x00 persistent.mapping.PersistentMapping 2 revisions
tid user='' tid user=''
tid description='added an OOBTree' tid description='added an OOBTree'
new revision persistent.mapping.PersistentMapping at 207 new revision persistent.mapping.PersistentMapping at 207
references 0x01 <unknown> at 207 references 0x01 BTrees._OOBTree.OOBTree at 207
tid 0x... offset=443 ... tid 0x... offset=443 ...
tid user='' tid user=''
tid description='circling back to the root' tid description='circling back to the root'
...@@ -149,7 +137,7 @@ oid 0x01 BTrees._OOBTree.OOBTree 2 revisions ...@@ -149,7 +137,7 @@ oid 0x01 BTrees._OOBTree.OOBTree 2 revisions
tid user='' tid user=''
tid description='circling back to the root' tid description='circling back to the root'
new revision BTrees._OOBTree.OOBTree at 491 new revision BTrees._OOBTree.OOBTree at 491
references 0x00 <unknown> at 491 references 0x00 persistent.mapping.PersistentMapping at 491
oid 0x02 <unknown> 0 revisions oid 0x02 <unknown> 0 revisions
this oid was not defined (no data record for it found) this oid was not defined (no data record for it found)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment