No more "<unknown>" class names. :-)

Most frequently "<unknown>" classes appears for objects changed
later in the same transaction with changes of an interesting objects
so we save data records for all objects changed in the transaction
and then check them for references.

trunk/src/ZODB/FileStorage/fsoids.py

@@ -14,7 +14,7 @@
 
 import ZODB.FileStorage
 from ZODB.utils import get_pickle_metadata
-from ZODB.utils import U64, p64, oid_repr, tid_repr, get_refs
+from ZODB.utils import p64, oid_repr, tid_repr, get_refs
 from ZODB.TimeStamp import TimeStamp
 
 # Extract module.class string from pickle.
@@ -125,6 +125,8 @@ class Tracer(object):
     def run(self):
         """Find all occurrences of the registered oids in the database."""
 
+        # Maps oid of a reference to its module.class name.
+        self._ref2name = {}
         for txn in ZODB.FileStorage.FileIterator(self.path):
             self._check_trec(txn)
 
@@ -133,43 +135,66 @@ class Tracer(object):
         # txn has members tid, status, user, description,
         # _extension, _pos, _tend, _file, _tpos
         self._produced_msg = False
+        # Map and list for save data records for current transaction.
+        self._records_map = {}
+        self._records = []
         for drec in txn:
+            self._save_references(drec)
+        for drec in self._records:
             self._check_drec(drec)
         if self._produced_msg:
             # Copy txn info for later output.
             self.tid2info[txn.tid] = (txn.status, txn.user, txn.description,
                                       txn._tpos)
 
-    # Process next data record.  If a message is produced, self._produced_msg
-    # will be set True.
-    def _check_drec(self, drec):
+    def _save_references(self, drec):
         # drec has members oid, tid, version, data, data_txn
         tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
         if pick:
-            oidclass = None
             if oid in self.oids:
-                oidclass = get_class(pick)
-                self._msg(oid, tid, "new revision", oidclass,
-                          "at", drec.pos)
+                klass = get_class(pick)
+                self._msg(oid, tid, "new revision", klass, "at", pos)
                 self.oids[oid] += 1
-                self.oid2name[oid] = oidclass
+                self.oid2name[oid] = self._ref2name[oid] = klass
+            self._records_map[oid] = drec
+            self._records.append(drec)
+        elif oid in self.oids:
+            # Or maybe it's a version abort.
+            self._msg(oid, tid, "creation undo at", pos)
 
+    # Process next data record.  If a message is produced, self._produced_msg
+    # will be set True.
+    def _check_drec(self, drec):
+        # drec has members oid, tid, version, data, data_txn
+        tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
+        ref2name = self._ref2name
+        ref2name_get = ref2name.get
+        records_map_get = self._records_map.get
+        if pick:
+            oid_in_oids = oid in self.oids
             for ref, klass in get_refs(pick):
-                if klass is None:
-                    klass = '<unknown>'
-                elif isinstance(klass, tuple):
-                    klass = "%s.%s" % klass
-
                 if ref in self.oids:
+                    oidclass = ref2name_get(oid, None)
                     if oidclass is None:
-                        oidclass = get_class(pick)
+                        ref2name[oid] = oidclass = get_class(pick)
                     self._msg(ref, tid, "referenced by", oid_repr(oid),
                               oidclass, "at", pos)
 
-                if oid in self.oids:
+                if oid_in_oids:
+                    if klass is None:
+                        klass = ref2name_get(ref, None)
+                        if klass is None:
+                            r = records_map_get(ref, None)
+                            # For save memory we only save references
+                            # seen in one transaction with interesting
+                            # objects changes. So in some circumstances
+                            # we may still got "<unknown>" class name.
+                            if r is None:
+                                klass = "<unknown>"
+                            else:
+                                ref2name[ref] = klass = get_class(r.data)
+                    elif isinstance(klass, tuple):
+                        ref2name[ref] = klass = "%s.%s" % klass
+
                     self._msg(oid, tid, "references", oid_repr(ref), klass,
                               "at", pos)
-
-        elif oid in self.oids:
-            # Or maybe it's a version abort.
-            self._msg(oid, tid, "creation undo at", pos)

trunk/src/ZODB/tests/testfsoids.py

@@ -94,7 +94,7 @@ oid 0x00 persistent.mapping.PersistentMapping 2 revisions
         tid user=''
         tid description='added an OOBTree'
         new revision persistent.mapping.PersistentMapping at 207
-        references 0x01 <unknown> at 207
+        references 0x01 BTrees._OOBTree.OOBTree at 207
 oid 0x01 BTrees._OOBTree.OOBTree 1 revision
     tid 0x... offset=168 ...
         tid user=''
@@ -103,18 +103,6 @@ oid 0x01 BTrees._OOBTree.OOBTree 1 revision
         referenced by 0x00 persistent.mapping.PersistentMapping at 207
 
 So there are two revisions of oid 0 now, and the second references oid 1.
-It's peculiar that the class shows as <unknown> in:
-
-        references 0x01 <unknown> at 207
-
-The code that does this takes long tours through undocumented code in
-cPickle.c (using cPickle features that aren't in pickle.py, and aren't even
-documented as existing).  Whatever the reason, ZODB/util.py's get_refs()
-function returns (oid_0x01, None) for the reference to oid 1, instead of the
-usual (oid, (module_name, class_name)) form.  Before I wrote this test,
-I never saw a case of that before!  "references" lines usually identify
-the class of the object.  Anyway, the correct class is given in the new
-output for oid 1.
 
 One more, storing a reference in the BTree back to the root object:
 
@@ -123,7 +111,7 @@ One more, storing a reference in the BTree back to the root object:
 >>> txn.get().note('circling back to the root')
 >>> txn.get().commit()
 >>> t = Tracer(path)
->>> t.register_oids(*range(3))
+>>> t.register_oids(0, 1, 2)
 >>> t.run(); t.report() #doctest: +ELLIPSIS
 oid 0x00 persistent.mapping.PersistentMapping 2 revisions
     tid 0x... offset=4 ...
@@ -134,7 +122,7 @@ oid 0x00 persistent.mapping.PersistentMapping 2 revisions
         tid user=''
         tid description='added an OOBTree'
         new revision persistent.mapping.PersistentMapping at 207
-        references 0x01 <unknown> at 207
+        references 0x01 BTrees._OOBTree.OOBTree at 207
     tid 0x... offset=443 ...
         tid user=''
         tid description='circling back to the root'
@@ -149,7 +137,7 @@ oid 0x01 BTrees._OOBTree.OOBTree 2 revisions
         tid user=''
         tid description='circling back to the root'
         new revision BTrees._OOBTree.OOBTree at 491
-        references 0x00 <unknown> at 491
+        references 0x00 persistent.mapping.PersistentMapping at 491
 oid 0x02 <unknown> 0 revisions
     this oid was not defined (no data record for it found)