Commit 746b14ac authored by Jim Fulton's avatar Jim Fulton

Cleaned up the serialization code code a bit:

- Added an "extended" reference format in preparation for adding
  cross-database references.

- Simplified referencesf and get_refs, added doc strings, and moved
  get_refs to be with the other serialization code.

Documented and slightly changed the api for get_refs.  Updated
  client code accordingly.
parent bd1194b3
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
############################################################################## ##############################################################################
import ZODB.FileStorage import ZODB.FileStorage
from ZODB.utils import get_pickle_metadata from ZODB.utils import get_pickle_metadata, p64, oid_repr, tid_repr
from ZODB.utils import p64, oid_repr, tid_repr, get_refs from ZODB.serialize import get_refs
from ZODB.TimeStamp import TimeStamp from ZODB.TimeStamp import TimeStamp
# Extract module.class string from pickle. # Extract module.class string from pickle.
......
...@@ -80,18 +80,57 @@ for some but not all ZClasses). ...@@ -80,18 +80,57 @@ for some but not all ZClasses).
Persistent references Persistent references
--------------------- ---------------------
A persistent reference is a pair containing an oid and class metadata.
When one persistent object pickle refers to another persistent object, When one persistent object pickle refers to another persistent object,
the database uses a persistent reference. The format allows a the database uses a persistent reference.
significant optimization, because ghosts can be created directly from
persistent references. If the reference was just an oid, a database ZODB persistent references are of the form::
access would be required to determine the class of the ghost.
oid
Because the persistent reference includes the class, it is not A simple object reference.
possible to change the class of a persistent object. If a transaction
changed the class of an object, a new record with new class metadata (oid, class meta data)
would be written but all the old references would still include the A persistent object reference
old class.
[reference_type, args]
An extended reference
Extension references come in a number of subforms, based on the
reference types.
The following reference types are defined:
'w'
Persistent weak reference. The arguments consist of an oid.
The following are planned for the future:
'n'
Multi-database simple object reference. The arguments consist
of a databaase name, and an object id.
'm'
Multi-database persistent object reference. The arguments consist
of a databaase name, an object id, and class meta data.
The following legacy format is also supported.
[oid]
A persistent weak reference
Because the persistent object reference forms include class
information, it is not possible to change the class of a persistent
object for which this form is used. If a transaction changed the
class of an object, a new record with new class metadata would be
written but all the old references would still use the old class. (It
is possible that we could deal with this limitation in the future.)
An object id is used alone when a class requires arguments
to it's __new__ method, which is signalled by the class having a
__getnewargs__ attribute.
A number of legacyforms are defined:
""" """
import cPickle import cPickle
...@@ -265,7 +304,7 @@ class ObjectWriter: ...@@ -265,7 +304,7 @@ class ObjectWriter:
obj._p_jar = self._jar obj._p_jar = self._jar
obj._p_oid = oid obj._p_oid = oid
self._stack.append(obj) self._stack.append(obj)
return [oid] return ['w', (oid, )]
# Since we have an oid, we have either a persistent instance # Since we have an oid, we have either a persistent instance
...@@ -385,11 +424,25 @@ class ObjectReader: ...@@ -385,11 +424,25 @@ class ObjectReader:
return unpickler return unpickler
def _persistent_load(self, oid): loaders = {}
if isinstance(oid, tuple):
def _persistent_load(self, reference):
if isinstance(reference, tuple):
return self.load_persistent(*reference)
elif isinstance(reference, str):
return self.load_oid(reference)
else:
try:
reference_type, args = reference
except ValueError:
# weakref
return self.loaders['w'](self, *reference)
else:
return self.loaders[reference_type](self, *args)
def load_persistent(self, oid, klass):
# Quick instance reference. We know all we need to know # Quick instance reference. We know all we need to know
# to create the instance w/o hitting the db, so go for it! # to create the instance w/o hitting the db, so go for it!
oid, klass = oid
obj = self._cache.get(oid, None) obj = self._cache.get(oid, None)
if obj is not None: if obj is not None:
...@@ -422,19 +475,24 @@ class ObjectReader: ...@@ -422,19 +475,24 @@ class ObjectReader:
self._cache[oid] = obj self._cache[oid] = obj
return obj return obj
elif isinstance(oid, list): loaders['p'] = load_persistent
# see weakref.py
[oid] = oid def load_persistent_weakref(self, oid):
obj = WeakRef.__new__(WeakRef) obj = WeakRef.__new__(WeakRef)
obj.oid = oid obj.oid = oid
obj.dm = self._conn obj.dm = self._conn
return obj return obj
loaders['w'] = load_persistent_weakref
def load_oid(self, oid):
obj = self._cache.get(oid, None) obj = self._cache.get(oid, None)
if obj is not None: if obj is not None:
return obj return obj
return self._conn.get(oid) return self._conn.get(oid)
loaders['o'] = load_oid
def _new_object(self, klass, args): def _new_object(self, klass, args):
if not args and not myhasattr(klass, "__getnewargs__"): if not args and not myhasattr(klass, "__getnewargs__"):
obj = klass.__new__(klass) obj = klass.__new__(klass)
...@@ -495,54 +553,89 @@ class ObjectReader: ...@@ -495,54 +553,89 @@ class ObjectReader:
state = self.getState(pickle) state = self.getState(pickle)
obj.__setstate__(state) obj.__setstate__(state)
def referencesf(p, rootl=None):
if rootl is None: oid_loaders = {
rootl = [] 'w': lambda oid: None,
}
def referencesf(p, oids=None):
"""Return a list of object ids found in a pickle
A list may be passed in, in which case, information is
appended to it.
Weak references are not included.
"""
refs = []
u = cPickle.Unpickler(cStringIO.StringIO(p)) u = cPickle.Unpickler(cStringIO.StringIO(p))
l = len(rootl) u.persistent_load = refs
u.persistent_load = rootl u.noload()
u.noload() u.noload()
# Now we have a list of referencs. Need to convert to list of
# oids:
if oids is None:
oids = []
for reference in refs:
if isinstance(reference, tuple):
oid = reference[0]
elif isinstance(reference, str):
oid = reference
else:
try: try:
reference_type, args = reference
except ValueError:
# weakref
continue
else:
oid = oid_loaders[reference_type](*args)
if oid:
oids.append(oid)
return oids
oid_klass_loaders = {
'w': lambda oid: None,
}
def get_refs(a_pickle):
"""Return oid and class information for references in a pickle
The result of a list of oid and class information tuples.
If the reference doesn't contain class information, then the
klass information is None.
"""
refs = []
u = cPickle.Unpickler(cStringIO.StringIO(a_pickle))
u.persistent_load = refs
u.noload() u.noload()
except:
# Hm. We failed to do second load. Maybe there wasn't a
# second pickle. Let's check:
f = cStringIO.StringIO(p)
u = cPickle.Unpickler(f)
u.persistent_load = []
u.noload() u.noload()
if len(p) > f.tell():
raise ValueError, 'Error unpickling %r' % p # Now we have a list of referencs. Need to convert to list of
# oids and class info:
# References may be: result = []
#
# - A tuple, in which case they are an oid and class. for reference in refs:
# In this case, just extract the first element, which is if isinstance(reference, tuple):
# the oid data = reference
# elif isinstance(reference, str):
# - A list, which is a weak reference. We skip those. data = reference, None
# else:
# - Anything else must be an oid. This means that an oid try:
# may not be a list or a tuple. This is a bit lame. reference_type, args = reference
# We could avoid this lamosity by allowing single-element except ValueError:
# tuples, so that we wrap oids that are lists or tuples in # weakref
# tuples.
#
# - oids may *not* be False. I'm not sure why.
out = []
for v in rootl:
assert v # Let's see if we ever get empty ones
if type(v) is list:
# skip wekrefs
continue continue
if type(v) is tuple: else:
v = v[0] data = oid_klass_loaders[reference_type](*args)
out.append(v)
rootl[:] = out if data:
result.append(data)
return rootl return result
...@@ -68,7 +68,8 @@ import types ...@@ -68,7 +68,8 @@ import types
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB.TimeStamp import TimeStamp from ZODB.TimeStamp import TimeStamp
from ZODB.utils import u64, oid_repr, get_refs, get_pickle_metadata from ZODB.utils import u64, oid_repr, get_pickle_metadata
from ZODB.serialize import get_refs
from ZODB.POSException import POSKeyError from ZODB.POSException import POSKeyError
VERBOSE = 0 VERBOSE = 0
...@@ -129,9 +130,8 @@ def main(path): ...@@ -129,9 +130,8 @@ def main(path):
refs = get_refs(data) refs = get_refs(data)
missing = [] # contains 3-tuples of oid, klass-metadata, reason missing = [] # contains 3-tuples of oid, klass-metadata, reason
for info in refs: for info in refs:
try:
ref, klass = info ref, klass = info
except (ValueError, TypeError): if klass is None:
# failed to unpack # failed to unpack
ref = info ref = info
klass = '<unknown>' klass = '<unknown>'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment