Commit 746b14ac authored by Jim Fulton's avatar Jim Fulton

Cleaned up the serialization code code a bit:

- Added an "extended" reference format in preparation for adding
  cross-database references.

- Simplified referencesf and get_refs, added doc strings, and moved
  get_refs to be with the other serialization code.

Documented and slightly changed the api for get_refs.  Updated
  client code accordingly.
parent bd1194b3
......@@ -13,8 +13,8 @@
##############################################################################
import ZODB.FileStorage
from ZODB.utils import get_pickle_metadata
from ZODB.utils import p64, oid_repr, tid_repr, get_refs
from ZODB.utils import get_pickle_metadata, p64, oid_repr, tid_repr
from ZODB.serialize import get_refs
from ZODB.TimeStamp import TimeStamp
# Extract module.class string from pickle.
......
......@@ -80,18 +80,57 @@ for some but not all ZClasses).
Persistent references
---------------------
A persistent reference is a pair containing an oid and class metadata.
When one persistent object pickle refers to another persistent object,
the database uses a persistent reference. The format allows a
significant optimization, because ghosts can be created directly from
persistent references. If the reference was just an oid, a database
access would be required to determine the class of the ghost.
Because the persistent reference includes the class, it is not
possible to change the class of a persistent object. If a transaction
changed the class of an object, a new record with new class metadata
would be written but all the old references would still include the
old class.
the database uses a persistent reference.
ZODB persistent references are of the form::
oid
A simple object reference.
(oid, class meta data)
A persistent object reference
[reference_type, args]
An extended reference
Extension references come in a number of subforms, based on the
reference types.
The following reference types are defined:
'w'
Persistent weak reference. The arguments consist of an oid.
The following are planned for the future:
'n'
Multi-database simple object reference. The arguments consist
of a databaase name, and an object id.
'm'
Multi-database persistent object reference. The arguments consist
of a databaase name, an object id, and class meta data.
The following legacy format is also supported.
[oid]
A persistent weak reference
Because the persistent object reference forms include class
information, it is not possible to change the class of a persistent
object for which this form is used. If a transaction changed the
class of an object, a new record with new class metadata would be
written but all the old references would still use the old class. (It
is possible that we could deal with this limitation in the future.)
An object id is used alone when a class requires arguments
to it's __new__ method, which is signalled by the class having a
__getnewargs__ attribute.
A number of legacyforms are defined:
"""
import cPickle
......@@ -265,7 +304,7 @@ class ObjectWriter:
obj._p_jar = self._jar
obj._p_oid = oid
self._stack.append(obj)
return [oid]
return ['w', (oid, )]
# Since we have an oid, we have either a persistent instance
......@@ -385,11 +424,25 @@ class ObjectReader:
return unpickler
def _persistent_load(self, oid):
if isinstance(oid, tuple):
loaders = {}
def _persistent_load(self, reference):
if isinstance(reference, tuple):
return self.load_persistent(*reference)
elif isinstance(reference, str):
return self.load_oid(reference)
else:
try:
reference_type, args = reference
except ValueError:
# weakref
return self.loaders['w'](self, *reference)
else:
return self.loaders[reference_type](self, *args)
def load_persistent(self, oid, klass):
# Quick instance reference. We know all we need to know
# to create the instance w/o hitting the db, so go for it!
oid, klass = oid
obj = self._cache.get(oid, None)
if obj is not None:
......@@ -422,19 +475,24 @@ class ObjectReader:
self._cache[oid] = obj
return obj
elif isinstance(oid, list):
# see weakref.py
[oid] = oid
loaders['p'] = load_persistent
def load_persistent_weakref(self, oid):
obj = WeakRef.__new__(WeakRef)
obj.oid = oid
obj.dm = self._conn
return obj
loaders['w'] = load_persistent_weakref
def load_oid(self, oid):
obj = self._cache.get(oid, None)
if obj is not None:
return obj
return self._conn.get(oid)
loaders['o'] = load_oid
def _new_object(self, klass, args):
if not args and not myhasattr(klass, "__getnewargs__"):
obj = klass.__new__(klass)
......@@ -495,54 +553,89 @@ class ObjectReader:
state = self.getState(pickle)
obj.__setstate__(state)
def referencesf(p, rootl=None):
if rootl is None:
rootl = []
oid_loaders = {
'w': lambda oid: None,
}
def referencesf(p, oids=None):
"""Return a list of object ids found in a pickle
A list may be passed in, in which case, information is
appended to it.
Weak references are not included.
"""
refs = []
u = cPickle.Unpickler(cStringIO.StringIO(p))
l = len(rootl)
u.persistent_load = rootl
u.persistent_load = refs
u.noload()
u.noload()
# Now we have a list of referencs. Need to convert to list of
# oids:
if oids is None:
oids = []
for reference in refs:
if isinstance(reference, tuple):
oid = reference[0]
elif isinstance(reference, str):
oid = reference
else:
try:
reference_type, args = reference
except ValueError:
# weakref
continue
else:
oid = oid_loaders[reference_type](*args)
if oid:
oids.append(oid)
return oids
oid_klass_loaders = {
'w': lambda oid: None,
}
def get_refs(a_pickle):
"""Return oid and class information for references in a pickle
The result of a list of oid and class information tuples.
If the reference doesn't contain class information, then the
klass information is None.
"""
refs = []
u = cPickle.Unpickler(cStringIO.StringIO(a_pickle))
u.persistent_load = refs
u.noload()
except:
# Hm. We failed to do second load. Maybe there wasn't a
# second pickle. Let's check:
f = cStringIO.StringIO(p)
u = cPickle.Unpickler(f)
u.persistent_load = []
u.noload()
if len(p) > f.tell():
raise ValueError, 'Error unpickling %r' % p
# References may be:
#
# - A tuple, in which case they are an oid and class.
# In this case, just extract the first element, which is
# the oid
#
# - A list, which is a weak reference. We skip those.
#
# - Anything else must be an oid. This means that an oid
# may not be a list or a tuple. This is a bit lame.
# We could avoid this lamosity by allowing single-element
# tuples, so that we wrap oids that are lists or tuples in
# tuples.
#
# - oids may *not* be False. I'm not sure why.
out = []
for v in rootl:
assert v # Let's see if we ever get empty ones
if type(v) is list:
# skip wekrefs
# Now we have a list of referencs. Need to convert to list of
# oids and class info:
result = []
for reference in refs:
if isinstance(reference, tuple):
data = reference
elif isinstance(reference, str):
data = reference, None
else:
try:
reference_type, args = reference
except ValueError:
# weakref
continue
if type(v) is tuple:
v = v[0]
out.append(v)
else:
data = oid_klass_loaders[reference_type](*args)
rootl[:] = out
if data:
result.append(data)
return rootl
return result
......@@ -68,7 +68,8 @@ import types
from ZODB.FileStorage import FileStorage
from ZODB.TimeStamp import TimeStamp
from ZODB.utils import u64, oid_repr, get_refs, get_pickle_metadata
from ZODB.utils import u64, oid_repr, get_pickle_metadata
from ZODB.serialize import get_refs
from ZODB.POSException import POSKeyError
VERBOSE = 0
......@@ -129,9 +130,8 @@ def main(path):
refs = get_refs(data)
missing = [] # contains 3-tuples of oid, klass-metadata, reason
for info in refs:
try:
ref, klass = info
except (ValueError, TypeError):
if klass is None:
# failed to unpack
ref = info
klass = '<unknown>'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment