Commit 8310bf30 authored by Jérome Perrin's avatar Jérome Perrin

ERP5Type/XMLExportImport: use zodbpickle pickler for OrderedPickler

With upcoming ZODB 5, oids (used as persistent references in pickles)
are no longer str as it use to be with ZODB 4, but instances of
zodbpickle.binary, which with zodbpickle 1 are a subclass of str on
python2.

OrderedPickler was a subclass of pickle.Pickler, the pickler from standard
library, but this pickler was not able to use a str subclass for persistent
references, when pickles are loaded with noload method, persistent_load
is called with `None` instead of the actual string subclass instance.
This was problematic in the XMLExportImport handling of business templates,
because ZODB.serialize.referencesf was unable to find persistent references.
The error was:

    ZODB-5.6.0-py2.7.egg/ZODB/serialize.py", line 664, in referencesf
        assert isinstance(reference, list)
    AssertionError

because the reference was None.

zodbpickle 2 changed to make zodbpickle.binary implemented in C, which
was failing earlier, because pickle.Pickle can not pickle these objects,
failing in an error like this:

    lib/python2.7/copy_reg.py", line 70, in _reduce_ex
        raise TypeError, "can't pickle %s objects" % base.__name__
    TypeError: can't pickle binary objects

This change also simplify our own implementation, by dropping jython
support and calling save_dict on the super class instead of copying the
implementation.

Further references:

- minimal script to reproduce the issues:

```python
from __future__ import print_function
import io
import pickle

import zodbpickle
import zodbpickle.pickle
import zodbpickle.fastpickle

class ExternalObject(object):
  def __init__(self, oid):
    self.oid = oid

def persistent_id(obj):
  if isinstance(obj, ExternalObject):
    return obj.oid

def persistent_load(persid):
  print('persistent_load called with persid', repr(persid))

o = ExternalObject(oid=zodbpickle.binary("binary persid"))

for pickler_class in pickle.Pickler, zodbpickle.pickle.Pickler:

  f = io.BytesIO()
  p = pickler_class(f, 1)
  p.persistent_id = persistent_id
  p.dump(o)

  print('dump with pickler %s:\n  %r' % (pickler_class, f.getvalue()))

  # ZODB uses this unpickler
  up = zodbpickle.fastpickle.Unpickler(io.BytesIO(f.getvalue()))
  up.persistent_load = persistent_load
  up.noload()
```

```console
$ python2 repro.py # with zodbpickle 1
dump with pickler pickle.Pickler:
  'ccopy_reg\n_reconstructor\nq\x00(czodbpickle\nbinary\nq\x01c__builtin__\nstr\nq\x02U\rbinary persidq\x03tq\x04Rq\x05Q.'
persistent_load called with persid None
dump with pickler zodbpickle.pickle_2.Pickler:
  'U\rbinary persidq\x00Q.'
persistent_load called with persid 'binary persid'
```

```console
$ python2 repro.py # with zodbpickle 2
Traceback (most recent call last):
  File "repro.py", line 45, in <module>
    p.dump(o)
  File ".../lib/python2.7/pickle.py", line 224, in dump
    self.save(obj)
  File ".../lib/python2.7/pickle.py", line 273, in save
    self.save_pers(pid)
  File ".../lib/python2.7/pickle.py", line 340, in save_pers
    self.save(pid)
  File ".../lib/python2.7/pickle.py", line 306, in save
    rv = reduce(self.proto)
  File ".../lib/python2.7/copy_reg.py", line 70, in _reduce_ex
    raise TypeError, "can't pickle %s objects" % base.__name__
TypeError: can't pickle binary objects
```

* ZODB change starting to use zodbpickle.binary instead of str:
12ee41c4 (-ZODB now uses pickle protocol 3 for both Python 2 and Python 3., 2018-03-26)
Since of 5.4.0 release

* zodbpickle change starting to use C objects for zodbpickle.binary:
bbef98c (Implement zodbpickle.binary in C for Py27., 2019-11-12)
Since of 2.0.0 release
parent 7040de85
Pipeline #19619 failed with stage
in 0 seconds
...@@ -29,8 +29,10 @@ ...@@ -29,8 +29,10 @@
from Acquisition import aq_base, aq_inner from Acquisition import aq_base, aq_inner
from collections import OrderedDict
from cStringIO import StringIO from cStringIO import StringIO
from pickle import Pickler, EMPTY_DICT, MARK, DICT, PyStringMap, DictionaryType from zodbpickle.pickle import Pickler
from types import DictionaryType
from xml.sax.saxutils import escape, unescape from xml.sax.saxutils import escape, unescape
from lxml import etree from lxml import etree
from lxml.etree import Element, SubElement from lxml.etree import Element, SubElement
...@@ -46,23 +48,17 @@ marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI, ...@@ -46,23 +48,17 @@ marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI,
as_tree=True).dumps as_tree=True).dumps
class OrderedPickler(Pickler): class OrderedPickler(Pickler):
"""Pickler producing consistent output by saving dicts in order
"""
dispatch = Pickler.dispatch.copy() dispatch = Pickler.dispatch.copy()
def save_dict(self, obj): def save_dict(self, obj):
write = self.write return Pickler.save_dict(
if self.bin: self,
write(EMPTY_DICT) OrderedDict(sorted(obj.items())))
else: # proto 0 -- can't use EMPTY_DICT
write(MARK + DICT)
self.memoize(obj)
item_list = obj.items()
item_list.sort()
self._batch_setitems(iter(item_list))
dispatch[DictionaryType] = save_dict dispatch[DictionaryType] = save_dict
if not PyStringMap is None:
dispatch[PyStringMap] = save_dict
# ERP5 specific pickle function - produces ordered pickles # ERP5 specific pickle function - produces ordered pickles
def dumps(obj, protocol=None): def dumps(obj, protocol=None):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment