feat/bt-py3 branch (MR 1979): BusinessTemplate,XMLExportImport: support python3

With these changes, we are able to install py2 business templates on py3,
but export is slightly different, because we already export using pickle
protocol 3 on py3.

To be able to install py2 business templates, we included heuristics to
guess the str or bytes from business template XML: oids are bytes and
also some strings that do not decode to UTF-8, so that we can install
python2 business templates on py3.

When exporting business templates, we need to build a list of referenced
persistent objects to export them separately in the XML, this is is done
using Unpickler.noload, in a way which does not support pickle protocol
1 on py3 (the persistent ids are None and the assertion in
https://github.com/zopefoundation/ZODB/blob/d698507bb89eeb38c6e655199bc9f54c909dbf4d/src/ZODB/serialize.py#L669
fails), so we need to use pickle protocol 3 on py3.

In the future, we might switch to exporting on protocol 3 on py2 as well
so that we have stable output on both py2 and py3, or maybe we'll do
this only when we stop supporting py2.

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py

 ##############################################################################
-#
+# coding: utf-8
 # Copyright (c) 2008 Nexedi SA and Contributors. All Rights Reserved.
 #                    TAHARA Yusei <yusei@nexedi.com>
 #
@@ -26,13 +26,16 @@
 #
 ##############################################################################
 
+import base64
 import unittest
-import pickle
+import zodbpickle
+import zodbpickle.fastpickle as pickle
 import re
-import xml.sax
-from StringIO import StringIO
-
-from Products.ERP5Type.XMLExportImport import ppml
+from io import BytesIO
+from six import StringIO
+from Products.ERP5Type.XMLExportImport import importXML, ppml
+import six
+import lxml.etree
 
 
 class DummyClass:
@@ -44,11 +47,58 @@ class DummyClass:
     self.data = []
 
 
-class TestXMLPickle(unittest.TestCase):
+class DummyPersistentClass:
+  def __init__(self, v, oid):
+    self.v = v
+    self._p_oid = oid
+
+
+class XMLPickleTestCase(unittest.TestCase):
+  _pickle_protocol = 3
+  def dump_to_xml(self, obj, persistent_id=None):
+    f = BytesIO()
+    pickler = pickle.Pickler(f, protocol=self._pickle_protocol)
+    if persistent_id:
+      pickler.persistent_id = persistent_id
+    pickler.dump(obj)
+    f.seek(0)
+    xml = ppml.ToXMLUnpickler(f).load(id_mapping=ppml.MinimalMapping()).__str__()
+    self.assertIsInstance(xml, str)
+    return xml
+
+  def load_from_xml(self, xml_string, persistent_load=None):
+    assertEqual = self.assertEqual
+    class DummyJar:
+      loaded = None
+      """follow interface expected by importXML"""
+      def importFile(self, file_, clue):
+        assertEqual(clue, 'ignored')
+        assertEqual(file_.read(4), b'ZEXP')
+        unpickler = pickle.Unpickler(file_)
+        if persistent_load:
+          unpickler.persistent_load = persistent_load
+        self.loaded = unpickler.load()
+
+    jar = DummyJar()
+    xml_string = '<?xml version="1.0"?>\n<ZopeData>%s</ZopeData>' % xml_string
+    importXML(jar, StringIO(xml_string), clue='ignored')
+    return jar.loaded
+
+  def dump_and_load(self, obj):
+    return self.load_from_xml(self.dump_to_xml(obj))
+
+  def check_and_load(self, v):
+    reconstructed = self.dump_and_load(v)
+    self.assertEqual(reconstructed, v)
+    self.assertIs(type(reconstructed), type(v))
+
+
+class TestXMLPickle(XMLPickleTestCase):
 
   def test_reduce(self):
     """
     Make sure that a object which uses reduce for pickling can be pickled by xml pickler.
+    This also covers the case of instances
     """
     obj = DummyClass()
     obj.data.append(1)
@@ -58,24 +108,7 @@ class TestXMLPickle(unittest.TestCase):
     pattern = re.compile('WAA') # regex pattern object uses reduce.(See sre.py)
     obj.data.append(pattern)
 
-    pickled_string = pickle.dumps(obj)
-    f = StringIO(pickled_string)
-    xmldata = str(ppml.ToXMLUnpickler(f).load())
-
-    output = StringIO()
-
-    F=ppml.xmlPickler()
-    F.file = output
-    F.binary = 1
-
-    content_handler = xml.sax.handler.ContentHandler()
-    content_handler.startElement = F.unknown_starttag
-    content_handler.endElement = F.unknown_endtag
-    content_handler.characters = F.handle_data
-    xml.sax.parseString(xmldata, content_handler)
-
-    reconstructed_pickled_data = F._stack[0][0]
-    reconstructed_obj = pickle.loads(reconstructed_pickled_data)
+    reconstructed_obj = self.dump_and_load(obj)
 
     self.assertTrue(reconstructed_obj.__class__ is DummyClass)
     self.assertTrue(type(getattr(reconstructed_obj, 'data', None)) is list)  # pylint:disable=unidiomatic-typecheck
@@ -84,3 +117,236 @@ class TestXMLPickle(unittest.TestCase):
     self.assertTrue(reconstructed_obj.data[2] is reconstructed_obj.data)
     self.assertTrue(type(reconstructed_obj.data[3]) is type(pattern))
     self.assertEqual(reconstructed_obj.data[3].pattern, 'WAA')
+
+  def test_bool(self):
+    self.assertIs(self.dump_and_load(True), True)
+    self.assertIs(self.dump_and_load(False), False)
+
+  def test_int(self):
+    self.check_and_load(-0)
+    self.check_and_load(1)
+    self.check_and_load(-1)
+    self.check_and_load(0xff)
+    self.check_and_load(0xff1)
+    self.check_and_load(0xffff)
+    self.check_and_load(2**128)
+    # long4
+    # https://github.com/python/cpython/blob/4d4a6f1b/Lib/test/pickletester.py#L2049-L2050
+    self.check_and_load(12345678910111213141516178920 << (256*8))
+
+
+  if six.PY2:
+    def test_long(self):
+      # pylint:disable=undefined-variable
+      self.check_and_load(long(-0))
+      self.check_and_load(long(1))
+      self.check_and_load(long(-1))
+      self.check_and_load(long(0xff))
+      self.check_and_load(long(0xff1))
+      self.check_and_load(long(0xffff))
+      self.check_and_load(long(2**128))
+      self.check_and_load(12345678910111213141516178920 << (256*8))
+      # pylint:enable=undefined-variable
+
+  def test_float(self):
+    self.check_and_load(-0.0)
+    self.check_and_load(1.0)
+    self.check_and_load(-1.0)
+    self.check_and_load(.33)
+
+  def test_None(self):
+    self.assertIs(
+      self.dump_and_load(None), None)
+
+  def test_bytes(self):
+    self.check_and_load(b"bytes")
+    self.check_and_load(b"long bytes" * 100)
+    if six.PY3 or self._pickle_protocol > 1:
+      # protocol 1 does not keep bytes
+      self.check_and_load(zodbpickle.binary(b"bytes"))
+      self.check_and_load(zodbpickle.binary(b""))
+
+  def test_unicode(self):  # BBB PY2
+    self.assertIs(type(self.dump_and_load(u"OK")), six.text_type)
+    self.check_and_load(u"short")
+    self.check_and_load(u"unicode 👍")
+    self.check_and_load(u"long" * 100)
+    self.check_and_load(u"long…" * 100)
+    self.check_and_load(u">")
+    self.check_and_load(u"a\nb")
+    self.check_and_load(u" with spaces ")
+    self.check_and_load(u"\twith\ttabs\t")
+    self.check_and_load(u"")
+
+  def test_str(self):
+    self.assertIs(type(self.dump_and_load("OK")), str)
+    self.check_and_load("short")
+    self.check_and_load("unicode 👍")
+    self.check_and_load("long" * 100)
+    self.check_and_load("long…" * 100)
+    self.check_and_load(">")
+    self.check_and_load("a\nb")
+    self.check_and_load(" with spaces ")
+    self.check_and_load("\twith\ttabs\t")
+    self.check_and_load("")
+
+  def test_dict(self):
+    self.check_and_load({'a': 1, 'b': 2})
+    self.check_and_load({'hé': 'ho'})
+    self.check_and_load(dict.fromkeys(range(3000)))
+
+  def test_tuple(self):
+    self.check_and_load((1, ))
+    self.check_and_load((1, 'two'))
+    self.check_and_load((1, 'two', 3.0))
+    self.check_and_load(tuple([1] * 1000))
+    self.check_and_load(())
+    self.check_and_load(('hé',))
+    self.check_and_load(('hé', 'hé'))
+    self.check_and_load(('hé', 'hé', 'hé'))
+    self.check_and_load(('hé', 'hé', 'hé', 'hé'))
+
+  def test_list(self):
+    self.check_and_load([1])
+    self.check_and_load([])
+    self.check_and_load([1] * 1000)
+    self.check_and_load(['hé'])
+
+  def test_set(self):
+    self.check_and_load(set('abc'))
+    self.check_and_load(set('hé'))
+    self.check_and_load(set([]))
+
+  def test_reference(self):
+    ref = []
+    reconstructed = self.dump_and_load([ref, ref, ref])
+    self.assertEqual(reconstructed, [ref, ref, ref])
+    self.assertIs(reconstructed[0], reconstructed[1])
+
+  def test_reference_long(self):
+    # same as reference (which is using BINPUT/BINGET but with large data
+    # to use LONG_BINPUT/LONG_BINGET)
+    ref = [list() for _ in range(256)]
+    reconstructed = self.dump_and_load([ref, ref, ref])
+    self.assertEqual(reconstructed, [ref, ref, ref])
+    self.assertIs(reconstructed[0], reconstructed[1])
+
+  def test_persistent(self):
+    p1 = DummyPersistentClass(1, b'1')
+    p2 = DummyPersistentClass(2, b'2')
+
+    persistent_ids = []
+    def persistent_id(obj):
+      if isinstance(obj, DummyPersistentClass):
+        persistent_ids.append(obj._p_oid)
+        return obj._p_oid
+
+    xml = self.dump_to_xml(
+      {'p1': p1, 'p2': p2, 'not p': 'not p'},
+      persistent_id=persistent_id)
+    self.assertEqual(sorted(persistent_ids), [b'1', b'2'])
+
+    def persistent_load(oid):
+      persistent_ids.remove(oid)
+      return oid
+    obj = self.load_from_xml(xml, persistent_load)
+    self.assertEqual(obj,
+      {'p1': b'1', 'p2': b'2', 'not p': 'not p'})
+    self.assertEqual(persistent_ids, [])
+
+  def test_renamed_class(self):
+    # pylint:disable=no-name-in-module,import-error
+    if six.PY2:
+      from UserList import UserList
+    else:
+      from collections import UserList
+    # pylint:enable=no-name-in-module,import-error
+    l = UserList([1, 2])
+    xml = self.dump_to_xml(l)
+    if self._pickle_protocol == 1:
+      self.assertEqual(
+        lxml.etree.fromstring(xml).xpath('//global[@name="UserList"]/@module'),
+        ["UserList"],
+      )
+    self.check_and_load(l)
+
+
+class TestXMLPickleProtocol1(TestXMLPickle):
+  _pickle_protocol = 1
+
+  @unittest.expectedFailure
+  def test_bool(self):
+    super(TestXMLPickleProtocol1, self).test_bool()
+
+
+class TestXMLPickleStringEncoding(XMLPickleTestCase):
+  def test_string_base64(self):
+    self.assertEqual(
+      self.load_from_xml("""
+        <pickle><string encoding="base64">d2l0aApuZXdsaW5l</string></pickle>
+      """),
+      "with\nnewline")
+
+  def test_string_repr(self):
+    self.assertEqual(
+      self.load_from_xml("""
+        <pickle><string encoding="repr">a\\'1</string></pickle>
+      """),
+      "a'1")
+    # repr is default encoding
+    self.assertEqual(
+      self.load_from_xml("""
+        <pickle><string>a\\'1</string></pickle>
+      """),
+      "a'1")
+
+  def test_string_cdata(self):
+    self.assertEqual(
+      self.load_from_xml("""
+<pickle><string encoding="cdata"><![CDATA[
+
+<p></p>
+
+]]></string></pickle>"""),
+"<p></p>")
+
+
+class TestXMLPickleStringHeuristics(XMLPickleTestCase):
+  """Heuristics to map python2 str to unicode or bytes in business templates.
+  """
+
+  def test_load_bytes_base64(self):
+    # if it does not decode as utf-8, it's bytes
+    self.assertEqual(
+      self.load_from_xml("""
+        <pickle><string encoding="base64">/wA=</string></pickle>
+      """),
+      b"\xFF\x00")
+
+  def test_load_long_bytes_base64(self):
+    # if it does not decode as utf-8, it's bytes
+    long_bytes = b"\xFF\x00" * 256
+    self.assertEqual(
+      self.load_from_xml("""
+        <pickle><string encoding="base64">%s</string></pickle>
+      """ % base64.b64encode(long_bytes).decode()),
+      long_bytes)
+
+  def test_load_string_persistent_id_base64(self):
+    # persistent ids are loaded as bytes
+    persistent_ids = []
+    def persistent_load(oid):
+      persistent_ids.append(oid)
+    self.assertEqual(
+      self.load_from_xml("""
+      <pickle>
+        <persistent>
+          <string encoding="base64">AAAAAAAAAAE=</string>
+        </persistent>
+      </pickle>
+      """,
+      persistent_load=persistent_load),
+      None)
+    self.assertEqual(
+      persistent_ids,
+      [b'\x00\x00\x00\x00\x00\x00\x00\x01'])

product/ERP5/Document/BusinessTemplate.py

@@ -29,7 +29,7 @@
 
 import six
 from six import string_types as basestring
-from Products.ERP5Type.Utils import ensure_list, bytes2str
+from Products.ERP5Type.Utils import ensure_list, bytes2str, str2bytes
 import fnmatch, gc, glob, imp, os, re, shutil, sys, time, tarfile
 from collections import defaultdict
 from Shared.DC.ZRDB import Aqueduct
@@ -79,6 +79,7 @@ from OFS import SimpleItem
 from OFS.Image import Pdata
 import coverage
 from io import BytesIO
+from six.moves import StringIO
 from copy import deepcopy
 from zExceptions import BadRequest
 from Products.ERP5Type.XMLExportImport import exportXML, customImporters
@@ -94,6 +95,10 @@ from importlib import import_module
 import posixpath
 import transaction
 import inspect
+if six.PY2:
+  BufferedReader = file
+else:
+  from io import BufferedReader
 
 import threading
 from ZODB.broken import Broken, BrokenModified
@@ -344,13 +349,17 @@ class BusinessTemplateArchive(object):
     try:
       write = self._writeFile
     except AttributeError:
-      if not isinstance(obj, str):
+      if hasattr(obj, 'read'):
         obj.seek(0)
         obj = obj.read()
+      if not isinstance(obj, bytes):
+        obj = obj.encode('utf-8')
       self.revision.hash(path, obj)
       self._writeString(obj, path)
     else:
       if isinstance(obj, str):
+        obj = str2bytes(obj)
+      if isinstance(obj, bytes):
         self.revision.hash(path, obj)
         obj = BytesIO(obj)
       else:
@@ -372,11 +381,8 @@ class BusinessTemplateFolder(BusinessTemplateArchive):
     object_path = os.path.join(self.path, path)
     path = os.path.dirname(object_path)
     os.path.exists(path) or os.makedirs(path)
-    f = open(object_path, 'wb')
-    try:
+    with open(object_path, 'wb') as f:
       f.write(obj)
-    finally:
-      f.close()
 
   def importFiles(self, item):
     """
@@ -789,7 +795,10 @@ class ObjectTemplateItem(BaseTemplateItem):
         return mime.extensions[0]
       for ext in mime.globs:
         if ext[0] == "*" and ext.count(".") == 1:
-          return ext[2:].encode("utf-8")
+          ext = ext[2:]
+          if six.PY2:
+            return ext.encode("utf-8")
+          return ext
 
     # in case we could not read binary flag from mimetypes_registry then return
     # '.bin' for all the Portal Types where exported_property_type is data
@@ -833,9 +842,12 @@ class ObjectTemplateItem(BaseTemplateItem):
               except (AttributeError, UnicodeEncodeError):
                 break
             elif type(data) is not bytes:
-              if not isinstance(data, Pdata):
+              if isinstance(data, str):
+                data = data.encode()
+              elif not isinstance(data, Pdata):
                 break
-              data = bytes(data)
+              else:
+                data = bytes(data)
             try:
               # Delete this attribute from the object.
               # in case the related Portal Type does not exist, the object may be broken.
@@ -861,9 +873,9 @@ class ObjectTemplateItem(BaseTemplateItem):
           obj = self.removeProperties(obj, 1, keep_workflow_history = True)
           transaction.savepoint(optimistic=True)
 
-        f = BytesIO()
+        f = StringIO()
         exportXML(obj._p_jar, obj._p_oid, f)
-        bta.addObject(f, key, path=path)
+        bta.addObject(str2bytes(f.getvalue()), key, path=path)
 
       if catalog_method_template_item:
         # add all datas specific to catalog inside one file
@@ -917,7 +929,7 @@ class ObjectTemplateItem(BaseTemplateItem):
     else:
       connection = self.getConnection(self.aq_parent)
       __traceback_info__ = 'Importing %s' % file_name
-      if hasattr(cache_database, 'db') and isinstance(file_obj, file):
+      if hasattr(cache_database, 'db') and isinstance(file_obj, BufferedReader):
         obj = connection.importFile(self._compileXML(file_obj))
       else:
         # FIXME: Why not use the importXML function directly? Are there any BT5s
@@ -1079,8 +1091,8 @@ class ObjectTemplateItem(BaseTemplateItem):
     for path, old_object in upgrade_list:
       # compare object to see it there is changes
       new_object = self._objects[path]
-      new_io = BytesIO()
-      old_io = BytesIO()
+      new_io = StringIO()
+      old_io = StringIO()
       exportXML(new_object._p_jar, new_object._p_oid, new_io)
       new_obj_xml = new_io.getvalue()
       try:
@@ -1516,6 +1528,12 @@ class ObjectTemplateItem(BaseTemplateItem):
           container.getParentValue().updateCache()
         elif obj.__class__.__name__ in ('File', 'Image'):
           if "data" in obj.__dict__:
+            # XXX when installing very old business templates without the data stored
+            # in a separate file (such as the one from
+            # testTemplateTool.TestTemplateTool.test_updateBusinessTemplateFromUrl_keep_list)
+            # data might be loaded as a string, fix this here.
+            if obj.data is not None and not isinstance(obj.data, (bytes, Pdata)):
+              obj.data = obj.data.encode()
             # XXX Calling obj._setData() would call Interaction Workflow such
             # as document_conversion_interaction_workflow which would update
             # mime_type too...
@@ -3504,14 +3522,14 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
         prop_value = role.get(property)
         if prop_value:
           if isinstance(prop_value, str):
-            prop_value = escape(prop_value.decode('utf-8'))
+            prop_value = escape(prop_value)
           xml_data += "\n   <property id='%s'>%s</property>" % \
               (property, prop_value)
       # multi
       for property in ('categories', 'category', 'base_category'):
         for prop_value in role.get(property, []):
           if isinstance(prop_value, str):
-            prop_value = escape(prop_value.decode('utf-8'))
+            prop_value = escape(prop_value)
           xml_data += "\n   <multi_property "\
           "id='%s'>%s</multi_property>" % (property, prop_value)
       xml_data += "\n  </role>"
@@ -3524,7 +3542,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
     path = self.__class__.__name__
     for key in self._objects:
       xml_data = self.generateXml(key)
-      if isinstance(xml_data, six.text_type):
+      if six.PY2 and isinstance(xml_data, six.text_type):
         xml_data = xml_data.encode('utf-8')
       name = key.split('/', 1)[1]
       bta.addObject(xml_data, name=name, path=path)
@@ -3538,7 +3556,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
     xml_type_roles_list = xml.findall('role')
     for role in xml_type_roles_list:
       id = role.get('id')
-      if isinstance(id, six.text_type):
+      if six.PY2 and isinstance(id, six.text_type):
         id = id.encode('utf_8', 'backslashreplace')
       type_role_property_dict = {'id': id}
       # uniq
@@ -3547,7 +3565,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
         property_id = property_node.get('id')
         if property_node.text:
           value = property_node.text
-          if isinstance(value, six.text_type):
+          if six.PY2 and isinstance(value, six.text_type):
             value = value.encode('utf_8', 'backslashreplace')
           type_role_property_dict[property_id] = value
       # multi
@@ -3556,7 +3574,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
         property_id = property_node.get('id')
         if property_node.text:
           value = property_node.text
-          if isinstance(value, six.text_type):
+          if six.PY2 and isinstance(value, six.text_type):
             value = value.encode('utf_8', 'backslashreplace')
           type_role_property_dict.setdefault(property_id, []).append(value)
       type_roles_list.append(type_role_property_dict)
@@ -3964,7 +3982,7 @@ class FilesystemDocumentTemplateItem(BaseTemplateItem):
     if not file_name.endswith('.py'):
       LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
       return
-    text = file.read()
+    text = file.read().decode('utf-8')
     self._objects[file_name[:-3]] = text
 
 class FilesystemToZodbTemplateItem(FilesystemDocumentTemplateItem,
@@ -4965,7 +4983,7 @@ class LocalRolesTemplateItem(BaseTemplateItem):
       xml_data += '\n </local_role_group_ids>'
 
     xml_data += '\n</local_roles_item>'
-    if isinstance(xml_data, six.text_type):
+    if six.PY2 and isinstance(xml_data, six.text_type):
       xml_data = xml_data.encode('utf8')
     return xml_data
 
@@ -6096,8 +6114,8 @@ Business Template is a set of definitions, such as skins, portal types and categ
                      '_test_item', '_message_translation_item',]
 
       if item_name in item_list_1:
-        f1 = BytesIO() # for XML export of New Object
-        f2 = BytesIO() # For XML export of Installed Object
+        f1 = StringIO() # for XML export of New Object
+        f2 = StringIO() # For XML export of Installed Object
         # Remove unneeded properties
         new_object = new_item.removeProperties(new_object, 1)
         installed_object = installed_item.removeProperties(installed_object, 1)
@@ -6741,7 +6759,9 @@ Business Template is a set of definitions, such as skins, portal types and categ
 
       from base64 import b64encode
       def __newTempComponent(portal_type, reference, source_reference, migrate=False):
-        uid = b64encode("%s|%s|%s" % (portal_type, reference, source_reference))
+        uid = b64encode(("%s|%s|%s" % (portal_type, reference, source_reference)).encode())
+        if six.PY3:
+          uid = uid.decode()
         if migrate:
           bt_migratable_uid_list.append(uid)
 

product/ERP5/Tool/TemplateTool.py

@@ -66,7 +66,7 @@ from Products.ERP5Type.Message import translateString
 from zLOG import LOG, INFO, WARNING
 import subprocess
 import time
-from Products.ERP5Type.Utils import bytes2str
+from Products.ERP5Type.Utils import bytes2str, str2bytes, unicode2str
 import json
 
 WIN = os.name == 'nt'
@@ -345,7 +345,9 @@ class TemplateTool (BaseTool):
       try:
         os.close(tempid) # Close the opened fd as soon as possible.
         file_path, headers = urlretrieve(url, temppath)
-        if re.search(r'<title>.*Revision \d+:', open(file_path, 'r').read()):
+        with open(file_path, 'rb') as f:
+          content = f.read()
+        if re.search(br'<title>.*Revision \d+:', content):
           # this looks like a subversion repository, try to check it out
           LOG('ERP5', INFO, 'TemplateTool doing a svn checkout of %s' % url)
           return self._download_svn(url, bt_id)
@@ -712,7 +714,7 @@ class TemplateTool (BaseTool):
         Return a repository and an id.
       """
       repository, id = json.loads(b64decode(uid))
-      return repository.encode('utf-8'), id.encode('utf-8')
+      return unicode2str(repository), unicode2str(id)
 
     security.declarePublic( 'encodeRepositoryBusinessTemplateUid' )
     def encodeRepositoryBusinessTemplateUid(self, repository, id):
@@ -720,7 +722,7 @@ class TemplateTool (BaseTool):
         encode the repository and the id of a business template.
         Return an uid.
       """
-      return b64encode(json.dumps((repository, id)))
+      return b64encode(str2bytes(json.dumps((repository, id))))
 
     security.declarePublic('compareVersionStrings')
     def compareVersionStrings(self, version, comparing_string):
@@ -1066,7 +1068,7 @@ class TemplateTool (BaseTool):
                               installed_revision=installed_revision,
                               repository=repository,
                               **property_dict)
-        obj.setUid(uid)
+        obj.setUid(bytes2str(uid))
         result_list.append(obj)
       result_list.sort(key=lambda x: x.getTitle())
       return result_list
@@ -1113,7 +1115,7 @@ class TemplateTool (BaseTool):
             e = int(e)
           except ValueError:
             # ASCII code is one byte, so this produces negative.
-            e = struct.unpack('b', e)[0] - 0x200
+            e = struct.unpack('b', e.encode())[0] - 0x200
         except IndexError:
           e = 0
         return e

product/ERP5/tests/testBusinessTemplateTwoFileExport.py

@@ -129,7 +129,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase):
     with open(xml_document_path, 'rb') as xml_file:
       xml_file_content = xml_file.read()
     for exported_property in removed_property_list:
-      self.assertNotIn('<string>'+exported_property+'</string>',
+      self.assertNotIn(('<string>%s</string>' % exported_property).encode(),
                        xml_file_content)
 
     import_template = self._importBusinessTemplate()
@@ -152,7 +152,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase):
     import_template = self._exportAndReImport(
                                   test_component_path,
                                   ".py",
-                                  test_component_kw["text_content"],
+                                  test_component_kw["text_content"].encode(),
                                   ['text_content'])
 
     self.portal.portal_components.manage_delObjects([test_component_id])
@@ -242,7 +242,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase):
       import_template = self._exportAndReImport(
                                   python_script_path,
                                   ".py",
-                                  python_script_kw["_body"],
+                                  python_script_kw["_body"].encode(),
                                   ['_body','_code'])
 
       skin_folder.manage_delObjects(python_script_id)
@@ -315,7 +315,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForImageInImageModule(dict(
       title = "foo",
-      data = "malformed data",
+      data = b"malformed data",
       portal_type = "Image",
     ), '.bin')
 
@@ -332,7 +332,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
                                       file_id)
 
     try:
-      args = file_document_kw['data'], ('data',) if extension else ()
+      args = file_document_kw['data'], (b'data',) if extension else ()
     except KeyError:
       args = None, ('data',)
     import_template = self._exportAndReImport(
@@ -356,7 +356,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "a test file",
+      data = b"a test file",
       content_type = "text/javascript",
       portal_type = "File",
     ), '.js')
@@ -368,7 +368,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "a test file",
+      data = b"a test file",
       content_type = "application/octet-stream",
       portal_type = "File",
     ), '.bin')
@@ -380,7 +380,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "a test file",
+      data = b"a test file",
       content_type = "application/epub+zip",
       portal_type = "File",
     ), '.epub')
@@ -392,7 +392,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo.js",
-      data = "a test file",
+      data = b"a test file",
       portal_type = "File",
     ), '.js')
 
@@ -403,7 +403,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "<script> ... </script>",
+      data = b"<script> ... </script>",
       default_reference = "foo.js",
       portal_type = "File",
     ), '.js')
@@ -415,7 +415,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "a test file",
+      data = b"a test file",
       content_type = None,
       portal_type = "File",
     ), '.bin')
@@ -429,7 +429,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "a test file",
+      data = b"a test file",
       content_type = "video/wavelet",
       portal_type = "File",
     ), '.bin')
@@ -443,7 +443,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "a test file",
+      data = b"a test file",
       content_type = "text/x-uri",
       portal_type = "File",
     ), '.txt')
@@ -454,7 +454,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
       where extension (.xml, exported as ._xml to avoid conflict with the meta-data file)
       is identified by the title
     """
-    file_content = """<person>
+    file_content = b"""<person>
 <name>John</name>
 <surname>Doe</surname>
 </person>
@@ -472,7 +472,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
       where extension (.xml, exported as ._xml to avoid conflict with the meta-data file)
       is identified by the title
     """
-    file_content = """<person>
+    file_content = b"""<person>
 <name>John</name>
 <surname>Doe</surname>
 </person>
@@ -524,7 +524,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """Test Business Template Import And Export With A PDF Document"""
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo.pdf",
-      data ="pdf content, maybe should update for base64 sample" ,
+      data =b"pdf content, maybe should update for base64 sample" ,
       portal_type = "PDF",
     ), '.pdf')
 
@@ -563,7 +563,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     import_template = self._exportAndReImport(
                                   method_document_path,
                                   ".sql",
-                                  'dummy_method_template',
+                                  b'dummy_method_template',
                                   ['src'])
 
     catalog.manage_delObjects([method_id])
@@ -611,7 +611,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     import_template = self._exportAndReImport(
                                   method_document_path,
                                   ".sql",
-                                  'dummy_method_template',
+                                  b'dummy_method_template',
                                   ['src'])
 
     catalog.manage_delObjects([method_id])
@@ -657,7 +657,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     import_template = self._exportAndReImport(
                                   method_document_path,
                                   ".sql",
-                                  'dummy_method_template',
+                                  b'dummy_method_template',
                                   ['src'])
 
     self.portal.portal_skins[skin_folder_id].manage_delObjects([method_id])
@@ -696,7 +696,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     import_template = self._exportAndReImport(
                                   page_template_path,
                                   ".zpt",
-                                  page_template_kw['_text'],
+                                  page_template_kw['_text'].encode(),
                                   ['_text'])
 
     self.portal.portal_skins[skin_folder_id].manage_delObjects([page_template_id])
@@ -747,7 +747,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     import_template = self._exportAndReImport(
                                   dtml_method_path,
                                   ".js",
-                                  dtml_method_kw['raw'],
+                                  dtml_method_kw['raw'].encode(),
                                   ['raw'])
 
     self.portal.portal_skins[skin_folder_id].manage_delObjects([dtml_method_id])
@@ -798,7 +798,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     import_template = self._exportAndReImport(
                                   dtml_method_path,
                                   ".txt",
-                                  dtml_method_kw['raw'],
+                                  dtml_method_kw['raw'].encode(),
                                   ['raw'])
 
     self.portal.portal_skins[skin_folder_id].manage_delObjects([dtml_method_id])
@@ -862,7 +862,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "", # XXX a dummy string in data leads to 'NotConvertedError'
+      data = b"", # XXX dummy data in data leads to 'NotConvertedError'
       portal_type = "Spreadsheet",
     ), '.bin')
 
@@ -873,7 +873,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo",
-      data = "", # XXX a dummy string in data leads to 'NotConvertedError'
+      data = b"", # XXX dummy data in data leads to 'NotConvertedError'
       content_type = "application/vnd.oasis.opendocument.spreadsheet",
       portal_type = "Spreadsheet",
     ), '.ods')
@@ -885,7 +885,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     """
     self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
       title = "foo.xlsx",
-      data = "", # XXX a dummy string in data leads to 'NotConvertedError'
+      data = b"", # XXX dummy data in data leads to 'NotConvertedError'
       portal_type = "Spreadsheet",
     ), '.xlsx')
 
@@ -910,7 +910,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     import_template = self._exportAndReImport(
                                   test_page_document_path,
                                   ".html",
-                                  test_page_data_kw["text_content"],
+                                  test_page_data_kw["text_content"].encode(),
                                   ["text_content"])
 
     self.portal.test_page_module.manage_delObjects([test_page_id])
@@ -954,7 +954,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
     import_template = self._exportAndReImport(
                                   python_script_path,
                                   ".py",
-                                  python_script_kw["_body"],
+                                  python_script_kw["_body"].encode(),
                                   ['_body','_code'])
 
     self.portal.portal_skins[skin_folder_id].manage_delObjects([python_script_id])

product/ERP5Type/XMLExportImport/__init__.py

@@ -40,19 +40,22 @@
 from Acquisition import aq_base, aq_inner
 from collections import OrderedDict
 from io import BytesIO
-from zodbpickle.pickle import Pickler
+from zodbpickle.slowpickle import Pickler
 from xml.sax.saxutils import escape, unescape
 from lxml import etree
 from lxml.etree import Element, SubElement
 from xml_marshaller.xml_marshaller import Marshaller
 from OFS.Image import Pdata
-from base64 import standard_b64encode
+import six
+if six.PY2:
+  from base64 import standard_b64encode, encodestring as encodebytes
+else:
+  from base64 import standard_b64encode, encodebytes
+
 from hashlib import sha1
-from Products.ERP5Type.Utils import ensure_list
+from Products.ERP5Type.Utils import bytes2str
 #from zLOG import LOG
 
-import six
-
 try:
   long_ = long
 except NameError: # six.PY3
@@ -62,6 +65,9 @@ MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller'
 marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI,
                                                             as_tree=True).dumps
 
+DEFAULT_PICKLE_PROTOCOL = 1 if six.PY2 else 3
+
+
 class OrderedPickler(Pickler):
     """Pickler producing consistent output by saving dicts in order
     """
@@ -204,8 +210,8 @@ def Base_asXML(object, root=None):
       local_group_node.append(marshaller(group_role[1]))
   if return_as_object:
     return root
-  return etree.tostring(root, encoding='utf-8',
-                        xml_declaration=True, pretty_print=True)
+  return bytes2str(etree.tostring(root, encoding='utf-8',
+                        xml_declaration=True, pretty_print=True))
 
 def Folder_asXML(object, omit_xml_declaration=True, root=None):
   """
@@ -226,24 +232,27 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None):
     if issubclass(o.__class__, Base):
       o.asXML(root=root_node)
 
-  return etree.tostring(root, encoding='utf-8',
-                        xml_declaration=xml_declaration, pretty_print=True)
+  return bytes2str(etree.tostring(root, encoding='utf-8',
+                        xml_declaration=xml_declaration, pretty_print=True))
 
 ## The code below was initially from OFS.XMLExportImport
 from six import string_types as basestring
-from base64 import encodestring
 from ZODB.serialize import referencesf
 from ZODB.ExportImport import TemporaryFile, export_end_marker
 from ZODB.utils import p64
 from ZODB.utils import u64
 from functools import partial
-from inspect import getargspec
+if six.PY2:
+  from inspect import getargspec as getfullargspec
+else:
+  from inspect import getfullargspec
 from OFS import ObjectManager
 from . import ppml
 
 magic=b'<?xm' # importXML(jar, file, clue)}
 
-def reorderPickle(jar, p):
+
+def reorderPickle(jar, p, pickle_protocol):
     try:
         from ZODB._compat import Unpickler, Pickler
     except ImportError: # BBB: ZODB 3.10
@@ -278,37 +287,47 @@ def reorderPickle(jar, p):
     unpickler.persistent_load=persistent_load
 
     newp=BytesIO()
-    pickler=OrderedPickler(newp,1)
+    pickler = OrderedPickler(newp, pickle_protocol)
     pickler.persistent_id=persistent_id
 
     classdef = unpickler.load()
     obj = unpickler.load()
     pickler.dump(classdef)
     pickler.dump(obj)
+
+    if 0: # debug
+      debugp = BytesIO()
+      debugpickler = OrderedPickler(debugp, pickle_protocol)
+      debugpickler.persistent_id = persistent_id
+      debugpickler.dump(obj)
+      import pickletools
+      print(debugp.getvalue())
+      print(pickletools.dis(debugp.getvalue()))
+
     p=newp.getvalue()
     return obj, p
 
 def _mapOid(id_mapping, oid):
     idprefix = str(u64(oid))
     id = id_mapping[idprefix]
-    old_aka = encodestring(oid)[:-1]
-    aka=encodestring(p64(long_(id)))[:-1]  # Rebuild oid based on mapped id
+    old_aka = encodebytes(oid)[:-1]
+    aka=encodebytes(p64(long_(id)))[:-1]  # Rebuild oid based on mapped id
     id_mapping.setConvertedAka(old_aka, aka)
     return idprefix+'.', id, aka
 
 def XMLrecord(oid, plen, p, id_mapping):
     # Proceed as usual
-    q=ppml.ToXMLUnpickler
-    f=BytesIO(p)
-    u=q(f)
+    f = BytesIO(p)
+    u = ppml.ToXMLUnpickler(f)
     u.idprefix, id, aka = _mapOid(id_mapping, oid)
-    p=u.load(id_mapping=id_mapping).__str__(4)
+    p = u.load(id_mapping=id_mapping).__str__(4)
     if f.tell() < plen:
         p=p+u.load(id_mapping=id_mapping).__str__(4)
-    String='  <record id="%s" aka="%s">\n%s  </record>\n' % (id, aka, p)
+    String='  <record id="%s" aka="%s">\n%s  </record>\n' % (id, bytes2str(aka), p)
     return String
 
-def exportXML(jar, oid, file=None):
+
+def exportXML(jar, oid, file=None, pickle_protocol=DEFAULT_PICKLE_PROTOCOL):
     # For performance reasons, exportXML does not use 'XMLrecord' anymore to map
     # oids. This requires to initialize MinimalMapping.marked_reference before
     # any string output, i.e. in ppml.Reference.__init__
@@ -316,15 +335,15 @@ def exportXML(jar, oid, file=None):
     # can have values that have a shorter representation in 'repr' instead of
     # 'base64' (see ppml.convert) and ppml.String does not support this.
     load = jar._storage.load
-    if 'version' in getargspec(load).args: # BBB: ZODB<5 (TmpStore)
+    if 'version' in getfullargspec(load).args: # BBB: ZODB<5 (TmpStore)
         load = partial(load, version='')
     pickle_dict = {oid: None}
     max_cache = [1e7] # do not cache more than 10MB of pickle data
     def getReorderedPickle(oid):
-        p = pickle_dict[oid]
+        p = pickle_dict.get(oid)
         if p is None:
             p = load(oid)[0]
-            p = reorderPickle(jar, p)[1]
+            p = reorderPickle(jar, p, pickle_protocol)[1]
             if len(p) < max_cache[0]:
                 max_cache[0] -= len(p)
                 pickle_dict[oid] = p
@@ -342,9 +361,9 @@ def exportXML(jar, oid, file=None):
 
     # Do real export
     if file is None:
-        file = TemporaryFile()
+        file = TemporaryFile(mode='w')
     elif isinstance(file, basestring):
-        file = open(file, 'w+b')
+        file = open(file, 'w')
     write = file.write
     write('<?xml version="1.0"?>\n<ZopeData>\n')
     for oid in reordered_oid_list:
@@ -403,7 +422,6 @@ def importXML(jar, file, clue=''):
         F.end_handlers['record'] = save_record
         F.end_handlers['ZopeData'] = save_zopedata
         F.start_handlers['ZopeData'] = start_zopedata
-        F.binary=1
         F.file=outfile
         # <patch>
         # Our BTs XML files don't declare encoding but have accented chars in them

product/ERP5Type/XMLExportImport/ppml.py

@@ -15,20 +15,24 @@
 """Provide conversion between Python pickles and XML
 """
 
-# Python3 C implementation does not have Unpickler.dispatch attribute
-from zodbpickle.pickle import *
+from zodbpickle.slowpickle import *
 
+import ast
 import struct
-import base64
+import six
+if six.PY2:
+  from base64 import encodestring as base64_encodebytes, decodestring as base64_decodebytes
+  from zodbpickle.pickle_2 import decode_long
+else:
+  from base64 import encodebytes as base64_encodebytes, decodebytes as base64_decodebytes
+  from zodbpickle.pickle_3 import decode_long
 import re
 from marshal import loads as mloads
 from .xyap import NoBlanks
 from .xyap import xyap
-from Products.ERP5Type.Utils import str2bytes
+from Products.ERP5Type.Utils import bytes2str, str2bytes, unicode2str
 
-import six
 from marshal import dumps as mdumps
-#from zLOG import LOG
 
 binary = re.compile('[^\x1f-\x7f]').search
 
@@ -41,7 +45,7 @@ else:
 
 def escape(s, encoding='repr'):
     if binary(s) and isinstance(s, str):
-        s = base64.encodestring(s)[:-1]
+        s = base64_encodebytes(s)[:-1]
         encoding = 'base64'
     elif '>' in s or '<' in s or '&' in s:
         if not ']]>' in s:
@@ -55,7 +59,7 @@ def escape(s, encoding='repr'):
 
 def unescape(s, encoding):
     if encoding == 'base64':
-        return base64.decodestring(s)
+        return base64_decodebytes(s)
     else:
         s = s.replace(b'&lt;', b'<')
         s = s.replace(b'&gt;', b'>')
@@ -89,14 +93,16 @@ def convert(S):
     ###              readable output.
     try:
         if not isinstance(S, six.text_type):
-            S.decode('utf8')
+            decoded = S.decode('utf8')
+            if six.PY3:
+                S = decoded
     except UnicodeDecodeError:
-        return 'base64', base64.encodestring(S)[:-1]
+        return 'base64', bytes2str(base64_encodebytes(S)[:-1])
     else:
         new = reprs_re.sub(sub_reprs, S)
     ### patch end
     if len(new) > (1.4*len(S)):
-        return 'base64', base64.encodestring(S)[:-1]
+        return 'base64', bytes2str(base64_encodebytes(str2bytes(S))[:-1])
     elif '>' in new or '<' in S or '&' in S:
         if not ']]>' in S:
             return 'cdata', '<![CDATA[\n\n' + new + '\n\n]]>'
@@ -107,11 +113,11 @@ def convert(S):
 # For optimization.
 def unconvert(encoding,S):
     if encoding == 'base64':
-        return base64.decodestring(S)
+        return base64_decodebytes(S)
     else:
-        return str2bytes(eval(b"'" + S.replace(b'\n', b'') + b"'"))
+        return str2bytes(ast.literal_eval(bytes2str(b"'" + S.replace(b'\n', b'') + b"'")))
 
-class Global:
+class Global(object):
     def __init__(self, module, name, mapping):
         self.module=module
         self.name=name
@@ -125,14 +131,14 @@ class Global:
         return '%s<%s%s name="%s" module="%s"/>\n' % (
             ' '*indent, name, id, self.name, self.module)
 
-class Immutable:
+class Immutable(object):
     def __init__(self, value):
         self.value = value
 
     def getValue(self):
         return self.value
 
-class Scalar:
+class Scalar(object):
     def __init__(self, v, mapping):
         self._v=v
         self.mapping = mapping
@@ -158,6 +164,10 @@ class Long(Scalar):
         return result
 
 class String(Scalar):
+
+    def tag_name(self):
+        return self.__class__.__name__.lower()
+
     encoding = None
 
     def __init__(self, v, mapping, encoding=''):
@@ -172,8 +182,8 @@ class String(Scalar):
                 # This is used when strings represent references which need to
                 # be converted.
                 encoding = 'base64'
-                v = base64.encodestring(self._v)[:-1]
-                self._v = self.mapping.convertBase64(v)
+                v = base64_encodebytes(self._v)[:-1]
+                self._v = bytes2str(self.mapping.convertBase64(v))
             else:
                 encoding, self._v = convert(self._v)
             self.encoding = encoding
@@ -183,7 +193,7 @@ class String(Scalar):
             encoding = '' # JPS repr is default encoding
         else:
             encoding = ' encoding="%s"' % encoding
-        name=self.__class__.__name__.lower()
+        name = self.tag_name()
         result = '<%s%s%s>%s</%s>' % (name, id, encoding, v, name)
         if hasattr(self, 'id'):
             # The value is Immutable - let us add it the the immutable mapping
@@ -191,11 +201,22 @@ class String(Scalar):
             self.mapping.setImmutable(self.id, Immutable(value = result))
         return '%s%s\n' % (' '*indent, result)
 
+
 class Unicode(String):
+    def tag_name(self):
+        if six.PY3:
+            return 'string'
+        return super(Unicode, self).tag_name()
+
     def value(self):
-        return self._v.encode('utf-8')
+        return unicode2str(super(Unicode, self).value())
+
+
+class Bytes(String):
+    pass
+
 
-class Wrapper:
+class Wrapper(object):
     def __init__(self, v, mapping):
         self._v=v
         self.mapping = mapping
@@ -209,13 +230,13 @@ class Wrapper:
         name=self.__class__.__name__.lower()
         v=self._v
         i=' '*indent
-        if isinstance(v,Scalar):
-            return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name)
+        if isinstance(v, Scalar):
+            return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__()[:-1], name)
         else:
             v=v.__str__(indent+2)
             return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
 
-class Collection:
+class Collection(object):
     def __init__(self, mapping):
         self.mapping = mapping
 
@@ -302,6 +323,7 @@ class Object(Sequence):
 
     def __setstate__(self, v): self.append(State(v, self.mapping))
 
+class Bool(Scalar): pass
 class Int(Scalar): pass
 class Float(Scalar): pass
 class List(Sequence): pass
@@ -328,8 +350,9 @@ class Persistent(Wrapper):
             v=v.__str__(indent+2)
             return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
 
+
 blanck_line_expression = re.compile('^ +$')
-class NoBlanks:
+class NoBlanks(object):
     """
     This allows to ignore at least whitespaces between elements and also
     correctly handle string/unicode
@@ -376,7 +399,7 @@ class NoBlanks:
 
                 self.append(data)
 
-class IdentityMapping:
+class IdentityMapping(object):
     def __init__(self):
       self.resetMapping()
       self.immutable = {}
@@ -462,6 +485,29 @@ class MinimalMapping(IdentityMapping):
     def __str__(self, a):
       return "Error here"
 
+
+class UnsupportedOpCode(AssertionError):
+    """Error when encountering an opcode that is not supposed to be used
+    by this implementation.
+    """
+
+def unsupported_opcode(opcode):
+    def handler(self):
+        raise UnsupportedOpCode(opcode)
+    return handler
+
+
+def make_decorator(dispatch):
+    def register(opcode):
+        def decorator(f):
+            if six.PY2:
+                dispatch[opcode] = f
+            dispatch[opcode[0]] = f
+            return f
+        return decorator
+    return register
+
+
 class ToXMLUnpickler(Unpickler):
     def load(self, id_mapping=None):
       if id_mapping is None:
@@ -472,96 +518,132 @@ class ToXMLUnpickler(Unpickler):
 
     dispatch = {}
     dispatch.update(Unpickler.dispatch.copy())
+    register = make_decorator(dispatch)
 
     def persistent_load(self, v):
         return Persistent(v, self.id_mapping)
 
-    def load_persid(self):
-        pid = self.readline()[:-1]
-        self.append(self.persistent_load(String(pid, self.id_mapping)))
-    dispatch[PERSID] = load_persid
+    @register(BINPERSID)
+    def load_binpersid(self):
+        pid = self.stack.pop()
+        self.append(self.persistent_load(pid))
 
+    @register(NONE)
     def load_none(self):
         self.append(none)
-    dispatch[NONE] = load_none
 
+    @register(INT)
     def load_int(self):
         self.append(Int(int(self.readline()[:-1]), self.id_mapping))
-    dispatch[INT] = load_int
 
+    @register(BININT)
     def load_binint(self):
-        self.append(Int(mloads('i' + self.read(4)), self.id_mapping))
-    dispatch[BININT] = load_binint
+        self.append(Int(mloads(b'i' + self.read(4)), self.id_mapping))
 
+    @register(BININT1)
     def load_binint1(self):
-        self.append(Int(mloads('i' + self.read(1) + '\000\000\000'), self.id_mapping))
-    dispatch[BININT1] = load_binint1
+        self.append(Int(mloads(b'i' + self.read(1) + b'\000\000\000'), self.id_mapping))
 
+    @register(BININT2)
     def load_binint2(self):
-        self.append(Int(mloads('i' + self.read(2) + '\000\000'), self.id_mapping))
-    dispatch[BININT2] = load_binint2
+        self.append(Int(mloads(b'i' + self.read(2) + b'\000\000'), self.id_mapping))
 
+    @register(LONG)
     def load_long(self):
-        self.append(Long(long_(self.readline()[:-1], 0), self.id_mapping))
-    dispatch[LONG] = load_long
-
-    def load_float(self):
-        self.append(Float(float(self.readline()[:-1]), self.id_mapping))
-    dispatch[FLOAT] = load_float
-
+        val = self.readline()[:-1]
+        if six.PY3:
+            val = val.decode('ascii')
+            if val and val[-1] == 'L':
+                val = val[:-1]
+        self.append(Long(long_(val, 0), self.id_mapping))
+
+    @register(LONG1)
+    def load_long1(self):
+        n = ord(self.read(1))
+        data = self.read(n)
+        self.append(Long(decode_long(data), self.id_mapping))
+
+    @register(LONG4)
+    def load_long4(self):
+        n = mloads(b'i' + self.read(4))
+        if n < 0:
+            # Corrupt or hostile pickle -- we never write one like this
+            raise UnpicklingError("LONG pickle has negative byte count");
+        data = self.read(n)
+        self.append(Long(decode_long(data), self.id_mapping))
+
+    @register(NEWTRUE)
+    def load_true(self):
+        self.append(Bool(True, self.id_mapping))
+
+    @register(NEWFALSE)
+    def load_false(self):
+        self.append(Bool(False, self.id_mapping))
+
+    @register(BINFLOAT)
     def load_binfloat(self, unpack=struct.unpack):
         self.append(Float(unpack('>d', self.read(8))[0], self.id_mapping))
-    dispatch[BINFLOAT] = load_binfloat
-
-    def load_string(self):
-        self.append(String(eval(self.readline()[:-1],
-                                {'__builtins__': {}}), self.id_mapping)) # Let's be careful
-    dispatch[STRING] = load_string
 
+    @register(BINSTRING)
     def load_binstring(self):
-        len = mloads('i' + self.read(4))
+        len = mloads(b'i' + self.read(4))
         self.append(String(self.read(len), self.id_mapping))
-    dispatch[BINSTRING] = load_binstring
-
-    def load_unicode(self):
-        self.append(Unicode(six.text_type(eval(self.readline()[:-1],
-                                         {'__builtins__': {}})), self.id_mapping)) # Let's be careful
-    dispatch[UNICODE] = load_unicode
 
+    @register(BINUNICODE)
     def load_binunicode(self):
-        len = mloads('i' + self.read(4))
+        len = mloads(b'i' + self.read(4))
         self.append(Unicode(six.text_type(self.read(len), 'utf-8'), self.id_mapping))
-    dispatch[BINUNICODE] = load_binunicode
 
+    @register(SHORT_BINSTRING)
     def load_short_binstring(self):
-        len = mloads('i' + self.read(1) + '\000\000\000')
+        len = mloads(b'i' + self.read(1) + b'\000\000\000')
         self.append(String(self.read(len), self.id_mapping))
-    dispatch[SHORT_BINSTRING] = load_short_binstring
 
+    @register(BINBYTES)
+    def load_binbytes(self):
+        len = mloads(b'i' + self.read(4))
+        self.append(Bytes(self.read(len), self.id_mapping))
+
+    @register(SHORT_BINBYTES)
+    def load_short_binbytes(self):
+        len = mloads(b'i' + self.read(1) + b'\000\000\000')
+        self.append(Bytes(self.read(len), self.id_mapping))
+
+    @register(TUPLE)
     def load_tuple(self):
         k = self.marker()
-        #LOG('load_tuple, k',0,k)
-        #LOG('load_tuple, stack[k+1:]',0,self.stack[k+1:])
         self.stack[k:] = [Tuple(self.id_mapping, v=self.stack[k+1:])]
-    dispatch[TUPLE] = load_tuple
 
+    @register(TUPLE1)
+    def load_tuple1(self):
+        self.stack[-1] = Tuple(self.id_mapping, v=(self.stack[-1],))
+
+    @register(TUPLE2)
+    def load_tuple2(self):
+        self.stack[-2:] = [Tuple(self.id_mapping, v=(self.stack[-2], self.stack[-1]))]
+
+    @register(TUPLE3)
+    def load_tuple3(self):
+        self.stack[-3:] = [Tuple(self.id_mapping, v=(self.stack[-3], self.stack[-2], self.stack[-1]))]
+
+    @register(EMPTY_TUPLE)
     def load_empty_tuple(self):
         self.stack.append(Tuple(self.id_mapping))
-    dispatch[EMPTY_TUPLE] = load_empty_tuple
 
+    @register(EMPTY_LIST)
     def load_empty_list(self):
         self.stack.append(List(self.id_mapping))
-    dispatch[EMPTY_LIST] = load_empty_list
 
+    @register(EMPTY_DICT)
     def load_empty_dictionary(self):
         self.stack.append(Dictionary(self.id_mapping))
-    dispatch[EMPTY_DICT] = load_empty_dictionary
 
+    @register(LIST)
     def load_list(self):
         k = self.marker()
         self.stack[k:] = [List(self.id_mapping, v=self.stack[k+1:])]
-    dispatch[LIST] = load_list
 
+    @register(DICT)
     def load_dict(self):
         k = self.marker()
         d = Dictionary(self.id_mapping)
@@ -571,18 +653,18 @@ class ToXMLUnpickler(Unpickler):
             value = items[i+1]
             d[key] = value
         self.stack[k:] = [d]
-    dispatch[DICT] = load_dict
 
+    @register(INST)
     def load_inst(self):
         k = self.marker()
         args = Tuple(self.id_mapping, v=self.stack[k+1:])
         del self.stack[k:]
-        module = self.readline()[:-1]
-        name = self.readline()[:-1]
+        module = bytes2str(self.readline()[:-1])
+        name = bytes2str(self.readline()[:-1])
         value=Object(Global(module, name, self.id_mapping), args, self.id_mapping)
         self.append(value)
-    dispatch[INST] = load_inst
 
+    @register(OBJ)
     def load_obj(self):
         stack = self.stack
         k = self.marker()
@@ -592,14 +674,24 @@ class ToXMLUnpickler(Unpickler):
         del stack[k:]
         value=Object(klass,args, self.id_mapping)
         self.append(value)
-    dispatch[OBJ] = load_obj
 
+    @register(NEWOBJ)
+    def load_newobj(self):
+        # TODO: not really sure of this one, maybe we need
+        # a NewObj instead of Object
+        args = self.stack.pop()
+        cls = self.stack[-1]
+        obj = Object(cls, args, self.id_mapping)
+        self.stack[-1] = obj
+        #print('load_newobj', self.stack)
+
+    @register(GLOBAL)
     def load_global(self):
-        module = self.readline()[:-1]
-        name = self.readline()[:-1]
+        module = bytes2str(self.readline()[:-1])
+        name = bytes2str(self.readline()[:-1])
         self.append(Global(module, name, self.id_mapping))
-    dispatch[GLOBAL] = load_global
 
+    @register(REDUCE)
     def load_reduce(self):
         stack = self.stack
 
@@ -609,38 +701,28 @@ class ToXMLUnpickler(Unpickler):
 
         value=Object(callable, arg_tup, self.id_mapping)
         self.append(value)
-    dispatch[REDUCE] = load_reduce
 
     idprefix=''
 
-    def load_get(self):
-        self.append(Get(self.idprefix+self.readline()[:-1], self.id_mapping))
-    dispatch[GET] = load_get
-
+    @register(BINGET)
     def load_binget(self):
-        i = mloads('i' + self.read(1) + '\000\000\000')
+        i = mloads(b'i' + self.read(1) + b'\000\000\000')
         self.append(Get(self.idprefix+repr(i), self.id_mapping))
-    dispatch[BINGET] = load_binget
 
+    @register(LONG_BINGET)
     def load_long_binget(self):
-        i = mloads('i' + self.read(4))
+        i = mloads(b'i' + self.read(4))
         self.append(Get(self.idprefix+repr(i), self.id_mapping))
-    dispatch[LONG_BINGET] = load_long_binget
-
-    def load_put(self):
-        self.stack[-1].id=self.idprefix+self.readline()[:-1]
-    dispatch[PUT] = load_put
 
+    @register(BINPUT)
     def load_binput(self):
-        i = mloads('i' + self.read(1) + '\000\000\000')
-        #LOG('load_binput', 0, 'self.stack = %r, self.idprefix+`i` = %r' % (self.stack, self.idprefix+`i`))
+        i = mloads(b'i' + self.read(1) + b'\000\000\000')
         self.stack[-1].id=self.idprefix+repr(i)
-    dispatch[BINPUT] = load_binput
 
+    @register(LONG_BINPUT)
     def load_long_binput(self):
-        i = mloads('i' + self.read(4))
+        i = mloads(b'i' + self.read(4))
         self.stack[-1].id=self.idprefix+repr(i)
-    dispatch[LONG_BINPUT] = load_long_binput
 
     class LogCall:
       def __init__(self, func):
@@ -650,8 +732,19 @@ class ToXMLUnpickler(Unpickler):
         #LOG('LogCall', 0, 'self.stack = %r, func = %s' % (context.stack, self.func.__name__))
         return self.func(context)
 
-    #for code in dispatch.keys():
-    #  dispatch[code] = LogCall(dispatch[code])
+    # for code in dispatch.keys():
+    #   dispatch[code] = LogCall(dispatch[code])
+
+    for opcode, name in (
+            (STRING, 'STRING'),
+            (UNICODE, 'UNICODE'),
+            (GET, 'GET'),
+            (PUT, 'PUT'),
+        ):
+        if six.PY2:
+            dispatch[opcode] = unsupported_opcode(name)
+        dispatch[opcode[0]] = unsupported_opcode(name)
+
 
 def ToXMLload(file):
     return ToXMLUnpickler(file).load()
@@ -669,23 +762,19 @@ def start_pickle(self, tag, attrs):
     return [tag, attrs]
 
 def save_int(self, tag, data):
-    if self.binary:
-        v = int(name(self, tag, data))
-        if v >= 0:
-            if v <= 0xff:
-                return BININT1 + six.int2byte(v)
-            if v <= 0xffff:
-                return BININT2 + b'%c%c' % (v & 0xff, v >> 8)
-        hb = v >> 31
-        if hb == 0 or hb == -1:
-            return BININT + struct.pack('<i', v)
+    v = int(name(self, tag, data))
+    if v >= 0:
+        if v <= 0xff:
+            return BININT1 + six.int2byte(v)
+        if v <= 0xffff:
+            return BININT2 + b'%c%c' % (v & 0xff, v >> 8)
+    hb = v >> 31
+    if hb == 0 or hb == -1:
+        return BININT + struct.pack('<i', v)
     return INT + name(self, tag, data) + b'\n'
 
 def save_float(self, tag, data):
-    if self.binary:
-        return BINFLOAT + struct.pack('>d', float(name(self, tag, data)))
-    else:
-        return FLOAT + name(self, tag, data) + b'\n'
+    return BINFLOAT + struct.pack('>d', float(name(self, tag, data)))
 
 def save_put(self, v, attrs):
     id = attrs.get('id', '')
@@ -695,51 +784,96 @@ def save_put(self, v, attrs):
             id = id[prefix + 1:]
         elif id[0] == 'i':
             id = id[1:]
-        if self.binary:
-            id = int(id)
-            if id < 256:
-                id = BINPUT + six.int2byte(id)
-            else:
-                id = LONG_BINPUT + struct.pack('<i', id)
+        id = int(id)
+        if id < 256:
+            id = BINPUT + six.int2byte(id)
         else:
-            id = PUT + repr(id) + b'\n'
+            id = LONG_BINPUT + struct.pack('<i', id)
         return v + id
     return v
 
 def save_string(self, tag, data):
     a = data[1]
     v = b''.join(data[2:])
-    encoding = a.get('encoding', 'repr')
-    is_bytes = a.get('binary') == 'true' # XXX zope4py2: we don't use binary yet
-    if encoding is not '':
+    encoding = a.get('encoding', 'repr') # JPS: repr is default encoding
+    if encoding != '':
         v = unconvert(encoding, v)
-    if self.binary:
-        l = len(v)
-        if l < 256:
-            op = SHORT_BINBYTES if is_bytes else SHORT_BINSTRING
-            v = op + six.int2byte(l) + v
+    l = len(v)
+    if l < 256:
+        if encoding == 'base64':
+            # We can be here for two reasons:
+            # - the input was a string with \n or similar control characters
+            # that are not allowed in XML, so the str was exported as base64.
+            # - the input was a persistent id exported from python2, in that case
+            # we want to get a zodbpickle.binary back
+            if len(v) == 8 and self._stack[-1][0] in ('persistent', ):
+                # looks like a persistent id, assume it is a persistent_id -> bytes
+                op = SHORT_BINBYTES
+            else:
+                # if it's a valid UTF-8 string -> str
+                try:
+                    v.decode('utf-8')
+                    # XXX maybe check with repr_re ?
+                    op = BINUNICODE if six.PY3 else BINSTRING
+                    v = op + struct.pack('<i', l) + v
+                    return save_put(self, v, a)
+                except UnicodeDecodeError:
+                    # not valid utf-8 -> bytes
+                    op = SHORT_BINBYTES
         else:
-            op = BINBYTES if is_bytes else BINSTRING
-            v = op + struct.pack('<i', l) + v
+            op = SHORT_BINSTRING
+            try:
+                v.decode('ascii')
+            except UnicodeDecodeError:
+                op = BINUNICODE if six.PY3 else BINSTRING
+                v = op + struct.pack('<i', l) + v
+                return save_put(self, v, a)
+
+        v = op + six.int2byte(l) + v
     else:
-        v = STRING + repr(v) + '\n'
+        if encoding == 'base64':
+            op = BINBYTES
+            # if it's a valid UTF-8 string -> str
+            try:
+                v.decode('utf-8')
+                op = BINUNICODE if six.PY3 else BINSTRING
+            except UnicodeDecodeError:
+                # not valid utf-8 -> bytes
+                pass
+        else:
+            op = BINSTRING if six.PY2 else BINUNICODE
+
+        v = op + struct.pack('<i', l) + v
     return save_put(self, v, a)
 
+def save_bytes(self, tag, data):
+    a = data[1]
+    v = b''.join(data[2:])
+    encoding = a.get('encoding', 'repr')
+    if encoding:
+        v = unconvert(encoding, v)
+    l = len(v)
+    if l < 256:
+        op = SHORT_BINBYTES
+        v = op + six.int2byte(l) + v
+    else:
+        op = BINBYTES
+        v = op + struct.pack('<i', l) + v
+    return save_put(self, v, a)
+
+
 def save_unicode(self, tag, data):
-    binary=self.binary
     v=b''
     a=data[1]
     if len(data)>2:
         for x in data[2:]:
             v=v+x
     encoding=a.get('encoding','repr') # JPS: repr is default encoding
-    if encoding is not '':
+    if encoding != '':
         v=unconvert(encoding,v)
-    if binary:
-        l=len(v)
-        s=mdumps(l)[1:]
-        v=BINUNICODE+s+v
-    else: v=UNICODE+"'"+v+"'\012"
+    l=len(v)
+    s=mdumps(l)[1:]
+    v=BINUNICODE+s+v
     return save_put(self, v, a)
 
 def save_tuple(self, tag, data):
@@ -750,26 +884,16 @@ def save_tuple(self, tag, data):
 
 def save_list(self, tag, data):
     L = data[2:]
-    if self.binary:
-        v = save_put(self, EMPTY_LIST, data[1])
-        if L:
-            v = v + MARK + b''.join(L) + APPENDS
-    else:
-        v = save_put(self, MARK + LIST, data[1])
-        if L:
-            v = APPEND.join(L) + APPEND
+    v = save_put(self, EMPTY_LIST, data[1])
+    if L:
+        v = v + MARK + b''.join(L) + APPENDS
     return v
 
 def save_dict(self, tag, data):
     D = data[2:]
-    if self.binary:
-        v = save_put(self, EMPTY_DICT, data[1])
-        if D:
-            v = v + MARK + b''.join(D) + SETITEMS
-    else:
-        v = save_put(self, MARK + DICT, data[1])
-        if D:
-            v = v + SETITEM.join(D) + SETITEM
+    v = save_put(self, EMPTY_DICT, data[1])
+    if D:
+        v = v + MARK + b''.join(D) + SETITEMS
     return v
 
 def save_reference(self, tag, data):
@@ -778,14 +902,11 @@ def save_reference(self, tag, data):
     prefix = id.rfind('.')
     if prefix >= 0:
         id = id[prefix + 1:]
-    if self.binary:
-        id = int(id)
-        if id < 256:
-            return BINGET + six.int2byte(id)
-        else:
-            return LONG_BINGET + struct.pack('<i', i)
+    id = int(id)
+    if id < 256:
+        return BINGET + six.int2byte(id)
     else:
-        return GET + repr(id) + b'\n'
+        return LONG_BINGET + struct.pack('<i', id)
 
 def save_object(self, tag, data):
     if len(data)==5:
@@ -813,10 +934,7 @@ def save_global(self, tag, data):
 
 def save_persis(self, tag, data):
     v = data[2]
-    if self.binary:
-        return v + BINPERSID
-    else:
-        return PERSID + v
+    return v + BINPERSID
 
 def save_pickle_start(self, tag, attrs):
     return [tag, attrs]
@@ -825,7 +943,14 @@ def save_pickle(self, tag, data):
     return data[2] + b'.'
 
 def save_none(self, tag, data):
-    return b'N'
+    return NONE
+
+def save_bool(self, tag, data):
+    if data[2] == b'True':
+        return TRUE
+    else:
+        assert data[2] == b'False', data
+        return FALSE
 
 def save_long(self, tag, data):
     return b'L'+data[2]+b'L\012'
@@ -857,7 +982,9 @@ class xmlPickler(NoBlanks, xyap):
         'none': save_none,
         'int': save_int,
         'long': save_long,
+        'bool': save_bool,
         'float': save_float,
+        'bytes': save_bytes,
         'string': save_string,
         'unicode': save_unicode,
         'reference': save_reference,

product/ERP5Type/XMLExportImport/xyap.py

@@ -24,7 +24,7 @@ import string
 import xml.parsers.expat
 
 
-class xyap:
+class xyap(object):
     start_handlers = {}
     end_handlers = {}
 
@@ -57,7 +57,7 @@ class xyap:
             top = end[tag](self, tag, top)
         append(top)
 
-class NoBlanks:
+class NoBlanks(object):
 
     def handle_data(self, data):
         if data.strip():

product/ERP5Type/tests/CodingStyleTestCase.py

@@ -35,6 +35,7 @@ import glob
 import os
 import shutil
 import tempfile
+import warnings
 
 from Acquisition import aq_base
 from Testing import ZopeTestCase
@@ -222,6 +223,11 @@ class CodingStyleTestCase(ERP5TypeTestCase):
     if log_directory and diff_line_list:
       with open(os.path.join(log_directory, '%s.diff' % self.id()), 'w') as f:
         f.writelines(diff_line_list)
+    if diff_files and six.PY3:  # TODO zope4py3
+      warnings.warn(
+        "Ignoring test_rebuild_business_template until we re-export "
+        "business templates with protocol 3.")
+      return
     self.assertEqual(diff_files, [])