Skip to content

erp5
erp5
Commit d25f925b authored by Jérome Perrin's avatar Jérome Perrin Committed by Arnaud Fontaine
Browse files
Options

feat/bt-py3 branch (MR 1979): BusinessTemplate,XMLExportImport: support python3 

With these changes, we are able to install py2 business templates on py3,
but export is slightly different, because we already export using pickle
protocol 3 on py3.

To be able to install py2 business templates, we included heuristics to
guess the str or bytes from business template XML: oids are bytes and
also some strings that do not decode to UTF-8, so that we can install
python2 business templates on py3.

When exporting business templates, we need to build a list of referenced
persistent objects to export them separately in the XML, this is is done
using Unpickler.noload, in a way which does not support pickle protocol
1 on py3 (the persistent ids are None and the assertion in
https://github.com/zopefoundation/ZODB/blob/d698507bb89eeb38c6e655199bc9f54c909dbf4d/src/ZODB/serialize.py#L669
fails), so we need to use pickle protocol 3 on py3.

In the future, we might switch to exporting on protocol 3 on py2 as well
so that we have stable output on both py2 and py3, or maybe we'll do
this only when we stop supporting py2.
parent d99521e1
Show whitespace changes
Inline Side-by-side
Showing with 715 additions and 276 deletions
bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py
View file @ d25f925b
############################################################################## ##############################################################################
# # coding: utf-8
# Copyright (c) 2008 Nexedi SA and Contributors. All Rights Reserved. # Copyright (c) 2008 Nexedi SA and Contributors. All Rights Reserved.
# TAHARA Yusei <yusei@nexedi.com> # TAHARA Yusei <yusei@nexedi.com>
# #
...@@ -26,13 +26,16 @@ ...@@ -26,13 +26,16 @@
# #
############################################################################## ##############################################################################
import base64
import unittest import unittest
import pickle import zodbpickle
import zodbpickle.fastpickle as pickle
import re import re
import xml.sax from io import BytesIO
from StringIO import StringIO from six import StringIO
from Products.ERP5Type.XMLExportImport import importXML, ppml
from Products.ERP5Type.XMLExportImport import ppml import six
import lxml.etree
class DummyClass: class DummyClass:
...@@ -44,11 +47,58 @@ class DummyClass: ...@@ -44,11 +47,58 @@ class DummyClass:
self.data = [] self.data = []
class TestXMLPickle(unittest.TestCase): class DummyPersistentClass:
def __init__(self, v, oid):
self.v = v
self._p_oid = oid
class XMLPickleTestCase(unittest.TestCase):
_pickle_protocol = 3
def dump_to_xml(self, obj, persistent_id=None):
f = BytesIO()
pickler = pickle.Pickler(f, protocol=self._pickle_protocol)
if persistent_id:
pickler.persistent_id = persistent_id
pickler.dump(obj)
f.seek(0)
xml = ppml.ToXMLUnpickler(f).load(id_mapping=ppml.MinimalMapping()).__str__()
self.assertIsInstance(xml, str)
return xml
def load_from_xml(self, xml_string, persistent_load=None):
assertEqual = self.assertEqual
class DummyJar:
loaded = None
"""follow interface expected by importXML"""
def importFile(self, file_, clue):
assertEqual(clue, 'ignored')
assertEqual(file_.read(4), b'ZEXP')
unpickler = pickle.Unpickler(file_)
if persistent_load:
unpickler.persistent_load = persistent_load
self.loaded = unpickler.load()
jar = DummyJar()
xml_string = '<?xml version="1.0"?>\n<ZopeData>%s</ZopeData>' % xml_string
importXML(jar, StringIO(xml_string), clue='ignored')
return jar.loaded
def dump_and_load(self, obj):
return self.load_from_xml(self.dump_to_xml(obj))
def check_and_load(self, v):
reconstructed = self.dump_and_load(v)
self.assertEqual(reconstructed, v)
self.assertIs(type(reconstructed), type(v))
class TestXMLPickle(XMLPickleTestCase):
def test_reduce(self): def test_reduce(self):
""" """
Make sure that a object which uses reduce for pickling can be pickled by xml pickler. Make sure that a object which uses reduce for pickling can be pickled by xml pickler.
This also covers the case of instances
""" """
obj = DummyClass() obj = DummyClass()
obj.data.append(1) obj.data.append(1)
...@@ -58,24 +108,7 @@ class TestXMLPickle(unittest.TestCase): ...@@ -58,24 +108,7 @@ class TestXMLPickle(unittest.TestCase):
pattern = re.compile('WAA') # regex pattern object uses reduce.(See sre.py) pattern = re.compile('WAA') # regex pattern object uses reduce.(See sre.py)
obj.data.append(pattern) obj.data.append(pattern)
pickled_string = pickle.dumps(obj) reconstructed_obj = self.dump_and_load(obj)
f = StringIO(pickled_string)
xmldata = str(ppml.ToXMLUnpickler(f).load())
output = StringIO()
F=ppml.xmlPickler()
F.file = output
F.binary = 1
content_handler = xml.sax.handler.ContentHandler()
content_handler.startElement = F.unknown_starttag
content_handler.endElement = F.unknown_endtag
content_handler.characters = F.handle_data
xml.sax.parseString(xmldata, content_handler)
reconstructed_pickled_data = F._stack[0][0]
reconstructed_obj = pickle.loads(reconstructed_pickled_data)
self.assertTrue(reconstructed_obj.__class__ is DummyClass) self.assertTrue(reconstructed_obj.__class__ is DummyClass)
self.assertTrue(type(getattr(reconstructed_obj, 'data', None)) is list) # pylint:disable=unidiomatic-typecheck self.assertTrue(type(getattr(reconstructed_obj, 'data', None)) is list) # pylint:disable=unidiomatic-typecheck
...@@ -84,3 +117,236 @@ class TestXMLPickle(unittest.TestCase): ...@@ -84,3 +117,236 @@ class TestXMLPickle(unittest.TestCase):
self.assertTrue(reconstructed_obj.data[2] is reconstructed_obj.data) self.assertTrue(reconstructed_obj.data[2] is reconstructed_obj.data)
self.assertTrue(type(reconstructed_obj.data[3]) is type(pattern)) self.assertTrue(type(reconstructed_obj.data[3]) is type(pattern))
self.assertEqual(reconstructed_obj.data[3].pattern, 'WAA') self.assertEqual(reconstructed_obj.data[3].pattern, 'WAA')
def test_bool(self):
self.assertIs(self.dump_and_load(True), True)
self.assertIs(self.dump_and_load(False), False)
def test_int(self):
self.check_and_load(-0)
self.check_and_load(1)
self.check_and_load(-1)
self.check_and_load(0xff)
self.check_and_load(0xff1)
self.check_and_load(0xffff)
self.check_and_load(2**128)
# long4
# https://github.com/python/cpython/blob/4d4a6f1b/Lib/test/pickletester.py#L2049-L2050
self.check_and_load(12345678910111213141516178920 << (256*8))
if six.PY2:
def test_long(self):
# pylint:disable=undefined-variable
self.check_and_load(long(-0))
self.check_and_load(long(1))
self.check_and_load(long(-1))
self.check_and_load(long(0xff))
self.check_and_load(long(0xff1))
self.check_and_load(long(0xffff))
self.check_and_load(long(2**128))
self.check_and_load(12345678910111213141516178920 << (256*8))
# pylint:enable=undefined-variable
def test_float(self):
self.check_and_load(-0.0)
self.check_and_load(1.0)
self.check_and_load(-1.0)
self.check_and_load(.33)
def test_None(self):
self.assertIs(
self.dump_and_load(None), None)
def test_bytes(self):
self.check_and_load(b"bytes")
self.check_and_load(b"long bytes" * 100)
if six.PY3 or self._pickle_protocol > 1:
# protocol 1 does not keep bytes
self.check_and_load(zodbpickle.binary(b"bytes"))
self.check_and_load(zodbpickle.binary(b""))
def test_unicode(self): # BBB PY2
self.assertIs(type(self.dump_and_load(u"OK")), six.text_type)
self.check_and_load(u"short")
self.check_and_load(u"unicode 👍")
self.check_and_load(u"long" * 100)
self.check_and_load(u"long…" * 100)
self.check_and_load(u">")
self.check_and_load(u"a\nb")
self.check_and_load(u" with spaces ")
self.check_and_load(u"\twith\ttabs\t")
self.check_and_load(u"")
def test_str(self):
self.assertIs(type(self.dump_and_load("OK")), str)
self.check_and_load("short")
self.check_and_load("unicode 👍")
self.check_and_load("long" * 100)
self.check_and_load("long…" * 100)
self.check_and_load(">")
self.check_and_load("a\nb")
self.check_and_load(" with spaces ")
self.check_and_load("\twith\ttabs\t")
self.check_and_load("")
def test_dict(self):
self.check_and_load({'a': 1, 'b': 2})
self.check_and_load({'hé': 'ho'})
self.check_and_load(dict.fromkeys(range(3000)))
def test_tuple(self):
self.check_and_load((1, ))
self.check_and_load((1, 'two'))
self.check_and_load((1, 'two', 3.0))
self.check_and_load(tuple([1] * 1000))
self.check_and_load(())
self.check_and_load(('hé',))
self.check_and_load(('hé', 'hé'))
self.check_and_load(('hé', 'hé', 'hé'))
self.check_and_load(('hé', 'hé', 'hé', 'hé'))
def test_list(self):
self.check_and_load([1])
self.check_and_load([])
self.check_and_load([1] * 1000)
self.check_and_load(['hé'])
def test_set(self):
self.check_and_load(set('abc'))
self.check_and_load(set('hé'))
self.check_and_load(set([]))
def test_reference(self):
ref = []
reconstructed = self.dump_and_load([ref, ref, ref])
self.assertEqual(reconstructed, [ref, ref, ref])
self.assertIs(reconstructed[0], reconstructed[1])
def test_reference_long(self):
# same as reference (which is using BINPUT/BINGET but with large data
# to use LONG_BINPUT/LONG_BINGET)
ref = [list() for _ in range(256)]
reconstructed = self.dump_and_load([ref, ref, ref])
self.assertEqual(reconstructed, [ref, ref, ref])
self.assertIs(reconstructed[0], reconstructed[1])
def test_persistent(self):
p1 = DummyPersistentClass(1, b'1')
p2 = DummyPersistentClass(2, b'2')
persistent_ids = []
def persistent_id(obj):
if isinstance(obj, DummyPersistentClass):
persistent_ids.append(obj._p_oid)
return obj._p_oid
xml = self.dump_to_xml(
{'p1': p1, 'p2': p2, 'not p': 'not p'},
persistent_id=persistent_id)
self.assertEqual(sorted(persistent_ids), [b'1', b'2'])
def persistent_load(oid):
persistent_ids.remove(oid)
return oid
obj = self.load_from_xml(xml, persistent_load)
self.assertEqual(obj,
{'p1': b'1', 'p2': b'2', 'not p': 'not p'})
self.assertEqual(persistent_ids, [])
def test_renamed_class(self):
# pylint:disable=no-name-in-module,import-error
if six.PY2:
from UserList import UserList
else:
from collections import UserList
# pylint:enable=no-name-in-module,import-error
l = UserList([1, 2])
xml = self.dump_to_xml(l)
if self._pickle_protocol == 1:
self.assertEqual(
lxml.etree.fromstring(xml).xpath('//global[@name="UserList"]/@module'),
["UserList"],
)
self.check_and_load(l)
class TestXMLPickleProtocol1(TestXMLPickle):
_pickle_protocol = 1
@unittest.expectedFailure
def test_bool(self):
super(TestXMLPickleProtocol1, self).test_bool()
class TestXMLPickleStringEncoding(XMLPickleTestCase):
def test_string_base64(self):
self.assertEqual(
self.load_from_xml("""
<pickle><string encoding="base64">d2l0aApuZXdsaW5l</string></pickle>
"""),
"with\nnewline")
def test_string_repr(self):
self.assertEqual(
self.load_from_xml("""
<pickle><string encoding="repr">a\\'1</string></pickle>
"""),
"a'1")
# repr is default encoding
self.assertEqual(
self.load_from_xml("""
<pickle><string>a\\'1</string></pickle>
"""),
"a'1")
def test_string_cdata(self):
self.assertEqual(
self.load_from_xml("""
<pickle><string encoding="cdata"><![CDATA[
<p></p>
]]></string></pickle>"""),
"<p></p>")
class TestXMLPickleStringHeuristics(XMLPickleTestCase):
"""Heuristics to map python2 str to unicode or bytes in business templates.
"""
def test_load_bytes_base64(self):
# if it does not decode as utf-8, it's bytes
self.assertEqual(
self.load_from_xml("""
<pickle><string encoding="base64">/wA=</string></pickle>
"""),
b"\xFF\x00")
def test_load_long_bytes_base64(self):
# if it does not decode as utf-8, it's bytes
long_bytes = b"\xFF\x00" * 256
self.assertEqual(
self.load_from_xml("""
<pickle><string encoding="base64">%s</string></pickle>
""" % base64.b64encode(long_bytes).decode()),
long_bytes)
def test_load_string_persistent_id_base64(self):
# persistent ids are loaded as bytes
persistent_ids = []
def persistent_load(oid):
persistent_ids.append(oid)
self.assertEqual(
self.load_from_xml("""
<pickle>
<persistent>
<string encoding="base64">AAAAAAAAAAE=</string>
</persistent>
</pickle>
""",
persistent_load=persistent_load),
None)
self.assertEqual(
persistent_ids,
[b'\x00\x00\x00\x00\x00\x00\x00\x01'])
product/ERP5/Document/BusinessTemplate.py
View file @ d25f925b
...@@ -29,7 +29,7 @@ ...@@ -29,7 +29,7 @@
import six import six
from six import string_types as basestring from six import string_types as basestring
from Products.ERP5Type.Utils import ensure_list, bytes2str from Products.ERP5Type.Utils import ensure_list, bytes2str, str2bytes
import fnmatch, gc, glob, imp, os, re, shutil, sys, time, tarfile import fnmatch, gc, glob, imp, os, re, shutil, sys, time, tarfile
from collections import defaultdict from collections import defaultdict
from Shared.DC.ZRDB import Aqueduct from Shared.DC.ZRDB import Aqueduct
...@@ -79,6 +79,7 @@ from OFS import SimpleItem ...@@ -79,6 +79,7 @@ from OFS import SimpleItem
from OFS.Image import Pdata from OFS.Image import Pdata
import coverage import coverage
from io import BytesIO from io import BytesIO
from six.moves import StringIO
from copy import deepcopy from copy import deepcopy
from zExceptions import BadRequest from zExceptions import BadRequest
from Products.ERP5Type.XMLExportImport import exportXML, customImporters from Products.ERP5Type.XMLExportImport import exportXML, customImporters
...@@ -94,6 +95,10 @@ from importlib import import_module ...@@ -94,6 +95,10 @@ from importlib import import_module
import posixpath import posixpath
import transaction import transaction
import inspect import inspect
if six.PY2:
BufferedReader = file
else:
from io import BufferedReader
import threading import threading
from ZODB.broken import Broken, BrokenModified from ZODB.broken import Broken, BrokenModified
...@@ -344,13 +349,17 @@ class BusinessTemplateArchive(object): ...@@ -344,13 +349,17 @@ class BusinessTemplateArchive(object):
try: try:
write = self._writeFile write = self._writeFile
except AttributeError: except AttributeError:
if not isinstance(obj, str): if hasattr(obj, 'read'):
obj.seek(0) obj.seek(0)
obj = obj.read() obj = obj.read()
if not isinstance(obj, bytes):
obj = obj.encode('utf-8')
self.revision.hash(path, obj) self.revision.hash(path, obj)
self._writeString(obj, path) self._writeString(obj, path)
else: else:
if isinstance(obj, str): if isinstance(obj, str):
obj = str2bytes(obj)
if isinstance(obj, bytes):
self.revision.hash(path, obj) self.revision.hash(path, obj)
obj = BytesIO(obj) obj = BytesIO(obj)
else: else:
...@@ -372,11 +381,8 @@ class BusinessTemplateFolder(BusinessTemplateArchive): ...@@ -372,11 +381,8 @@ class BusinessTemplateFolder(BusinessTemplateArchive):
object_path = os.path.join(self.path, path) object_path = os.path.join(self.path, path)
path = os.path.dirname(object_path) path = os.path.dirname(object_path)
os.path.exists(path) or os.makedirs(path) os.path.exists(path) or os.makedirs(path)
f = open(object_path, 'wb') with open(object_path, 'wb') as f:
try:
f.write(obj) f.write(obj)
finally:
f.close()
def importFiles(self, item): def importFiles(self, item):
""" """
...@@ -789,7 +795,10 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -789,7 +795,10 @@ class ObjectTemplateItem(BaseTemplateItem):
return mime.extensions[0] return mime.extensions[0]
for ext in mime.globs: for ext in mime.globs:
if ext[0] == "*" and ext.count(".") == 1: if ext[0] == "*" and ext.count(".") == 1:
return ext[2:].encode("utf-8") ext = ext[2:]
if six.PY2:
return ext.encode("utf-8")
return ext
# in case we could not read binary flag from mimetypes_registry then return # in case we could not read binary flag from mimetypes_registry then return
# '.bin' for all the Portal Types where exported_property_type is data # '.bin' for all the Portal Types where exported_property_type is data
...@@ -833,8 +842,11 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -833,8 +842,11 @@ class ObjectTemplateItem(BaseTemplateItem):
except (AttributeError, UnicodeEncodeError): except (AttributeError, UnicodeEncodeError):
break break
elif type(data) is not bytes: elif type(data) is not bytes:
if not isinstance(data, Pdata): if isinstance(data, str):
data = data.encode()
elif not isinstance(data, Pdata):
break break
else:
data = bytes(data) data = bytes(data)
try: try:
# Delete this attribute from the object. # Delete this attribute from the object.
...@@ -861,9 +873,9 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -861,9 +873,9 @@ class ObjectTemplateItem(BaseTemplateItem):
obj = self.removeProperties(obj, 1, keep_workflow_history = True) obj = self.removeProperties(obj, 1, keep_workflow_history = True)
transaction.savepoint(optimistic=True) transaction.savepoint(optimistic=True)
f = BytesIO() f = StringIO()
exportXML(obj._p_jar, obj._p_oid, f) exportXML(obj._p_jar, obj._p_oid, f)
bta.addObject(f, key, path=path) bta.addObject(str2bytes(f.getvalue()), key, path=path)
if catalog_method_template_item: if catalog_method_template_item:
# add all datas specific to catalog inside one file # add all datas specific to catalog inside one file
...@@ -917,7 +929,7 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -917,7 +929,7 @@ class ObjectTemplateItem(BaseTemplateItem):
else: else:
connection = self.getConnection(self.aq_parent) connection = self.getConnection(self.aq_parent)
__traceback_info__ = 'Importing %s' % file_name __traceback_info__ = 'Importing %s' % file_name
if hasattr(cache_database, 'db') and isinstance(file_obj, file): if hasattr(cache_database, 'db') and isinstance(file_obj, BufferedReader):
obj = connection.importFile(self._compileXML(file_obj)) obj = connection.importFile(self._compileXML(file_obj))
else: else:
# FIXME: Why not use the importXML function directly? Are there any BT5s # FIXME: Why not use the importXML function directly? Are there any BT5s
...@@ -1079,8 +1091,8 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -1079,8 +1091,8 @@ class ObjectTemplateItem(BaseTemplateItem):
for path, old_object in upgrade_list: for path, old_object in upgrade_list:
# compare object to see it there is changes # compare object to see it there is changes
new_object = self._objects[path] new_object = self._objects[path]
new_io = BytesIO() new_io = StringIO()
old_io = BytesIO() old_io = StringIO()
exportXML(new_object._p_jar, new_object._p_oid, new_io) exportXML(new_object._p_jar, new_object._p_oid, new_io)
new_obj_xml = new_io.getvalue() new_obj_xml = new_io.getvalue()
try: try:
...@@ -1516,6 +1528,12 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -1516,6 +1528,12 @@ class ObjectTemplateItem(BaseTemplateItem):
container.getParentValue().updateCache() container.getParentValue().updateCache()
elif obj.__class__.__name__ in ('File', 'Image'): elif obj.__class__.__name__ in ('File', 'Image'):
if "data" in obj.__dict__: if "data" in obj.__dict__:
# XXX when installing very old business templates without the data stored
# in a separate file (such as the one from
# testTemplateTool.TestTemplateTool.test_updateBusinessTemplateFromUrl_keep_list)
# data might be loaded as a string, fix this here.
if obj.data is not None and not isinstance(obj.data, (bytes, Pdata)):
obj.data = obj.data.encode()
# XXX Calling obj._setData() would call Interaction Workflow such # XXX Calling obj._setData() would call Interaction Workflow such
# as document_conversion_interaction_workflow which would update # as document_conversion_interaction_workflow which would update
# mime_type too... # mime_type too...
...@@ -3504,14 +3522,14 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem): ...@@ -3504,14 +3522,14 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
prop_value = role.get(property) prop_value = role.get(property)
if prop_value: if prop_value:
if isinstance(prop_value, str): if isinstance(prop_value, str):
prop_value = escape(prop_value.decode('utf-8')) prop_value = escape(prop_value)
xml_data += "\n <property id='%s'>%s</property>" % \ xml_data += "\n <property id='%s'>%s</property>" % \
(property, prop_value) (property, prop_value)
# multi # multi
for property in ('categories', 'category', 'base_category'): for property in ('categories', 'category', 'base_category'):
for prop_value in role.get(property, []): for prop_value in role.get(property, []):
if isinstance(prop_value, str): if isinstance(prop_value, str):
prop_value = escape(prop_value.decode('utf-8')) prop_value = escape(prop_value)
xml_data += "\n <multi_property "\ xml_data += "\n <multi_property "\
"id='%s'>%s</multi_property>" % (property, prop_value) "id='%s'>%s</multi_property>" % (property, prop_value)
xml_data += "\n </role>" xml_data += "\n </role>"
...@@ -3524,7 +3542,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem): ...@@ -3524,7 +3542,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
path = self.__class__.__name__ path = self.__class__.__name__
for key in self._objects: for key in self._objects:
xml_data = self.generateXml(key) xml_data = self.generateXml(key)
if isinstance(xml_data, six.text_type): if six.PY2 and isinstance(xml_data, six.text_type):
xml_data = xml_data.encode('utf-8') xml_data = xml_data.encode('utf-8')
name = key.split('/', 1)[1] name = key.split('/', 1)[1]
bta.addObject(xml_data, name=name, path=path) bta.addObject(xml_data, name=name, path=path)
...@@ -3538,7 +3556,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem): ...@@ -3538,7 +3556,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
xml_type_roles_list = xml.findall('role') xml_type_roles_list = xml.findall('role')
for role in xml_type_roles_list: for role in xml_type_roles_list:
id = role.get('id') id = role.get('id')
if isinstance(id, six.text_type): if six.PY2 and isinstance(id, six.text_type):
id = id.encode('utf_8', 'backslashreplace') id = id.encode('utf_8', 'backslashreplace')
type_role_property_dict = {'id': id} type_role_property_dict = {'id': id}
# uniq # uniq
...@@ -3547,7 +3565,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem): ...@@ -3547,7 +3565,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
property_id = property_node.get('id') property_id = property_node.get('id')
if property_node.text: if property_node.text:
value = property_node.text value = property_node.text
if isinstance(value, six.text_type): if six.PY2 and isinstance(value, six.text_type):
value = value.encode('utf_8', 'backslashreplace') value = value.encode('utf_8', 'backslashreplace')
type_role_property_dict[property_id] = value type_role_property_dict[property_id] = value
# multi # multi
...@@ -3556,7 +3574,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem): ...@@ -3556,7 +3574,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
property_id = property_node.get('id') property_id = property_node.get('id')
if property_node.text: if property_node.text:
value = property_node.text value = property_node.text
if isinstance(value, six.text_type): if six.PY2 and isinstance(value, six.text_type):
value = value.encode('utf_8', 'backslashreplace') value = value.encode('utf_8', 'backslashreplace')
type_role_property_dict.setdefault(property_id, []).append(value) type_role_property_dict.setdefault(property_id, []).append(value)
type_roles_list.append(type_role_property_dict) type_roles_list.append(type_role_property_dict)
...@@ -3964,7 +3982,7 @@ class FilesystemDocumentTemplateItem(BaseTemplateItem): ...@@ -3964,7 +3982,7 @@ class FilesystemDocumentTemplateItem(BaseTemplateItem):
if not file_name.endswith('.py'): if not file_name.endswith('.py'):
LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, )) LOG('Business Template', 0, 'Skipping file "%s"' % (file_name, ))
return return
text = file.read() text = file.read().decode('utf-8')
self._objects[file_name[:-3]] = text self._objects[file_name[:-3]] = text
class FilesystemToZodbTemplateItem(FilesystemDocumentTemplateItem, class FilesystemToZodbTemplateItem(FilesystemDocumentTemplateItem,
...@@ -4965,7 +4983,7 @@ class LocalRolesTemplateItem(BaseTemplateItem): ...@@ -4965,7 +4983,7 @@ class LocalRolesTemplateItem(BaseTemplateItem):
xml_data += '\n </local_role_group_ids>' xml_data += '\n </local_role_group_ids>'
xml_data += '\n</local_roles_item>' xml_data += '\n</local_roles_item>'
if isinstance(xml_data, six.text_type): if six.PY2 and isinstance(xml_data, six.text_type):
xml_data = xml_data.encode('utf8') xml_data = xml_data.encode('utf8')
return xml_data return xml_data
...@@ -6096,8 +6114,8 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6096,8 +6114,8 @@ Business Template is a set of definitions, such as skins, portal types and categ
'_test_item', '_message_translation_item',] '_test_item', '_message_translation_item',]
if item_name in item_list_1: if item_name in item_list_1:
f1 = BytesIO() # for XML export of New Object f1 = StringIO() # for XML export of New Object
f2 = BytesIO() # For XML export of Installed Object f2 = StringIO() # For XML export of Installed Object
# Remove unneeded properties # Remove unneeded properties
new_object = new_item.removeProperties(new_object, 1) new_object = new_item.removeProperties(new_object, 1)
installed_object = installed_item.removeProperties(installed_object, 1) installed_object = installed_item.removeProperties(installed_object, 1)
...@@ -6741,7 +6759,9 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6741,7 +6759,9 @@ Business Template is a set of definitions, such as skins, portal types and categ
from base64 import b64encode from base64 import b64encode
def __newTempComponent(portal_type, reference, source_reference, migrate=False): def __newTempComponent(portal_type, reference, source_reference, migrate=False):
uid = b64encode("%s|%s|%s" % (portal_type, reference, source_reference)) uid = b64encode(("%s|%s|%s" % (portal_type, reference, source_reference)).encode())
if six.PY3:
uid = uid.decode()
if migrate: if migrate:
bt_migratable_uid_list.append(uid) bt_migratable_uid_list.append(uid)
......
product/ERP5/Tool/TemplateTool.py
View file @ d25f925b
...@@ -66,7 +66,7 @@ from Products.ERP5Type.Message import translateString ...@@ -66,7 +66,7 @@ from Products.ERP5Type.Message import translateString
from zLOG import LOG, INFO, WARNING from zLOG import LOG, INFO, WARNING
import subprocess import subprocess
import time import time
from Products.ERP5Type.Utils import bytes2str from Products.ERP5Type.Utils import bytes2str, str2bytes, unicode2str
import json import json
WIN = os.name == 'nt' WIN = os.name == 'nt'
...@@ -345,7 +345,9 @@ class TemplateTool (BaseTool): ...@@ -345,7 +345,9 @@ class TemplateTool (BaseTool):
try: try:
os.close(tempid) # Close the opened fd as soon as possible. os.close(tempid) # Close the opened fd as soon as possible.
file_path, headers = urlretrieve(url, temppath) file_path, headers = urlretrieve(url, temppath)
if re.search(r'<title>.*Revision \d+:', open(file_path, 'r').read()): with open(file_path, 'rb') as f:
content = f.read()
if re.search(br'<title>.*Revision \d+:', content):
# this looks like a subversion repository, try to check it out # this looks like a subversion repository, try to check it out
LOG('ERP5', INFO, 'TemplateTool doing a svn checkout of %s' % url) LOG('ERP5', INFO, 'TemplateTool doing a svn checkout of %s' % url)
return self._download_svn(url, bt_id) return self._download_svn(url, bt_id)
...@@ -712,7 +714,7 @@ class TemplateTool (BaseTool): ...@@ -712,7 +714,7 @@ class TemplateTool (BaseTool):
Return a repository and an id. Return a repository and an id.
""" """
repository, id = json.loads(b64decode(uid)) repository, id = json.loads(b64decode(uid))
return repository.encode('utf-8'), id.encode('utf-8') return unicode2str(repository), unicode2str(id)
security.declarePublic( 'encodeRepositoryBusinessTemplateUid' ) security.declarePublic( 'encodeRepositoryBusinessTemplateUid' )
def encodeRepositoryBusinessTemplateUid(self, repository, id): def encodeRepositoryBusinessTemplateUid(self, repository, id):
...@@ -720,7 +722,7 @@ class TemplateTool (BaseTool): ...@@ -720,7 +722,7 @@ class TemplateTool (BaseTool):
encode the repository and the id of a business template. encode the repository and the id of a business template.
Return an uid. Return an uid.
""" """
return b64encode(json.dumps((repository, id))) return b64encode(str2bytes(json.dumps((repository, id))))
security.declarePublic('compareVersionStrings') security.declarePublic('compareVersionStrings')
def compareVersionStrings(self, version, comparing_string): def compareVersionStrings(self, version, comparing_string):
...@@ -1066,7 +1068,7 @@ class TemplateTool (BaseTool): ...@@ -1066,7 +1068,7 @@ class TemplateTool (BaseTool):
installed_revision=installed_revision, installed_revision=installed_revision,
repository=repository, repository=repository,
**property_dict) **property_dict)
obj.setUid(uid) obj.setUid(bytes2str(uid))
result_list.append(obj) result_list.append(obj)
result_list.sort(key=lambda x: x.getTitle()) result_list.sort(key=lambda x: x.getTitle())
return result_list return result_list
...@@ -1113,7 +1115,7 @@ class TemplateTool (BaseTool): ...@@ -1113,7 +1115,7 @@ class TemplateTool (BaseTool):
e = int(e) e = int(e)
except ValueError: except ValueError:
# ASCII code is one byte, so this produces negative. # ASCII code is one byte, so this produces negative.
e = struct.unpack('b', e)[0] - 0x200 e = struct.unpack('b', e.encode())[0] - 0x200
except IndexError: except IndexError:
e = 0 e = 0
return e return e
......
product/ERP5/tests/testBusinessTemplateTwoFileExport.py
View file @ d25f925b
...@@ -129,7 +129,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase): ...@@ -129,7 +129,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase):
with open(xml_document_path, 'rb') as xml_file: with open(xml_document_path, 'rb') as xml_file:
xml_file_content = xml_file.read() xml_file_content = xml_file.read()
for exported_property in removed_property_list: for exported_property in removed_property_list:
self.assertNotIn('<string>'+exported_property+'</string>', self.assertNotIn(('<string>%s</string>' % exported_property).encode(),
xml_file_content) xml_file_content)
import_template = self._importBusinessTemplate() import_template = self._importBusinessTemplate()
...@@ -152,7 +152,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase): ...@@ -152,7 +152,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase):
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
test_component_path, test_component_path,
".py", ".py",
test_component_kw["text_content"], test_component_kw["text_content"].encode(),
['text_content']) ['text_content'])
self.portal.portal_components.manage_delObjects([test_component_id]) self.portal.portal_components.manage_delObjects([test_component_id])
...@@ -242,7 +242,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase): ...@@ -242,7 +242,7 @@ class TestBusinessTemplateTwoFileExport(ERP5TypeTestCase):
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
python_script_path, python_script_path,
".py", ".py",
python_script_kw["_body"], python_script_kw["_body"].encode(),
['_body','_code']) ['_body','_code'])
skin_folder.manage_delObjects(python_script_id) skin_folder.manage_delObjects(python_script_id)
...@@ -315,7 +315,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -315,7 +315,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForImageInImageModule(dict( self._checkTwoFileImportExportForImageInImageModule(dict(
title = "foo", title = "foo",
data = "malformed data", data = b"malformed data",
portal_type = "Image", portal_type = "Image",
), '.bin') ), '.bin')
...@@ -332,7 +332,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -332,7 +332,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
file_id) file_id)
try: try:
args = file_document_kw['data'], ('data',) if extension else () args = file_document_kw['data'], (b'data',) if extension else ()
except KeyError: except KeyError:
args = None, ('data',) args = None, ('data',)
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
...@@ -356,7 +356,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -356,7 +356,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "a test file", data = b"a test file",
content_type = "text/javascript", content_type = "text/javascript",
portal_type = "File", portal_type = "File",
), '.js') ), '.js')
...@@ -368,7 +368,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -368,7 +368,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "a test file", data = b"a test file",
content_type = "application/octet-stream", content_type = "application/octet-stream",
portal_type = "File", portal_type = "File",
), '.bin') ), '.bin')
...@@ -380,7 +380,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -380,7 +380,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "a test file", data = b"a test file",
content_type = "application/epub+zip", content_type = "application/epub+zip",
portal_type = "File", portal_type = "File",
), '.epub') ), '.epub')
...@@ -392,7 +392,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -392,7 +392,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo.js", title = "foo.js",
data = "a test file", data = b"a test file",
portal_type = "File", portal_type = "File",
), '.js') ), '.js')
...@@ -403,7 +403,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -403,7 +403,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "<script> ... </script>", data = b"<script> ... </script>",
default_reference = "foo.js", default_reference = "foo.js",
portal_type = "File", portal_type = "File",
), '.js') ), '.js')
...@@ -415,7 +415,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -415,7 +415,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "a test file", data = b"a test file",
content_type = None, content_type = None,
portal_type = "File", portal_type = "File",
), '.bin') ), '.bin')
...@@ -429,7 +429,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -429,7 +429,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "a test file", data = b"a test file",
content_type = "video/wavelet", content_type = "video/wavelet",
portal_type = "File", portal_type = "File",
), '.bin') ), '.bin')
...@@ -443,7 +443,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -443,7 +443,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "a test file", data = b"a test file",
content_type = "text/x-uri", content_type = "text/x-uri",
portal_type = "File", portal_type = "File",
), '.txt') ), '.txt')
...@@ -454,7 +454,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -454,7 +454,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
where extension (.xml, exported as ._xml to avoid conflict with the meta-data file) where extension (.xml, exported as ._xml to avoid conflict with the meta-data file)
is identified by the title is identified by the title
""" """
file_content = """<person> file_content = b"""<person>
<name>John</name> <name>John</name>
<surname>Doe</surname> <surname>Doe</surname>
</person> </person>
...@@ -472,7 +472,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -472,7 +472,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
where extension (.xml, exported as ._xml to avoid conflict with the meta-data file) where extension (.xml, exported as ._xml to avoid conflict with the meta-data file)
is identified by the title is identified by the title
""" """
file_content = """<person> file_content = b"""<person>
<name>John</name> <name>John</name>
<surname>Doe</surname> <surname>Doe</surname>
</person> </person>
...@@ -524,7 +524,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -524,7 +524,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
"""Test Business Template Import And Export With A PDF Document""" """Test Business Template Import And Export With A PDF Document"""
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo.pdf", title = "foo.pdf",
data ="pdf content, maybe should update for base64 sample" , data =b"pdf content, maybe should update for base64 sample" ,
portal_type = "PDF", portal_type = "PDF",
), '.pdf') ), '.pdf')
...@@ -563,7 +563,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -563,7 +563,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
method_document_path, method_document_path,
".sql", ".sql",
'dummy_method_template', b'dummy_method_template',
['src']) ['src'])
catalog.manage_delObjects([method_id]) catalog.manage_delObjects([method_id])
...@@ -611,7 +611,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -611,7 +611,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
method_document_path, method_document_path,
".sql", ".sql",
'dummy_method_template', b'dummy_method_template',
['src']) ['src'])
catalog.manage_delObjects([method_id]) catalog.manage_delObjects([method_id])
...@@ -657,7 +657,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -657,7 +657,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
method_document_path, method_document_path,
".sql", ".sql",
'dummy_method_template', b'dummy_method_template',
['src']) ['src'])
self.portal.portal_skins[skin_folder_id].manage_delObjects([method_id]) self.portal.portal_skins[skin_folder_id].manage_delObjects([method_id])
...@@ -696,7 +696,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -696,7 +696,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
page_template_path, page_template_path,
".zpt", ".zpt",
page_template_kw['_text'], page_template_kw['_text'].encode(),
['_text']) ['_text'])
self.portal.portal_skins[skin_folder_id].manage_delObjects([page_template_id]) self.portal.portal_skins[skin_folder_id].manage_delObjects([page_template_id])
...@@ -747,7 +747,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -747,7 +747,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
dtml_method_path, dtml_method_path,
".js", ".js",
dtml_method_kw['raw'], dtml_method_kw['raw'].encode(),
['raw']) ['raw'])
self.portal.portal_skins[skin_folder_id].manage_delObjects([dtml_method_id]) self.portal.portal_skins[skin_folder_id].manage_delObjects([dtml_method_id])
...@@ -798,7 +798,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -798,7 +798,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
dtml_method_path, dtml_method_path,
".txt", ".txt",
dtml_method_kw['raw'], dtml_method_kw['raw'].encode(),
['raw']) ['raw'])
self.portal.portal_skins[skin_folder_id].manage_delObjects([dtml_method_id]) self.portal.portal_skins[skin_folder_id].manage_delObjects([dtml_method_id])
...@@ -862,7 +862,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -862,7 +862,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "", # XXX a dummy string in data leads to 'NotConvertedError' data = b"", # XXX dummy data in data leads to 'NotConvertedError'
portal_type = "Spreadsheet", portal_type = "Spreadsheet",
), '.bin') ), '.bin')
...@@ -873,7 +873,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -873,7 +873,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo", title = "foo",
data = "", # XXX a dummy string in data leads to 'NotConvertedError' data = b"", # XXX dummy data in data leads to 'NotConvertedError'
content_type = "application/vnd.oasis.opendocument.spreadsheet", content_type = "application/vnd.oasis.opendocument.spreadsheet",
portal_type = "Spreadsheet", portal_type = "Spreadsheet",
), '.ods') ), '.ods')
...@@ -885,7 +885,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -885,7 +885,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
""" """
self._checkTwoFileImportExportForDocumentInDocumentModule(dict( self._checkTwoFileImportExportForDocumentInDocumentModule(dict(
title = "foo.xlsx", title = "foo.xlsx",
data = "", # XXX a dummy string in data leads to 'NotConvertedError' data = b"", # XXX dummy data in data leads to 'NotConvertedError'
portal_type = "Spreadsheet", portal_type = "Spreadsheet",
), '.xlsx') ), '.xlsx')
...@@ -910,7 +910,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -910,7 +910,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
test_page_document_path, test_page_document_path,
".html", ".html",
test_page_data_kw["text_content"], test_page_data_kw["text_content"].encode(),
["text_content"]) ["text_content"])
self.portal.test_page_module.manage_delObjects([test_page_id]) self.portal.test_page_module.manage_delObjects([test_page_id])
...@@ -954,7 +954,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO ...@@ -954,7 +954,7 @@ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO
import_template = self._exportAndReImport( import_template = self._exportAndReImport(
python_script_path, python_script_path,
".py", ".py",
python_script_kw["_body"], python_script_kw["_body"].encode(),
['_body','_code']) ['_body','_code'])
self.portal.portal_skins[skin_folder_id].manage_delObjects([python_script_id]) self.portal.portal_skins[skin_folder_id].manage_delObjects([python_script_id])
......
product/ERP5Type/XMLExportImport/__init__.py
View file @ d25f925b
...@@ -40,19 +40,22 @@ ...@@ -40,19 +40,22 @@
from Acquisition import aq_base, aq_inner from Acquisition import aq_base, aq_inner
from collections import OrderedDict from collections import OrderedDict
from io import BytesIO from io import BytesIO
from zodbpickle.pickle import Pickler from zodbpickle.slowpickle import Pickler
from xml.sax.saxutils import escape, unescape from xml.sax.saxutils import escape, unescape
from lxml import etree from lxml import etree
from lxml.etree import Element, SubElement from lxml.etree import Element, SubElement
from xml_marshaller.xml_marshaller import Marshaller from xml_marshaller.xml_marshaller import Marshaller
from OFS.Image import Pdata from OFS.Image import Pdata
from base64 import standard_b64encode import six
if six.PY2:
from base64 import standard_b64encode, encodestring as encodebytes
else:
from base64 import standard_b64encode, encodebytes
from hashlib import sha1 from hashlib import sha1
from Products.ERP5Type.Utils import ensure_list from Products.ERP5Type.Utils import bytes2str
#from zLOG import LOG #from zLOG import LOG
import six
try: try:
long_ = long long_ = long
except NameError: # six.PY3 except NameError: # six.PY3
...@@ -62,6 +65,9 @@ MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller' ...@@ -62,6 +65,9 @@ MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller'
marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI, marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI,
as_tree=True).dumps as_tree=True).dumps
DEFAULT_PICKLE_PROTOCOL = 1 if six.PY2 else 3
class OrderedPickler(Pickler): class OrderedPickler(Pickler):
"""Pickler producing consistent output by saving dicts in order """Pickler producing consistent output by saving dicts in order
""" """
...@@ -204,8 +210,8 @@ def Base_asXML(object, root=None): ...@@ -204,8 +210,8 @@ def Base_asXML(object, root=None):
local_group_node.append(marshaller(group_role[1])) local_group_node.append(marshaller(group_role[1]))
if return_as_object: if return_as_object:
return root return root
return etree.tostring(root, encoding='utf-8', return bytes2str(etree.tostring(root, encoding='utf-8',
xml_declaration=True, pretty_print=True) xml_declaration=True, pretty_print=True))
def Folder_asXML(object, omit_xml_declaration=True, root=None): def Folder_asXML(object, omit_xml_declaration=True, root=None):
""" """
...@@ -226,24 +232,27 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None): ...@@ -226,24 +232,27 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None):
if issubclass(o.__class__, Base): if issubclass(o.__class__, Base):
o.asXML(root=root_node) o.asXML(root=root_node)
return etree.tostring(root, encoding='utf-8', return bytes2str(etree.tostring(root, encoding='utf-8',
xml_declaration=xml_declaration, pretty_print=True) xml_declaration=xml_declaration, pretty_print=True))
## The code below was initially from OFS.XMLExportImport ## The code below was initially from OFS.XMLExportImport
from six import string_types as basestring from six import string_types as basestring
from base64 import encodestring
from ZODB.serialize import referencesf from ZODB.serialize import referencesf
from ZODB.ExportImport import TemporaryFile, export_end_marker from ZODB.ExportImport import TemporaryFile, export_end_marker
from ZODB.utils import p64 from ZODB.utils import p64
from ZODB.utils import u64 from ZODB.utils import u64
from functools import partial from functools import partial
from inspect import getargspec if six.PY2:
from inspect import getargspec as getfullargspec
else:
from inspect import getfullargspec
from OFS import ObjectManager from OFS import ObjectManager
from . import ppml from . import ppml
magic=b'<?xm' # importXML(jar, file, clue)} magic=b'<?xm' # importXML(jar, file, clue)}
def reorderPickle(jar, p):
def reorderPickle(jar, p, pickle_protocol):
try: try:
from ZODB._compat import Unpickler, Pickler from ZODB._compat import Unpickler, Pickler
except ImportError: # BBB: ZODB 3.10 except ImportError: # BBB: ZODB 3.10
...@@ -278,37 +287,47 @@ def reorderPickle(jar, p): ...@@ -278,37 +287,47 @@ def reorderPickle(jar, p):
unpickler.persistent_load=persistent_load unpickler.persistent_load=persistent_load
newp=BytesIO() newp=BytesIO()
pickler=OrderedPickler(newp,1) pickler = OrderedPickler(newp, pickle_protocol)
pickler.persistent_id=persistent_id pickler.persistent_id=persistent_id
classdef = unpickler.load() classdef = unpickler.load()
obj = unpickler.load() obj = unpickler.load()
pickler.dump(classdef) pickler.dump(classdef)
pickler.dump(obj) pickler.dump(obj)
if 0: # debug
debugp = BytesIO()
debugpickler = OrderedPickler(debugp, pickle_protocol)
debugpickler.persistent_id = persistent_id
debugpickler.dump(obj)
import pickletools
print(debugp.getvalue())
print(pickletools.dis(debugp.getvalue()))
p=newp.getvalue() p=newp.getvalue()
return obj, p return obj, p
def _mapOid(id_mapping, oid): def _mapOid(id_mapping, oid):
idprefix = str(u64(oid)) idprefix = str(u64(oid))
id = id_mapping[idprefix] id = id_mapping[idprefix]
old_aka = encodestring(oid)[:-1] old_aka = encodebytes(oid)[:-1]
aka=encodestring(p64(long_(id)))[:-1] # Rebuild oid based on mapped id aka=encodebytes(p64(long_(id)))[:-1] # Rebuild oid based on mapped id
id_mapping.setConvertedAka(old_aka, aka) id_mapping.setConvertedAka(old_aka, aka)
return idprefix+'.', id, aka return idprefix+'.', id, aka
def XMLrecord(oid, plen, p, id_mapping): def XMLrecord(oid, plen, p, id_mapping):
# Proceed as usual # Proceed as usual
q=ppml.ToXMLUnpickler f = BytesIO(p)
f=BytesIO(p) u = ppml.ToXMLUnpickler(f)
u=q(f)
u.idprefix, id, aka = _mapOid(id_mapping, oid) u.idprefix, id, aka = _mapOid(id_mapping, oid)
p=u.load(id_mapping=id_mapping).__str__(4) p = u.load(id_mapping=id_mapping).__str__(4)
if f.tell() < plen: if f.tell() < plen:
p=p+u.load(id_mapping=id_mapping).__str__(4) p=p+u.load(id_mapping=id_mapping).__str__(4)
String=' <record id="%s" aka="%s">\n%s </record>\n' % (id, aka, p) String=' <record id="%s" aka="%s">\n%s </record>\n' % (id, bytes2str(aka), p)
return String return String
def exportXML(jar, oid, file=None):
def exportXML(jar, oid, file=None, pickle_protocol=DEFAULT_PICKLE_PROTOCOL):
# For performance reasons, exportXML does not use 'XMLrecord' anymore to map # For performance reasons, exportXML does not use 'XMLrecord' anymore to map
# oids. This requires to initialize MinimalMapping.marked_reference before # oids. This requires to initialize MinimalMapping.marked_reference before
# any string output, i.e. in ppml.Reference.__init__ # any string output, i.e. in ppml.Reference.__init__
...@@ -316,15 +335,15 @@ def exportXML(jar, oid, file=None): ...@@ -316,15 +335,15 @@ def exportXML(jar, oid, file=None):
# can have values that have a shorter representation in 'repr' instead of # can have values that have a shorter representation in 'repr' instead of
# 'base64' (see ppml.convert) and ppml.String does not support this. # 'base64' (see ppml.convert) and ppml.String does not support this.
load = jar._storage.load load = jar._storage.load
if 'version' in getargspec(load).args: # BBB: ZODB<5 (TmpStore) if 'version' in getfullargspec(load).args: # BBB: ZODB<5 (TmpStore)
load = partial(load, version='') load = partial(load, version='')
pickle_dict = {oid: None} pickle_dict = {oid: None}
max_cache = [1e7] # do not cache more than 10MB of pickle data max_cache = [1e7] # do not cache more than 10MB of pickle data
def getReorderedPickle(oid): def getReorderedPickle(oid):
p = pickle_dict[oid] p = pickle_dict.get(oid)
if p is None: if p is None:
p = load(oid)[0] p = load(oid)[0]
p = reorderPickle(jar, p)[1] p = reorderPickle(jar, p, pickle_protocol)[1]
if len(p) < max_cache[0]: if len(p) < max_cache[0]:
max_cache[0] -= len(p) max_cache[0] -= len(p)
pickle_dict[oid] = p pickle_dict[oid] = p
...@@ -342,9 +361,9 @@ def exportXML(jar, oid, file=None): ...@@ -342,9 +361,9 @@ def exportXML(jar, oid, file=None):
# Do real export # Do real export
if file is None: if file is None:
file = TemporaryFile() file = TemporaryFile(mode='w')
elif isinstance(file, basestring): elif isinstance(file, basestring):
file = open(file, 'w+b') file = open(file, 'w')
write = file.write write = file.write
write('<?xml version="1.0"?>\n<ZopeData>\n') write('<?xml version="1.0"?>\n<ZopeData>\n')
for oid in reordered_oid_list: for oid in reordered_oid_list:
...@@ -403,7 +422,6 @@ def importXML(jar, file, clue=''): ...@@ -403,7 +422,6 @@ def importXML(jar, file, clue=''):
F.end_handlers['record'] = save_record F.end_handlers['record'] = save_record
F.end_handlers['ZopeData'] = save_zopedata F.end_handlers['ZopeData'] = save_zopedata
F.start_handlers['ZopeData'] = start_zopedata F.start_handlers['ZopeData'] = start_zopedata
F.binary=1
F.file=outfile F.file=outfile
# <patch> # <patch>
# Our BTs XML files don't declare encoding but have accented chars in them # Our BTs XML files don't declare encoding but have accented chars in them
......
product/ERP5Type/XMLExportImport/ppml.py
View file @ d25f925b
...@@ -15,20 +15,24 @@ ...@@ -15,20 +15,24 @@
"""Provide conversion between Python pickles and XML """Provide conversion between Python pickles and XML
""" """
# Python3 C implementation does not have Unpickler.dispatch attribute from zodbpickle.slowpickle import *
from zodbpickle.pickle import *
import ast
import struct import struct
import base64 import six
if six.PY2:
from base64 import encodestring as base64_encodebytes, decodestring as base64_decodebytes
from zodbpickle.pickle_2 import decode_long
else:
from base64 import encodebytes as base64_encodebytes, decodebytes as base64_decodebytes
from zodbpickle.pickle_3 import decode_long
import re import re
from marshal import loads as mloads from marshal import loads as mloads
from .xyap import NoBlanks from .xyap import NoBlanks
from .xyap import xyap from .xyap import xyap
from Products.ERP5Type.Utils import str2bytes from Products.ERP5Type.Utils import bytes2str, str2bytes, unicode2str
import six
from marshal import dumps as mdumps from marshal import dumps as mdumps
#from zLOG import LOG
binary = re.compile('[^\x1f-\x7f]').search binary = re.compile('[^\x1f-\x7f]').search
...@@ -41,7 +45,7 @@ else: ...@@ -41,7 +45,7 @@ else:
def escape(s, encoding='repr'): def escape(s, encoding='repr'):
if binary(s) and isinstance(s, str): if binary(s) and isinstance(s, str):
s = base64.encodestring(s)[:-1] s = base64_encodebytes(s)[:-1]
encoding = 'base64' encoding = 'base64'
elif '>' in s or '<' in s or '&' in s: elif '>' in s or '<' in s or '&' in s:
if not ']]>' in s: if not ']]>' in s:
...@@ -55,7 +59,7 @@ def escape(s, encoding='repr'): ...@@ -55,7 +59,7 @@ def escape(s, encoding='repr'):
def unescape(s, encoding): def unescape(s, encoding):
if encoding == 'base64': if encoding == 'base64':
return base64.decodestring(s) return base64_decodebytes(s)
else: else:
s = s.replace(b'&lt;', b'<') s = s.replace(b'&lt;', b'<')
s = s.replace(b'&gt;', b'>') s = s.replace(b'&gt;', b'>')
...@@ -89,14 +93,16 @@ def convert(S): ...@@ -89,14 +93,16 @@ def convert(S):
### readable output. ### readable output.
try: try:
if not isinstance(S, six.text_type): if not isinstance(S, six.text_type):
S.decode('utf8') decoded = S.decode('utf8')
if six.PY3:
S = decoded
except UnicodeDecodeError: except UnicodeDecodeError:
return 'base64', base64.encodestring(S)[:-1] return 'base64', bytes2str(base64_encodebytes(S)[:-1])
else: else:
new = reprs_re.sub(sub_reprs, S) new = reprs_re.sub(sub_reprs, S)
### patch end ### patch end
if len(new) > (1.4*len(S)): if len(new) > (1.4*len(S)):
return 'base64', base64.encodestring(S)[:-1] return 'base64', bytes2str(base64_encodebytes(str2bytes(S))[:-1])
elif '>' in new or '<' in S or '&' in S: elif '>' in new or '<' in S or '&' in S:
if not ']]>' in S: if not ']]>' in S:
return 'cdata', '<![CDATA[\n\n' + new + '\n\n]]>' return 'cdata', '<![CDATA[\n\n' + new + '\n\n]]>'
...@@ -107,11 +113,11 @@ def convert(S): ...@@ -107,11 +113,11 @@ def convert(S):
# For optimization. # For optimization.
def unconvert(encoding,S): def unconvert(encoding,S):
if encoding == 'base64': if encoding == 'base64':
return base64.decodestring(S) return base64_decodebytes(S)
else: else:
return str2bytes(eval(b"'" + S.replace(b'\n', b'') + b"'")) return str2bytes(ast.literal_eval(bytes2str(b"'" + S.replace(b'\n', b'') + b"'")))
class Global: class Global(object):
def __init__(self, module, name, mapping): def __init__(self, module, name, mapping):
self.module=module self.module=module
self.name=name self.name=name
...@@ -125,14 +131,14 @@ class Global: ...@@ -125,14 +131,14 @@ class Global:
return '%s<%s%s name="%s" module="%s"/>\n' % ( return '%s<%s%s name="%s" module="%s"/>\n' % (
' '*indent, name, id, self.name, self.module) ' '*indent, name, id, self.name, self.module)
class Immutable: class Immutable(object):
def __init__(self, value): def __init__(self, value):
self.value = value self.value = value
def getValue(self): def getValue(self):
return self.value return self.value
class Scalar: class Scalar(object):
def __init__(self, v, mapping): def __init__(self, v, mapping):
self._v=v self._v=v
self.mapping = mapping self.mapping = mapping
...@@ -158,6 +164,10 @@ class Long(Scalar): ...@@ -158,6 +164,10 @@ class Long(Scalar):
return result return result
class String(Scalar): class String(Scalar):
def tag_name(self):
return self.__class__.__name__.lower()
encoding = None encoding = None
def __init__(self, v, mapping, encoding=''): def __init__(self, v, mapping, encoding=''):
...@@ -172,8 +182,8 @@ class String(Scalar): ...@@ -172,8 +182,8 @@ class String(Scalar):
# This is used when strings represent references which need to # This is used when strings represent references which need to
# be converted. # be converted.
encoding = 'base64' encoding = 'base64'
v = base64.encodestring(self._v)[:-1] v = base64_encodebytes(self._v)[:-1]
self._v = self.mapping.convertBase64(v) self._v = bytes2str(self.mapping.convertBase64(v))
else: else:
encoding, self._v = convert(self._v) encoding, self._v = convert(self._v)
self.encoding = encoding self.encoding = encoding
...@@ -183,7 +193,7 @@ class String(Scalar): ...@@ -183,7 +193,7 @@ class String(Scalar):
encoding = '' # JPS repr is default encoding encoding = '' # JPS repr is default encoding
else: else:
encoding = ' encoding="%s"' % encoding encoding = ' encoding="%s"' % encoding
name=self.__class__.__name__.lower() name = self.tag_name()
result = '<%s%s%s>%s</%s>' % (name, id, encoding, v, name) result = '<%s%s%s>%s</%s>' % (name, id, encoding, v, name)
if hasattr(self, 'id'): if hasattr(self, 'id'):
# The value is Immutable - let us add it the the immutable mapping # The value is Immutable - let us add it the the immutable mapping
...@@ -191,11 +201,22 @@ class String(Scalar): ...@@ -191,11 +201,22 @@ class String(Scalar):
self.mapping.setImmutable(self.id, Immutable(value = result)) self.mapping.setImmutable(self.id, Immutable(value = result))
return '%s%s\n' % (' '*indent, result) return '%s%s\n' % (' '*indent, result)
class Unicode(String): class Unicode(String):
def tag_name(self):
if six.PY3:
return 'string'
return super(Unicode, self).tag_name()
def value(self): def value(self):
return self._v.encode('utf-8') return unicode2str(super(Unicode, self).value())
class Wrapper: class Bytes(String):
pass
class Wrapper(object):
def __init__(self, v, mapping): def __init__(self, v, mapping):
self._v=v self._v=v
self.mapping = mapping self.mapping = mapping
...@@ -209,13 +230,13 @@ class Wrapper: ...@@ -209,13 +230,13 @@ class Wrapper:
name=self.__class__.__name__.lower() name=self.__class__.__name__.lower()
v=self._v v=self._v
i=' '*indent i=' '*indent
if isinstance(v,Scalar): if isinstance(v, Scalar):
return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name) return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__()[:-1], name)
else: else:
v=v.__str__(indent+2) v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name) return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
class Collection: class Collection(object):
def __init__(self, mapping): def __init__(self, mapping):
self.mapping = mapping self.mapping = mapping
...@@ -302,6 +323,7 @@ class Object(Sequence): ...@@ -302,6 +323,7 @@ class Object(Sequence):
def __setstate__(self, v): self.append(State(v, self.mapping)) def __setstate__(self, v): self.append(State(v, self.mapping))
class Bool(Scalar): pass
class Int(Scalar): pass class Int(Scalar): pass
class Float(Scalar): pass class Float(Scalar): pass
class List(Sequence): pass class List(Sequence): pass
...@@ -328,8 +350,9 @@ class Persistent(Wrapper): ...@@ -328,8 +350,9 @@ class Persistent(Wrapper):
v=v.__str__(indent+2) v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name) return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
blanck_line_expression = re.compile('^ +$') blanck_line_expression = re.compile('^ +$')
class NoBlanks: class NoBlanks(object):
""" """
This allows to ignore at least whitespaces between elements and also This allows to ignore at least whitespaces between elements and also
correctly handle string/unicode correctly handle string/unicode
...@@ -376,7 +399,7 @@ class NoBlanks: ...@@ -376,7 +399,7 @@ class NoBlanks:
self.append(data) self.append(data)
class IdentityMapping: class IdentityMapping(object):
def __init__(self): def __init__(self):
self.resetMapping() self.resetMapping()
self.immutable = {} self.immutable = {}
...@@ -462,6 +485,29 @@ class MinimalMapping(IdentityMapping): ...@@ -462,6 +485,29 @@ class MinimalMapping(IdentityMapping):
def __str__(self, a): def __str__(self, a):
return "Error here" return "Error here"
class UnsupportedOpCode(AssertionError):
"""Error when encountering an opcode that is not supposed to be used
by this implementation.
"""
def unsupported_opcode(opcode):
def handler(self):
raise UnsupportedOpCode(opcode)
return handler
def make_decorator(dispatch):
def register(opcode):
def decorator(f):
if six.PY2:
dispatch[opcode] = f
dispatch[opcode[0]] = f
return f
return decorator
return register
class ToXMLUnpickler(Unpickler): class ToXMLUnpickler(Unpickler):
def load(self, id_mapping=None): def load(self, id_mapping=None):
if id_mapping is None: if id_mapping is None:
...@@ -472,96 +518,132 @@ class ToXMLUnpickler(Unpickler): ...@@ -472,96 +518,132 @@ class ToXMLUnpickler(Unpickler):
dispatch = {} dispatch = {}
dispatch.update(Unpickler.dispatch.copy()) dispatch.update(Unpickler.dispatch.copy())
register = make_decorator(dispatch)
def persistent_load(self, v): def persistent_load(self, v):
return Persistent(v, self.id_mapping) return Persistent(v, self.id_mapping)
def load_persid(self): @register(BINPERSID)
pid = self.readline()[:-1] def load_binpersid(self):
self.append(self.persistent_load(String(pid, self.id_mapping))) pid = self.stack.pop()
dispatch[PERSID] = load_persid self.append(self.persistent_load(pid))
@register(NONE)
def load_none(self): def load_none(self):
self.append(none) self.append(none)
dispatch[NONE] = load_none
@register(INT)
def load_int(self): def load_int(self):
self.append(Int(int(self.readline()[:-1]), self.id_mapping)) self.append(Int(int(self.readline()[:-1]), self.id_mapping))
dispatch[INT] = load_int
@register(BININT)
def load_binint(self): def load_binint(self):
self.append(Int(mloads('i' + self.read(4)), self.id_mapping)) self.append(Int(mloads(b'i' + self.read(4)), self.id_mapping))
dispatch[BININT] = load_binint
@register(BININT1)
def load_binint1(self): def load_binint1(self):
self.append(Int(mloads('i' + self.read(1) + '\000\000\000'), self.id_mapping)) self.append(Int(mloads(b'i' + self.read(1) + b'\000\000\000'), self.id_mapping))
dispatch[BININT1] = load_binint1
@register(BININT2)
def load_binint2(self): def load_binint2(self):
self.append(Int(mloads('i' + self.read(2) + '\000\000'), self.id_mapping)) self.append(Int(mloads(b'i' + self.read(2) + b'\000\000'), self.id_mapping))
dispatch[BININT2] = load_binint2
@register(LONG)
def load_long(self): def load_long(self):
self.append(Long(long_(self.readline()[:-1], 0), self.id_mapping)) val = self.readline()[:-1]
dispatch[LONG] = load_long if six.PY3:
val = val.decode('ascii')
def load_float(self): if val and val[-1] == 'L':
self.append(Float(float(self.readline()[:-1]), self.id_mapping)) val = val[:-1]
dispatch[FLOAT] = load_float self.append(Long(long_(val, 0), self.id_mapping))
@register(LONG1)
def load_long1(self):
n = ord(self.read(1))
data = self.read(n)
self.append(Long(decode_long(data), self.id_mapping))
@register(LONG4)
def load_long4(self):
n = mloads(b'i' + self.read(4))
if n < 0:
# Corrupt or hostile pickle -- we never write one like this
raise UnpicklingError("LONG pickle has negative byte count");
data = self.read(n)
self.append(Long(decode_long(data), self.id_mapping))
@register(NEWTRUE)
def load_true(self):
self.append(Bool(True, self.id_mapping))
@register(NEWFALSE)
def load_false(self):
self.append(Bool(False, self.id_mapping))
@register(BINFLOAT)
def load_binfloat(self, unpack=struct.unpack): def load_binfloat(self, unpack=struct.unpack):
self.append(Float(unpack('>d', self.read(8))[0], self.id_mapping)) self.append(Float(unpack('>d', self.read(8))[0], self.id_mapping))
dispatch[BINFLOAT] = load_binfloat
def load_string(self):
self.append(String(eval(self.readline()[:-1],
{'__builtins__': {}}), self.id_mapping)) # Let's be careful
dispatch[STRING] = load_string
@register(BINSTRING)
def load_binstring(self): def load_binstring(self):
len = mloads('i' + self.read(4)) len = mloads(b'i' + self.read(4))
self.append(String(self.read(len), self.id_mapping)) self.append(String(self.read(len), self.id_mapping))
dispatch[BINSTRING] = load_binstring
def load_unicode(self):
self.append(Unicode(six.text_type(eval(self.readline()[:-1],
{'__builtins__': {}})), self.id_mapping)) # Let's be careful
dispatch[UNICODE] = load_unicode
@register(BINUNICODE)
def load_binunicode(self): def load_binunicode(self):
len = mloads('i' + self.read(4)) len = mloads(b'i' + self.read(4))
self.append(Unicode(six.text_type(self.read(len), 'utf-8'), self.id_mapping)) self.append(Unicode(six.text_type(self.read(len), 'utf-8'), self.id_mapping))
dispatch[BINUNICODE] = load_binunicode
@register(SHORT_BINSTRING)
def load_short_binstring(self): def load_short_binstring(self):
len = mloads('i' + self.read(1) + '\000\000\000') len = mloads(b'i' + self.read(1) + b'\000\000\000')
self.append(String(self.read(len), self.id_mapping)) self.append(String(self.read(len), self.id_mapping))
dispatch[SHORT_BINSTRING] = load_short_binstring
@register(BINBYTES)
def load_binbytes(self):
len = mloads(b'i' + self.read(4))
self.append(Bytes(self.read(len), self.id_mapping))
@register(SHORT_BINBYTES)
def load_short_binbytes(self):
len = mloads(b'i' + self.read(1) + b'\000\000\000')
self.append(Bytes(self.read(len), self.id_mapping))
@register(TUPLE)
def load_tuple(self): def load_tuple(self):
k = self.marker() k = self.marker()
#LOG('load_tuple, k',0,k)
#LOG('load_tuple, stack[k+1:]',0,self.stack[k+1:])
self.stack[k:] = [Tuple(self.id_mapping, v=self.stack[k+1:])] self.stack[k:] = [Tuple(self.id_mapping, v=self.stack[k+1:])]
dispatch[TUPLE] = load_tuple
@register(TUPLE1)
def load_tuple1(self):
self.stack[-1] = Tuple(self.id_mapping, v=(self.stack[-1],))
@register(TUPLE2)
def load_tuple2(self):
self.stack[-2:] = [Tuple(self.id_mapping, v=(self.stack[-2], self.stack[-1]))]
@register(TUPLE3)
def load_tuple3(self):
self.stack[-3:] = [Tuple(self.id_mapping, v=(self.stack[-3], self.stack[-2], self.stack[-1]))]
@register(EMPTY_TUPLE)
def load_empty_tuple(self): def load_empty_tuple(self):
self.stack.append(Tuple(self.id_mapping)) self.stack.append(Tuple(self.id_mapping))
dispatch[EMPTY_TUPLE] = load_empty_tuple
@register(EMPTY_LIST)
def load_empty_list(self): def load_empty_list(self):
self.stack.append(List(self.id_mapping)) self.stack.append(List(self.id_mapping))
dispatch[EMPTY_LIST] = load_empty_list
@register(EMPTY_DICT)
def load_empty_dictionary(self): def load_empty_dictionary(self):
self.stack.append(Dictionary(self.id_mapping)) self.stack.append(Dictionary(self.id_mapping))
dispatch[EMPTY_DICT] = load_empty_dictionary
@register(LIST)
def load_list(self): def load_list(self):
k = self.marker() k = self.marker()
self.stack[k:] = [List(self.id_mapping, v=self.stack[k+1:])] self.stack[k:] = [List(self.id_mapping, v=self.stack[k+1:])]
dispatch[LIST] = load_list
@register(DICT)
def load_dict(self): def load_dict(self):
k = self.marker() k = self.marker()
d = Dictionary(self.id_mapping) d = Dictionary(self.id_mapping)
...@@ -571,18 +653,18 @@ class ToXMLUnpickler(Unpickler): ...@@ -571,18 +653,18 @@ class ToXMLUnpickler(Unpickler):
value = items[i+1] value = items[i+1]
d[key] = value d[key] = value
self.stack[k:] = [d] self.stack[k:] = [d]
dispatch[DICT] = load_dict
@register(INST)
def load_inst(self): def load_inst(self):
k = self.marker() k = self.marker()
args = Tuple(self.id_mapping, v=self.stack[k+1:]) args = Tuple(self.id_mapping, v=self.stack[k+1:])
del self.stack[k:] del self.stack[k:]
module = self.readline()[:-1] module = bytes2str(self.readline()[:-1])
name = self.readline()[:-1] name = bytes2str(self.readline()[:-1])
value=Object(Global(module, name, self.id_mapping), args, self.id_mapping) value=Object(Global(module, name, self.id_mapping), args, self.id_mapping)
self.append(value) self.append(value)
dispatch[INST] = load_inst
@register(OBJ)
def load_obj(self): def load_obj(self):
stack = self.stack stack = self.stack
k = self.marker() k = self.marker()
...@@ -592,14 +674,24 @@ class ToXMLUnpickler(Unpickler): ...@@ -592,14 +674,24 @@ class ToXMLUnpickler(Unpickler):
del stack[k:] del stack[k:]
value=Object(klass,args, self.id_mapping) value=Object(klass,args, self.id_mapping)
self.append(value) self.append(value)
dispatch[OBJ] = load_obj
@register(NEWOBJ)
def load_newobj(self):
# TODO: not really sure of this one, maybe we need
# a NewObj instead of Object
args = self.stack.pop()
cls = self.stack[-1]
obj = Object(cls, args, self.id_mapping)
self.stack[-1] = obj
#print('load_newobj', self.stack)
@register(GLOBAL)
def load_global(self): def load_global(self):
module = self.readline()[:-1] module = bytes2str(self.readline()[:-1])
name = self.readline()[:-1] name = bytes2str(self.readline()[:-1])
self.append(Global(module, name, self.id_mapping)) self.append(Global(module, name, self.id_mapping))
dispatch[GLOBAL] = load_global
@register(REDUCE)
def load_reduce(self): def load_reduce(self):
stack = self.stack stack = self.stack
...@@ -609,38 +701,28 @@ class ToXMLUnpickler(Unpickler): ...@@ -609,38 +701,28 @@ class ToXMLUnpickler(Unpickler):
value=Object(callable, arg_tup, self.id_mapping) value=Object(callable, arg_tup, self.id_mapping)
self.append(value) self.append(value)
dispatch[REDUCE] = load_reduce
idprefix='' idprefix=''
def load_get(self): @register(BINGET)
self.append(Get(self.idprefix+self.readline()[:-1], self.id_mapping))
dispatch[GET] = load_get
def load_binget(self): def load_binget(self):
i = mloads('i' + self.read(1) + '\000\000\000') i = mloads(b'i' + self.read(1) + b'\000\000\000')
self.append(Get(self.idprefix+repr(i), self.id_mapping)) self.append(Get(self.idprefix+repr(i), self.id_mapping))
dispatch[BINGET] = load_binget
@register(LONG_BINGET)
def load_long_binget(self): def load_long_binget(self):
i = mloads('i' + self.read(4)) i = mloads(b'i' + self.read(4))
self.append(Get(self.idprefix+repr(i), self.id_mapping)) self.append(Get(self.idprefix+repr(i), self.id_mapping))
dispatch[LONG_BINGET] = load_long_binget
def load_put(self):
self.stack[-1].id=self.idprefix+self.readline()[:-1]
dispatch[PUT] = load_put
@register(BINPUT)
def load_binput(self): def load_binput(self):
i = mloads('i' + self.read(1) + '\000\000\000') i = mloads(b'i' + self.read(1) + b'\000\000\000')
#LOG('load_binput', 0, 'self.stack = %r, self.idprefix+`i` = %r' % (self.stack, self.idprefix+`i`))
self.stack[-1].id=self.idprefix+repr(i) self.stack[-1].id=self.idprefix+repr(i)
dispatch[BINPUT] = load_binput
@register(LONG_BINPUT)
def load_long_binput(self): def load_long_binput(self):
i = mloads('i' + self.read(4)) i = mloads(b'i' + self.read(4))
self.stack[-1].id=self.idprefix+repr(i) self.stack[-1].id=self.idprefix+repr(i)
dispatch[LONG_BINPUT] = load_long_binput
class LogCall: class LogCall:
def __init__(self, func): def __init__(self, func):
...@@ -650,9 +732,20 @@ class ToXMLUnpickler(Unpickler): ...@@ -650,9 +732,20 @@ class ToXMLUnpickler(Unpickler):
#LOG('LogCall', 0, 'self.stack = %r, func = %s' % (context.stack, self.func.__name__)) #LOG('LogCall', 0, 'self.stack = %r, func = %s' % (context.stack, self.func.__name__))
return self.func(context) return self.func(context)
#for code in dispatch.keys(): # for code in dispatch.keys():
# dispatch[code] = LogCall(dispatch[code]) # dispatch[code] = LogCall(dispatch[code])
for opcode, name in (
(STRING, 'STRING'),
(UNICODE, 'UNICODE'),
(GET, 'GET'),
(PUT, 'PUT'),
):
if six.PY2:
dispatch[opcode] = unsupported_opcode(name)
dispatch[opcode[0]] = unsupported_opcode(name)
def ToXMLload(file): def ToXMLload(file):
return ToXMLUnpickler(file).load() return ToXMLUnpickler(file).load()
...@@ -669,7 +762,6 @@ def start_pickle(self, tag, attrs): ...@@ -669,7 +762,6 @@ def start_pickle(self, tag, attrs):
return [tag, attrs] return [tag, attrs]
def save_int(self, tag, data): def save_int(self, tag, data):
if self.binary:
v = int(name(self, tag, data)) v = int(name(self, tag, data))
if v >= 0: if v >= 0:
if v <= 0xff: if v <= 0xff:
...@@ -682,10 +774,7 @@ def save_int(self, tag, data): ...@@ -682,10 +774,7 @@ def save_int(self, tag, data):
return INT + name(self, tag, data) + b'\n' return INT + name(self, tag, data) + b'\n'
def save_float(self, tag, data): def save_float(self, tag, data):
if self.binary:
return BINFLOAT + struct.pack('>d', float(name(self, tag, data))) return BINFLOAT + struct.pack('>d', float(name(self, tag, data)))
else:
return FLOAT + name(self, tag, data) + b'\n'
def save_put(self, v, attrs): def save_put(self, v, attrs):
id = attrs.get('id', '') id = attrs.get('id', '')
...@@ -695,51 +784,96 @@ def save_put(self, v, attrs): ...@@ -695,51 +784,96 @@ def save_put(self, v, attrs):
id = id[prefix + 1:] id = id[prefix + 1:]
elif id[0] == 'i': elif id[0] == 'i':
id = id[1:] id = id[1:]
if self.binary:
id = int(id) id = int(id)
if id < 256: if id < 256:
id = BINPUT + six.int2byte(id) id = BINPUT + six.int2byte(id)
else: else:
id = LONG_BINPUT + struct.pack('<i', id) id = LONG_BINPUT + struct.pack('<i', id)
else:
id = PUT + repr(id) + b'\n'
return v + id return v + id
return v return v
def save_string(self, tag, data): def save_string(self, tag, data):
a = data[1] a = data[1]
v = b''.join(data[2:]) v = b''.join(data[2:])
encoding = a.get('encoding', 'repr') encoding = a.get('encoding', 'repr') # JPS: repr is default encoding
is_bytes = a.get('binary') == 'true' # XXX zope4py2: we don't use binary yet if encoding != '':
if encoding is not '':
v = unconvert(encoding, v) v = unconvert(encoding, v)
if self.binary:
l = len(v) l = len(v)
if l < 256: if l < 256:
op = SHORT_BINBYTES if is_bytes else SHORT_BINSTRING if encoding == 'base64':
# We can be here for two reasons:
# - the input was a string with \n or similar control characters
# that are not allowed in XML, so the str was exported as base64.
# - the input was a persistent id exported from python2, in that case
# we want to get a zodbpickle.binary back
if len(v) == 8 and self._stack[-1][0] in ('persistent', ):
# looks like a persistent id, assume it is a persistent_id -> bytes
op = SHORT_BINBYTES
else:
# if it's a valid UTF-8 string -> str
try:
v.decode('utf-8')
# XXX maybe check with repr_re ?
op = BINUNICODE if six.PY3 else BINSTRING
v = op + struct.pack('<i', l) + v
return save_put(self, v, a)
except UnicodeDecodeError:
# not valid utf-8 -> bytes
op = SHORT_BINBYTES
else:
op = SHORT_BINSTRING
try:
v.decode('ascii')
except UnicodeDecodeError:
op = BINUNICODE if six.PY3 else BINSTRING
v = op + struct.pack('<i', l) + v
return save_put(self, v, a)
v = op + six.int2byte(l) + v v = op + six.int2byte(l) + v
else: else:
op = BINBYTES if is_bytes else BINSTRING if encoding == 'base64':
op = BINBYTES
# if it's a valid UTF-8 string -> str
try:
v.decode('utf-8')
op = BINUNICODE if six.PY3 else BINSTRING
except UnicodeDecodeError:
# not valid utf-8 -> bytes
pass
else:
op = BINSTRING if six.PY2 else BINUNICODE
v = op + struct.pack('<i', l) + v v = op + struct.pack('<i', l) + v
return save_put(self, v, a)
def save_bytes(self, tag, data):
a = data[1]
v = b''.join(data[2:])
encoding = a.get('encoding', 'repr')
if encoding:
v = unconvert(encoding, v)
l = len(v)
if l < 256:
op = SHORT_BINBYTES
v = op + six.int2byte(l) + v
else: else:
v = STRING + repr(v) + '\n' op = BINBYTES
v = op + struct.pack('<i', l) + v
return save_put(self, v, a) return save_put(self, v, a)
def save_unicode(self, tag, data): def save_unicode(self, tag, data):
binary=self.binary
v=b'' v=b''
a=data[1] a=data[1]
if len(data)>2: if len(data)>2:
for x in data[2:]: for x in data[2:]:
v=v+x v=v+x
encoding=a.get('encoding','repr') # JPS: repr is default encoding encoding=a.get('encoding','repr') # JPS: repr is default encoding
if encoding is not '': if encoding != '':
v=unconvert(encoding,v) v=unconvert(encoding,v)
if binary:
l=len(v) l=len(v)
s=mdumps(l)[1:] s=mdumps(l)[1:]
v=BINUNICODE+s+v v=BINUNICODE+s+v
else: v=UNICODE+"'"+v+"'\012"
return save_put(self, v, a) return save_put(self, v, a)
def save_tuple(self, tag, data): def save_tuple(self, tag, data):
...@@ -750,26 +884,16 @@ def save_tuple(self, tag, data): ...@@ -750,26 +884,16 @@ def save_tuple(self, tag, data):
def save_list(self, tag, data): def save_list(self, tag, data):
L = data[2:] L = data[2:]
if self.binary:
v = save_put(self, EMPTY_LIST, data[1]) v = save_put(self, EMPTY_LIST, data[1])
if L: if L:
v = v + MARK + b''.join(L) + APPENDS v = v + MARK + b''.join(L) + APPENDS
else:
v = save_put(self, MARK + LIST, data[1])
if L:
v = APPEND.join(L) + APPEND
return v return v
def save_dict(self, tag, data): def save_dict(self, tag, data):
D = data[2:] D = data[2:]
if self.binary:
v = save_put(self, EMPTY_DICT, data[1]) v = save_put(self, EMPTY_DICT, data[1])
if D: if D:
v = v + MARK + b''.join(D) + SETITEMS v = v + MARK + b''.join(D) + SETITEMS
else:
v = save_put(self, MARK + DICT, data[1])
if D:
v = v + SETITEM.join(D) + SETITEM
return v return v
def save_reference(self, tag, data): def save_reference(self, tag, data):
...@@ -778,14 +902,11 @@ def save_reference(self, tag, data): ...@@ -778,14 +902,11 @@ def save_reference(self, tag, data):
prefix = id.rfind('.') prefix = id.rfind('.')
if prefix >= 0: if prefix >= 0:
id = id[prefix + 1:] id = id[prefix + 1:]
if self.binary:
id = int(id) id = int(id)
if id < 256: if id < 256:
return BINGET + six.int2byte(id) return BINGET + six.int2byte(id)
else: else:
return LONG_BINGET + struct.pack('<i', i) return LONG_BINGET + struct.pack('<i', id)
else:
return GET + repr(id) + b'\n'
def save_object(self, tag, data): def save_object(self, tag, data):
if len(data)==5: if len(data)==5:
...@@ -813,10 +934,7 @@ def save_global(self, tag, data): ...@@ -813,10 +934,7 @@ def save_global(self, tag, data):
def save_persis(self, tag, data): def save_persis(self, tag, data):
v = data[2] v = data[2]
if self.binary:
return v + BINPERSID return v + BINPERSID
else:
return PERSID + v
def save_pickle_start(self, tag, attrs): def save_pickle_start(self, tag, attrs):
return [tag, attrs] return [tag, attrs]
...@@ -825,7 +943,14 @@ def save_pickle(self, tag, data): ...@@ -825,7 +943,14 @@ def save_pickle(self, tag, data):
return data[2] + b'.' return data[2] + b'.'
def save_none(self, tag, data): def save_none(self, tag, data):
return b'N' return NONE
def save_bool(self, tag, data):
if data[2] == b'True':
return TRUE
else:
assert data[2] == b'False', data
return FALSE
def save_long(self, tag, data): def save_long(self, tag, data):
return b'L'+data[2]+b'L\012' return b'L'+data[2]+b'L\012'
...@@ -857,7 +982,9 @@ class xmlPickler(NoBlanks, xyap): ...@@ -857,7 +982,9 @@ class xmlPickler(NoBlanks, xyap):
'none': save_none, 'none': save_none,
'int': save_int, 'int': save_int,
'long': save_long, 'long': save_long,
'bool': save_bool,
'float': save_float, 'float': save_float,
'bytes': save_bytes,
'string': save_string, 'string': save_string,
'unicode': save_unicode, 'unicode': save_unicode,
'reference': save_reference, 'reference': save_reference,
......
product/ERP5Type/XMLExportImport/xyap.py
View file @ d25f925b
...@@ -24,7 +24,7 @@ import string ...@@ -24,7 +24,7 @@ import string
import xml.parsers.expat import xml.parsers.expat
class xyap: class xyap(object):
start_handlers = {} start_handlers = {}
end_handlers = {} end_handlers = {}
...@@ -57,7 +57,7 @@ class xyap: ...@@ -57,7 +57,7 @@ class xyap:
top = end[tag](self, tag, top) top = end[tag](self, tag, top)
append(top) append(top)
class NoBlanks: class NoBlanks(object):
def handle_data(self, data): def handle_data(self, data):
if data.strip(): if data.strip():
......
product/ERP5Type/tests/CodingStyleTestCase.py
View file @ d25f925b
...@@ -35,6 +35,7 @@ import glob ...@@ -35,6 +35,7 @@ import glob
import os import os
import shutil import shutil
import tempfile import tempfile
import warnings
from Acquisition import aq_base from Acquisition import aq_base
from Testing import ZopeTestCase from Testing import ZopeTestCase
...@@ -222,6 +223,11 @@ class CodingStyleTestCase(ERP5TypeTestCase): ...@@ -222,6 +223,11 @@ class CodingStyleTestCase(ERP5TypeTestCase):
if log_directory and diff_line_list: if log_directory and diff_line_list:
with open(os.path.join(log_directory, '%s.diff' % self.id()), 'w') as f: with open(os.path.join(log_directory, '%s.diff' % self.id()), 'w') as f:
f.writelines(diff_line_list) f.writelines(diff_line_list)
if diff_files and six.PY3: # TODO zope4py3
warnings.warn(
"Ignoring test_rebuild_business_template until we re-export "
"business templates with protocol 3.")
return
self.assertEqual(diff_files, []) self.assertEqual(diff_files, [])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7