Commit 376bc263 authored by Arnaud Fontaine's avatar Arnaud Fontaine

py3: XMLExportImport: Port to Python3.

parent 8be67a1f
...@@ -79,8 +79,7 @@ from OFS.Image import Pdata ...@@ -79,8 +79,7 @@ from OFS.Image import Pdata
from io import BytesIO as StringIO from io import BytesIO as StringIO
from copy import deepcopy from copy import deepcopy
from zExceptions import BadRequest from zExceptions import BadRequest
from Products.ERP5Type.XMLExportImport import exportXML from Products.ERP5Type.XMLExportImport import exportXML, customImporters
from OFS.ObjectManager import customImporters
from Products.ERP5Type.Workflow import WorkflowHistoryList from Products.ERP5Type.Workflow import WorkflowHistoryList
from zLOG import LOG, WARNING, INFO from zLOG import LOG, WARNING, INFO
from warnings import warn from warnings import warn
......
...@@ -235,7 +235,7 @@ from inspect import getargspec ...@@ -235,7 +235,7 @@ from inspect import getargspec
from OFS import ObjectManager from OFS import ObjectManager
from . import ppml from . import ppml
magic='<?xm' # importXML(jar, file, clue)} magic=b'<?xm' # importXML(jar, file, clue)}
def reorderPickle(jar, p): def reorderPickle(jar, p):
try: try:
...@@ -351,7 +351,7 @@ class zopedata: ...@@ -351,7 +351,7 @@ class zopedata:
def __init__(self, parser, tag, attrs): def __init__(self, parser, tag, attrs):
self.file=parser.file self.file=parser.file
write=self.file.write write=self.file.write
write('ZEXP') write(b'ZEXP')
def append(self, data): def append(self, data):
file=self.file file=self.file
...@@ -378,7 +378,7 @@ def save_record(parser, tag, data): ...@@ -378,7 +378,7 @@ def save_record(parser, tag, data):
a=data[1] a=data[1]
if 'id' in a: oid=a['id'] if 'id' in a: oid=a['id']
oid=p64(int(oid)) oid=p64(int(oid))
v='' v=b''
for x in data[2:]: for x in data[2:]:
v=v+x v=v+x
l=p64(len(v)) l=p64(len(v))
...@@ -402,6 +402,7 @@ def importXML(jar, file, clue=''): ...@@ -402,6 +402,7 @@ def importXML(jar, file, clue=''):
# So we have to declare an encoding but not use unicode, so the unpickler # So we have to declare an encoding but not use unicode, so the unpickler
# can deal with the utf-8 strings directly # can deal with the utf-8 strings directly
p=xml.parsers.expat.ParserCreate('utf-8') p=xml.parsers.expat.ParserCreate('utf-8')
if six.PY2:
p.returns_unicode = False p.returns_unicode = False
# </patch> # </patch>
p.CharacterDataHandler=F.handle_data p.CharacterDataHandler=F.handle_data
...@@ -410,3 +411,7 @@ def importXML(jar, file, clue=''): ...@@ -410,3 +411,7 @@ def importXML(jar, file, clue=''):
r=p.Parse(data) r=p.Parse(data)
outfile.seek(0) outfile.seek(0)
return jar.importFile(outfile,clue) return jar.importFile(outfile,clue)
customImporters = {
magic: importXML
}
...@@ -15,13 +15,16 @@ ...@@ -15,13 +15,16 @@
"""Provide conversion between Python pickles and XML """Provide conversion between Python pickles and XML
""" """
from pickle import * from zodbpickle.pickle import *
# Python3 C implementation does not have Unpickler.dispatch attribute
from zodbpickle.pickle_3 import _Unpickler as Unpickler
import struct import struct
import base64 import base64
import re import re
from marshal import loads as mloads from marshal import loads as mloads
from .xyap import NoBlanks from .xyap import NoBlanks
from .xyap import xyap from .xyap import xyap
from Products.ERP5Type.Utils import str2bytes
import six import six
from marshal import dumps as mdumps from marshal import dumps as mdumps
...@@ -29,6 +32,13 @@ from marshal import dumps as mdumps ...@@ -29,6 +32,13 @@ from marshal import dumps as mdumps
binary = re.compile('[^\x1f-\x7f]').search binary = re.compile('[^\x1f-\x7f]').search
if six.PY2:
data_encoding = 'raw_unicode_escape'
long_ = long
else:
data_encoding = 'utf-8'
long_ = int
def escape(s, encoding='repr'): def escape(s, encoding='repr'):
if binary(s) and isinstance(s, str): if binary(s) and isinstance(s, str):
s = base64.encodestring(s)[:-1] s = base64.encodestring(s)[:-1]
...@@ -47,9 +57,9 @@ def unescape(s, encoding): ...@@ -47,9 +57,9 @@ def unescape(s, encoding):
if encoding == 'base64': if encoding == 'base64':
return base64.decodestring(s) return base64.decodestring(s)
else: else:
s = s.replace('&lt;', '<') s = s.replace(b'&lt;', b'<')
s = s.replace('&gt;', '>') s = s.replace(b'&gt;', b'>')
return s.replace('&amp;', '&') return s.replace(b'&amp;', b'&')
# For converting to a more readable expression. # For converting to a more readable expression.
reprs = {} reprs = {}
...@@ -99,7 +109,7 @@ def unconvert(encoding,S): ...@@ -99,7 +109,7 @@ def unconvert(encoding,S):
if encoding == 'base64': if encoding == 'base64':
return base64.decodestring(S) return base64.decodestring(S)
else: else:
return eval("'" + S.replace('\n', '') + "'") return str2bytes(eval(b"'" + S.replace(b'\n', b'') + b"'"))
class Global: class Global:
def __init__(self, module, name, mapping): def __init__(self, module, name, mapping):
...@@ -338,7 +348,7 @@ class NoBlanks: ...@@ -338,7 +348,7 @@ class NoBlanks:
# Ignore element data between elements (eg '<e> <f> </f> </e>')... # Ignore element data between elements (eg '<e> <f> </f> </e>')...
if data.strip(): if data.strip():
if isinstance(data, six.text_type): if isinstance(data, six.text_type):
data = data.encode('raw_unicode_escape') data = data.encode(data_encoding)
self.append(data) self.append(data)
# Except for strings and unicode data as whitespaces should be # Except for strings and unicode data as whitespaces should be
...@@ -362,7 +372,7 @@ class NoBlanks: ...@@ -362,7 +372,7 @@ class NoBlanks:
self.previous_stack_end = None self.previous_stack_end = None
if isinstance(data, six.text_type): if isinstance(data, six.text_type):
data = data.encode('raw_unicode_escape') data = data.encode(data_encoding)
self.append(data) self.append(data)
...@@ -461,7 +471,7 @@ class ToXMLUnpickler(Unpickler): ...@@ -461,7 +471,7 @@ class ToXMLUnpickler(Unpickler):
return Pickle(Unpickler.load(self), self.id_mapping) return Pickle(Unpickler.load(self), self.id_mapping)
dispatch = {} dispatch = {}
dispatch.update(Unpickler.dispatch) dispatch.update(Unpickler.dispatch.copy())
def persistent_load(self, v): def persistent_load(self, v):
return Persistent(v, self.id_mapping) return Persistent(v, self.id_mapping)
...@@ -492,7 +502,7 @@ class ToXMLUnpickler(Unpickler): ...@@ -492,7 +502,7 @@ class ToXMLUnpickler(Unpickler):
dispatch[BININT2] = load_binint2 dispatch[BININT2] = load_binint2
def load_long(self): def load_long(self):
self.append(Long(long(self.readline()[:-1], 0), self.id_mapping)) self.append(Long(long_(self.readline()[:-1], 0), self.id_mapping))
dispatch[LONG] = load_long dispatch[LONG] = load_long
def load_float(self): def load_float(self):
...@@ -652,7 +662,7 @@ def ToXMLloads(str): ...@@ -652,7 +662,7 @@ def ToXMLloads(str):
return ToXMLUnpickler(file).load() return ToXMLUnpickler(file).load()
def name(self, tag, data): def name(self, tag, data):
return ''.join(data[2:]).strip() return b''.join(data[2:]).strip()
def start_pickle(self, tag, attrs): def start_pickle(self, tag, attrs):
self._pickleids = {} self._pickleids = {}
...@@ -663,19 +673,19 @@ def save_int(self, tag, data): ...@@ -663,19 +673,19 @@ def save_int(self, tag, data):
v = int(name(self, tag, data)) v = int(name(self, tag, data))
if v >= 0: if v >= 0:
if v <= 0xff: if v <= 0xff:
return BININT1 + chr(v) return BININT1 + bytes([v])
if v <= 0xffff: if v <= 0xffff:
return '%c%c%c' % (BININT2, v & 0xff, v >> 8) return BININT2 + b'%c%c' % (v & 0xff, v >> 8)
hb = v >> 31 hb = v >> 31
if hb == 0 or hb == -1: if hb == 0 or hb == -1:
return BININT + struct.pack('<i', v) return BININT + struct.pack('<i', v)
return INT + name(self, tag, data) + '\n' return INT + name(self, tag, data) + b'\n'
def save_float(self, tag, data): def save_float(self, tag, data):
if self.binary: if self.binary:
return BINFLOAT + struct.pack('>d', float(name(self, tag, data))) return BINFLOAT + struct.pack('>d', float(name(self, tag, data)))
else: else:
return FLOAT + name(self, tag, data) + '\n' return FLOAT + name(self, tag, data) + b'\n'
def save_put(self, v, attrs): def save_put(self, v, attrs):
id = attrs.get('id', '') id = attrs.get('id', '')
...@@ -688,38 +698,41 @@ def save_put(self, v, attrs): ...@@ -688,38 +698,41 @@ def save_put(self, v, attrs):
if self.binary: if self.binary:
id = int(id) id = int(id)
if id < 256: if id < 256:
id = BINPUT + chr(id) id = BINPUT + str2bytes(chr(id))
else: else:
id = LONG_BINPUT + struct.pack('<i', id) id = LONG_BINPUT + struct.pack('<i', id)
else: else:
id = PUT + repr(id) + '\n' id = PUT + repr(id) + b'\n'
return v + id return v + id
return v return v
def save_string(self, tag, data): def save_string(self, tag, data):
binary=self.binary a = data[1]
v='' v = b''.join(data[2:])
a=data[1] encoding = a.get('encoding', 'repr') # JPS: repr is default encoding
if len(data)>2:
v = ''.join(data[2:])
encoding=a.get('encoding','repr') # JPS: repr is default encoding
if encoding is not '': if encoding is not '':
v=unconvert(encoding,v) v = unconvert(encoding, v)
put='p' if self.binary:
if binary: l = len(v)
l=len(v) if l < 256:
s=mdumps(l)[1:] if encoding == 'base64':
if (l<256): op = SHORT_BINBYTES
v='U'+s[0]+v else:
op = SHORT_BINSTRING
v = op + bytes([l]) + v
else:
if encoding == 'base64':
op = BINBYTES
else:
op = BINSTRING
v = op + struct.pack('<i', l) + v
else: else:
v='T'+s+v v = STRING + repr(v) + '\n'
put='q'
else: v="S'"+v+"'\012"
return save_put(self, v, a) return save_put(self, v, a)
def save_unicode(self, tag, data): def save_unicode(self, tag, data):
binary=self.binary binary=self.binary
v='' v=b''
a=data[1] a=data[1]
if len(data)>2: if len(data)>2:
for x in data[2:]: for x in data[2:]:
...@@ -738,14 +751,14 @@ def save_tuple(self, tag, data): ...@@ -738,14 +751,14 @@ def save_tuple(self, tag, data):
T = data[2:] T = data[2:]
if not T: if not T:
return EMPTY_TUPLE return EMPTY_TUPLE
return save_put(self, MARK + ''.join(T) + TUPLE, data[1]) return save_put(self, MARK + b''.join(T) + TUPLE, data[1])
def save_list(self, tag, data): def save_list(self, tag, data):
L = data[2:] L = data[2:]
if self.binary: if self.binary:
v = save_put(self, EMPTY_LIST, data[1]) v = save_put(self, EMPTY_LIST, data[1])
if L: if L:
v = v + MARK + ''.join(L) + APPENDS v = v + MARK + b''.join(L) + APPENDS
else: else:
v = save_put(self, MARK + LIST, data[1]) v = save_put(self, MARK + LIST, data[1])
if L: if L:
...@@ -757,7 +770,7 @@ def save_dict(self, tag, data): ...@@ -757,7 +770,7 @@ def save_dict(self, tag, data):
if self.binary: if self.binary:
v = save_put(self, EMPTY_DICT, data[1]) v = save_put(self, EMPTY_DICT, data[1])
if D: if D:
v = v + MARK + ''.join(D) + SETITEMS v = v + MARK + b''.join(D) + SETITEMS
else: else:
v = save_put(self, MARK + DICT, data[1]) v = save_put(self, MARK + DICT, data[1])
if D: if D:
...@@ -773,34 +786,35 @@ def save_reference(self, tag, data): ...@@ -773,34 +786,35 @@ def save_reference(self, tag, data):
if self.binary: if self.binary:
id = int(id) id = int(id)
if id < 256: if id < 256:
return BINGET + chr(id) return BINGET + str2bytes(chr(id))
else: else:
return LONG_BINGET + struct.pack('<i', i) return LONG_BINGET + struct.pack('<i', i)
else: else:
return GET + repr(id) + '\n' return GET + repr(id) + b'\n'
def save_object(self, tag, data): def save_object(self, tag, data):
if len(data)==5: if len(data)==5:
#OBJECT #OBJECT
v='('+data[2] v=b'('+data[2]
x=data[3][1:] x=data[3][1:]
stop=x.rfind('t') # This seems stop=x.rfind(b't') # This seems
if stop>=0: x=x[:stop] # wrong! if stop>=0: x=x[:stop] # wrong!
v=save_put(self, v+x+'o', data[1]) v=save_put(self, v+x+b'o', data[1])
v=v+data[4]+'b' # state v=v+data[4]+b'b' # state
return v return v
else: else:
#REDUCE #REDUCE
#data does not contain state.(See Object.__setstate__ definition) #data does not contain state.(See Object.__setstate__ definition)
#So, we can assume that this is a reduce. (Yusei) #So, we can assume that this is a reduce. (Yusei)
v='('+data[2] v=b'('+data[2]
v=save_put(self, data[2]+data[3], data[1]) v=save_put(self, data[2]+data[3], data[1])
v=v+'R' v=v+b'R'
return v return v
def save_global(self, tag, data): def save_global(self, tag, data):
a = data[1] a = data[1]
return save_put(self, GLOBAL + a['module'] + '\n' + a['name'] + '\n', a) return save_put(self, GLOBAL + str2bytes(a['module']) + b'\n' +
str2bytes(a['name']) + b'\n', a)
def save_persis(self, tag, data): def save_persis(self, tag, data):
v = data[2] v = data[2]
...@@ -813,16 +827,16 @@ def save_pickle_start(self, tag, attrs): ...@@ -813,16 +827,16 @@ def save_pickle_start(self, tag, attrs):
return [tag, attrs] return [tag, attrs]
def save_pickle(self, tag, data): def save_pickle(self, tag, data):
return data[2] + '.' return data[2] + b'.'
def save_none(self, tag, data): def save_none(self, tag, data):
return 'N' return b'N'
def save_long(self, tag, data): def save_long(self, tag, data):
return 'L'+data[2]+'L\012' return b'L'+data[2]+b'L\012'
def save_item(self, tag, data): def save_item(self, tag, data):
return ''.join(data[2:]) return b''.join(data[2:])
def save_value(self, tag, data): def save_value(self, tag, data):
return data[2] return data[2]
...@@ -837,7 +851,7 @@ class xmlPickler(NoBlanks, xyap): ...@@ -837,7 +851,7 @@ class xmlPickler(NoBlanks, xyap):
if tag in end: if tag in end:
top = end[tag](self, tag, top) top = end[tag](self, tag, top)
if isinstance(top, six.text_type): if isinstance(top, six.text_type):
top = top.encode('raw_unicode_escape') top = top.encode(data_encoding)
append(top) append(top)
start_handlers={ start_handlers={
......
...@@ -94,21 +94,21 @@ class xmlrpc(NoBlanks, XYap): ...@@ -94,21 +94,21 @@ class xmlrpc(NoBlanks, XYap):
'param': lambda self, tag, data: data[2], 'param': lambda self, tag, data: data[2],
'value': lambda self, tag, data: data[2], 'value': lambda self, tag, data: data[2],
'i4': 'i4':
lambda self, tag, data, atoi=string.atoi, name=name: lambda self, tag, data, atoi=int, name=name:
atoi(name(self, tag, data)), atoi(name(self, tag, data)),
'int': 'int':
lambda self, tag, data, atoi=string.atoi, name=name: lambda self, tag, data, atoi=int, name=name:
atoi(name(self, tag, data)), atoi(name(self, tag, data)),
'boolean': 'boolean':
lambda self, tag, data, atoi=string.atoi, name=name: lambda self, tag, data, atoi=int, name=name:
atoi(name(self, tag, data)), atoi(name(self, tag, data)),
'string': lambda self, tag, data, join=string.join: 'string': lambda self, tag, data, join=''.join:
join(data[2:], ''), join(data[2:]),
'double': 'double':
lambda self, tag, data, atof=string.atof, name=name: lambda self, tag, data, atof=float, name=name:
atof(name(self, tag, data)), atof(name(self, tag, data)),
'float': 'float':
lambda self, tag, data, atof=string.atof, name=name: lambda self, tag, data, atof=float, name=name:
atof(name(self, tag, data)), atof(name(self, tag, data)),
'struct': struct, 'struct': struct,
'member': tuplef, 'member': tuplef,
......
...@@ -14,12 +14,7 @@ ...@@ -14,12 +14,7 @@
# Import: add rename feature and make _importObjectFromFile return the object # Import: add rename feature and make _importObjectFromFile return the object
from OFS.ObjectManager import ObjectManager from OFS.ObjectManager import ObjectManager
from OFS.XMLExportImport import importXML from Products.ERP5Type.XMLExportImport import customImporters
from OFS.XMLExportImport import magic
customImporters={magic: importXML,
}
def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id=None, suppress_events=False): def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id=None, suppress_events=False):
#LOG('_importObjectFromFile, filepath',0,filepath) #LOG('_importObjectFromFile, filepath',0,filepath)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment