Commit d303ca49 authored by Arnaud Fontaine's avatar Arnaud Fontaine

zope4: XML Export/Import feature was removed from Zope4.

Only ZEXP Export/Import is possible. These modules (namely OFS.XMLExportImport
and Shared.DC.xml.*) were heavily monkey-patched anyway and are only used for
BusinessTemplates.

* ERP5Type/XMLExportImport.py  => ERP5Type/XMLExportImport/__init__.py
* OFS/XMLExportImport.py       => ERP5Type/XMLExportImport/__init__.py
* Shared/DC/xml/{xyap,ppml}.py => ERP5Type/XMLExportImport/{xyap,ppml}.py
parent c020b284
Pipeline #20997 failed with stage
in 0 seconds
...@@ -31,7 +31,7 @@ import pickle ...@@ -31,7 +31,7 @@ import pickle
import re import re
import xml.parsers.pyexpat import xml.parsers.pyexpat
from StringIO import StringIO from StringIO import StringIO
from Shared.DC.xml import ppml from Products.ERP5Type.XMLExportImport import ppml
class DummyClass: class DummyClass:
......
...@@ -71,16 +71,13 @@ from Products.ERP5Type.dynamic.portal_type_class import synchronizeDynamicModule ...@@ -71,16 +71,13 @@ from Products.ERP5Type.dynamic.portal_type_class import synchronizeDynamicModule
from Products.ERP5Type.Core.PropertySheet import PropertySheet as PropertySheetDocument from Products.ERP5Type.Core.PropertySheet import PropertySheet as PropertySheetDocument
from Products.ERP5Type.TransactionalVariable import getTransactionalVariable from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
from OFS.Traversable import NotFound from OFS.Traversable import NotFound
from OFS import SimpleItem, XMLExportImport from OFS import SimpleItem
from OFS.Image import Pdata from OFS.Image import Pdata
from cStringIO import StringIO from cStringIO import StringIO
from copy import deepcopy from copy import deepcopy
from zExceptions import BadRequest from zExceptions import BadRequest
import OFS.XMLExportImport from Products.ERP5Type.XMLExportImport import exportXML
from Products.ERP5Type.patches.ppml import importXML from OFS.ObjectManager import customImporters
customImporters={
XMLExportImport.magic: importXML,
}
from Products.ERP5Type.Workflow import WorkflowHistoryList from Products.ERP5Type.Workflow import WorkflowHistoryList
from zLOG import LOG, WARNING, INFO from zLOG import LOG, WARNING, INFO
from warnings import warn from warnings import warn
...@@ -858,7 +855,7 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -858,7 +855,7 @@ class ObjectTemplateItem(BaseTemplateItem):
transaction.savepoint(optimistic=True) transaction.savepoint(optimistic=True)
f = StringIO() f = StringIO()
XMLExportImport.exportXML(obj._p_jar, obj._p_oid, f) exportXML(obj._p_jar, obj._p_oid, f)
bta.addObject(f, key, path=path) bta.addObject(f, key, path=path)
if catalog_method_template_item: if catalog_method_template_item:
...@@ -1015,8 +1012,8 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -1015,8 +1012,8 @@ class ObjectTemplateItem(BaseTemplateItem):
pass pass
#LOG('Business Template', 0, 'Compiling %s...' % (name,)) #LOG('Business Template', 0, 'Compiling %s...' % (name,))
from Shared.DC.xml import ppml from Products.ERP5Type.XMLExportImport import (ppml,
from OFS.XMLExportImport import start_zopedata, save_record, save_zopedata start_zopedata, save_record, save_zopedata)
import xml.parsers.expat import xml.parsers.expat
outfile=StringIO() outfile=StringIO()
try: try:
...@@ -1069,10 +1066,10 @@ class ObjectTemplateItem(BaseTemplateItem): ...@@ -1069,10 +1066,10 @@ class ObjectTemplateItem(BaseTemplateItem):
new_object = self._objects[path] new_object = self._objects[path]
new_io = StringIO() new_io = StringIO()
old_io = StringIO() old_io = StringIO()
OFS.XMLExportImport.exportXML(new_object._p_jar, new_object._p_oid, new_io) exportXML(new_object._p_jar, new_object._p_oid, new_io)
new_obj_xml = new_io.getvalue() new_obj_xml = new_io.getvalue()
try: try:
OFS.XMLExportImport.exportXML(old_object._p_jar, old_object._p_oid, old_io) exportXML(old_object._p_jar, old_object._p_oid, old_io)
old_obj_xml = old_io.getvalue() old_obj_xml = old_io.getvalue()
except (ImportError, UnicodeDecodeError), e: # module is already except (ImportError, UnicodeDecodeError), e: # module is already
# removed etc. # removed etc.
...@@ -6167,8 +6164,8 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6167,8 +6164,8 @@ Business Template is a set of definitions, such as skins, portal types and categ
new_object = new_item.removeProperties(new_object, 1) new_object = new_item.removeProperties(new_object, 1)
installed_object = installed_item.removeProperties(installed_object, 1) installed_object = installed_item.removeProperties(installed_object, 1)
# XML Export in memory # XML Export in memory
OFS.XMLExportImport.exportXML(new_object._p_jar, new_object._p_oid, f1) exportXML(new_object._p_jar, new_object._p_oid, f1)
OFS.XMLExportImport.exportXML(installed_object._p_jar, exportXML(installed_object._p_jar,
installed_object._p_oid, f2) installed_object._p_oid, f2)
new_obj_xml = f1.getvalue() new_obj_xml = f1.getvalue()
f1.close() f1.close()
...@@ -6503,6 +6500,8 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6503,6 +6500,8 @@ Business Template is a set of definitions, such as skins, portal types and categ
'Products.ERP5Type.interfaces.json_representable', 'Products.ERP5Type.interfaces.json_representable',
'Products.ERP5Type.mixin.json_representable', 'Products.ERP5Type.mixin.json_representable',
'Products.ERP5Type.XMLExportImport', 'Products.ERP5Type.XMLExportImport',
'Products.ERP5Type.XMLExportImport.ppml',
'Products.ERP5Type.XMLExportImport.xyap',
'Products.ERP5Type.mixin.property_translatable', 'Products.ERP5Type.mixin.property_translatable',
'Products.ERP5Type.Error', 'Products.ERP5Type.Error',
'Products.ERP5Type.Errors', 'Products.ERP5Type.Errors',
......
...@@ -32,7 +32,7 @@ import xml.dom.minidom ...@@ -32,7 +32,7 @@ import xml.dom.minidom
from urllib import url2pathname from urllib import url2pathname
from ZODB.DemoStorage import DemoStorage from ZODB.DemoStorage import DemoStorage
from ZODB import DB from ZODB import DB
from OFS.XMLExportImport import importXML from Products.ERP5Type.XMLExportImport import importXML
if int(os.environ.get('erp5_report_new_simulation_failures') or 0): if int(os.environ.get('erp5_report_new_simulation_failures') or 0):
newSimulationExpectedFailure = lambda test: test newSimulationExpectedFailure = lambda test: test
......
##############################################################################
#
# Copyright (c) 2001,2002 Zope Foundation and Contributors.
# Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
############################################################################## ##############################################################################
# #
# Copyright (c) 2002-2003 Nexedi SARL and Contributors. All Rights Reserved. # Copyright (c) 2001,2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
# Copyright (c) 2002-2005 Nexedi SARL and Contributors. All Rights Reserved.
# Sebastien Robin <seb@nexedi.com> # Sebastien Robin <seb@nexedi.com>
# #
# WARNING: This program as such is intended to be used by professional # WARNING: This program as such is intended to be used by professional
...@@ -27,8 +36,8 @@ ...@@ -27,8 +36,8 @@
# #
############################################################################## ##############################################################################
## The code below was initially in ERP5Type/XMLExportImport.py
from Acquisition import aq_base, aq_inner from Acquisition import aq_base, aq_inner
from collections import OrderedDict from collections import OrderedDict
from cStringIO import StringIO from cStringIO import StringIO
from zodbpickle.pickle import Pickler from zodbpickle.pickle import Pickler
...@@ -38,10 +47,9 @@ from lxml import etree ...@@ -38,10 +47,9 @@ from lxml import etree
from lxml.etree import Element, SubElement from lxml.etree import Element, SubElement
from xml_marshaller.xml_marshaller import Marshaller from xml_marshaller.xml_marshaller import Marshaller
from OFS.Image import Pdata from OFS.Image import Pdata
from zLOG import LOG
from base64 import standard_b64encode from base64 import standard_b64encode
from hashlib import sha1 from hashlib import sha1
#from zLOG import LOG
MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller' MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller'
marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI, marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI,
...@@ -207,3 +215,191 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None): ...@@ -207,3 +215,191 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None):
return etree.tostring(root, encoding='utf-8', return etree.tostring(root, encoding='utf-8',
xml_declaration=xml_declaration, pretty_print=True) xml_declaration=xml_declaration, pretty_print=True)
## The code below was initially from OFS.XMLExportImport
from base64 import encodestring
from ZODB.serialize import referencesf
from ZODB.ExportImport import TemporaryFile, export_end_marker
from ZODB.utils import p64
from ZODB.utils import u64
from functools import partial
from inspect import getargspec
from types import TupleType
from OFS import ObjectManager
from . import ppml
magic='<?xm' # importXML(jar, file, clue)}
def reorderPickle(jar, p):
try:
from ZODB._compat import Unpickler, Pickler
except ImportError: # BBB: ZODB 3.10
from ZODB.ExportImport import Unpickler, Pickler
from ZODB.ExportImport import Ghost, persistent_id
oids = {}
storage = jar._storage
new_oid = storage.new_oid
store = storage.store
def persistent_load(ooid,
Ghost=Ghost,
oids=oids, wrote_oid=oids.has_key,
new_oid=storage.new_oid):
"Remap a persistent id to an existing ID and create a ghost for it."
if type(ooid) is TupleType: ooid, klass = ooid
else: klass=None
try:
Ghost=Ghost()
Ghost.oid=ooid
except TypeError:
Ghost=Ghost(ooid)
return Ghost
# Reorder pickle by doing I/O
pfile = StringIO(p)
unpickler=Unpickler(pfile)
unpickler.persistent_load=persistent_load
newp=StringIO()
pickler=OrderedPickler(newp,1)
pickler.persistent_id=persistent_id
classdef = unpickler.load()
obj = unpickler.load()
pickler.dump(classdef)
pickler.dump(obj)
p=newp.getvalue()
return obj, p
def _mapOid(id_mapping, oid):
idprefix = str(u64(oid))
id = id_mapping[idprefix]
old_aka = encodestring(oid)[:-1]
aka=encodestring(p64(long(id)))[:-1] # Rebuild oid based on mapped id
id_mapping.setConvertedAka(old_aka, aka)
return idprefix+'.', id, aka
def XMLrecord(oid, plen, p, id_mapping):
# Proceed as usual
q=ppml.ToXMLUnpickler
f=StringIO(p)
u=q(f)
u.idprefix, id, aka = _mapOid(id_mapping, oid)
p=u.load(id_mapping=id_mapping).__str__(4)
if f.tell() < plen:
p=p+u.load(id_mapping=id_mapping).__str__(4)
String=' <record id="%s" aka="%s">\n%s </record>\n' % (id, aka, p)
return String
def exportXML(jar, oid, file=None):
# For performance reasons, exportXML does not use 'XMLrecord' anymore to map
# oids. This requires to initialize MinimalMapping.marked_reference before
# any string output, i.e. in ppml.Reference.__init__
# This also fixed random failures when DemoStorage is used, because oids
# can have values that have a shorter representation in 'repr' instead of
# 'base64' (see ppml.convert) and ppml.String does not support this.
load = jar._storage.load
if 'version' in getargspec(load).args: # BBB: ZODB<5 (TmpStore)
load = partial(load, version='')
pickle_dict = {oid: None}
max_cache = [1e7] # do not cache more than 10MB of pickle data
def getReorderedPickle(oid):
p = pickle_dict[oid]
if p is None:
p = load(oid)[0]
p = reorderPickle(jar, p)[1]
if len(p) < max_cache[0]:
max_cache[0] -= len(p)
pickle_dict[oid] = p
return p
# Sort records and initialize id_mapping
id_mapping = ppml.MinimalMapping()
reordered_oid_list = [oid]
for oid in reordered_oid_list:
_mapOid(id_mapping, oid)
for oid in referencesf(getReorderedPickle(oid)):
if oid not in pickle_dict:
pickle_dict[oid] = None
reordered_oid_list.append(oid)
# Do real export
if file is None:
file = TemporaryFile()
elif isinstance(file, basestring):
file = open(file, 'w+b')
write = file.write
write('<?xml version="1.0"?>\n<ZopeData>\n')
for oid in reordered_oid_list:
p = getReorderedPickle(oid)
write(XMLrecord(oid, len(p), p, id_mapping))
write('</ZopeData>\n')
return file
class zopedata:
def __init__(self, parser, tag, attrs):
self.file=parser.file
write=self.file.write
write('ZEXP')
def append(self, data):
file=self.file
write=file.write
pos=file.tell()
file.seek(pos)
write(data)
def start_zopedata(parser, tag, data):
return zopedata(parser, tag, data)
def save_zopedata(parser, tag, data):
file=parser.file
write=file.write
pos=file.tell()
file.seek(pos)
write(export_end_marker)
def save_record(parser, tag, data):
file=parser.file
write=file.write
pos=file.tell()
file.seek(pos)
a=data[1]
if a.has_key('id'): oid=a['id']
oid=p64(int(oid))
v=''
for x in data[2:]:
v=v+x
l=p64(len(v))
v=oid+l+v
return v
import xml.parsers.expat
def importXML(jar, file, clue=''):
if type(file) is str:
file=open(file, 'rb')
outfile=TemporaryFile()
data=file.read()
F=ppml.xmlPickler()
F.end_handlers['record'] = save_record
F.end_handlers['ZopeData'] = save_zopedata
F.start_handlers['ZopeData'] = start_zopedata
F.binary=1
F.file=outfile
# <patch>
# Our BTs XML files don't declare encoding but have accented chars in them
# So we have to declare an encoding but not use unicode, so the unpickler
# can deal with the utf-8 strings directly
p=xml.parsers.expat.ParserCreate('utf-8')
p.returns_unicode = False
# </patch>
p.CharacterDataHandler=F.handle_data
p.StartElementHandler=F.unknown_starttag
p.EndElementHandler=F.unknown_endtag
r=p.Parse(data)
outfile.seek(0)
return jar.importFile(outfile,clue)
# -*- coding: utf-8 -*-
############################################################################## ##############################################################################
# #
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. # Copyright (c) 2001,2002 Zope Corporation and Contributors. All Rights Reserved.
# Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved. # Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved.
# #
# This software is subject to the provisions of the Zope Public License, # This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE # FOR A PARTICULAR PURPOSE
# #
############################################################################## ##############################################################################
"""Provide conversion between Python pickles and XML
"""
from pickle import *
import struct
import base64
import re import re
from marshal import loads as mloads
from .xyap import NoBlanks
from .xyap import xyap
# Import everything right now, not after import re
# or new patch will not work
from Shared.DC.xml.ppml import *
from Shared.DC.xml import ppml
from marshal import dumps as mdumps from marshal import dumps as mdumps
from zLOG import LOG #from zLOG import LOG
binary = re.compile('[^\x1f-\x7f]').search
def escape(s, encoding='repr'):
if binary(s) and isinstance(s, str):
s = base64.encodestring(s)[:-1]
encoding = 'base64'
elif '>' in s or '<' in s or '&' in s:
if not ']]>' in s:
s = '<![CDATA[' + s + ']]>'
encoding = 'cdata'
else:
s = s.replace('&', '&amp;')
s = s.replace('>', '&gt;')
s = s.replace('<', '&lt;')
return encoding, s
def unescape(s, encoding):
if encoding == 'base64':
return base64.decodestring(s)
else:
s = s.replace('&lt;', '<')
s = s.replace('&gt;', '>')
return s.replace('&amp;', '&')
# For converting to a more readable expression. # For converting to a more readable expression.
reprs = {} reprs = {}
...@@ -38,15 +65,12 @@ reprs2={} ...@@ -38,15 +65,12 @@ reprs2={}
reprs2['<'] = "\\074" reprs2['<'] = "\\074"
reprs2['>'] = "\\076" reprs2['>'] = "\\076"
reprs2['&'] = "\\046" reprs2['&'] = "\\046"
reprs_re = re.compile('|'.join(re.escape(k) for k in reprs.keys())) reprs_re = re.compile('|'.join(re.escape(k) for k in reprs.keys()))
def sub_reprs(m): def sub_reprs(m):
return reprs[m.group(0)] return reprs[m.group(0)]
reprs2_re = re.compile('|'.join(re.escape(k) for k in reprs2.keys())) reprs2_re = re.compile('|'.join(re.escape(k) for k in reprs2.keys()))
def sub_reprs2(m): def sub_reprs2(m):
return reprs2[m.group(0)] return reprs2[m.group(0)]
def convert(S): def convert(S):
new = '' new = ''
### patch begin: if the input string is a valid utf8 string, only ### patch begin: if the input string is a valid utf8 string, only
...@@ -69,8 +93,6 @@ def convert(S): ...@@ -69,8 +93,6 @@ def convert(S):
return 'repr', reprs2_re.sub(sub_reprs2, new) return 'repr', reprs2_re.sub(sub_reprs2, new)
return 'repr', new return 'repr', new
ppml.convert = convert
# For optimization. # For optimization.
def unconvert(encoding,S): def unconvert(encoding,S):
if encoding == 'base64': if encoding == 'base64':
...@@ -78,10 +100,7 @@ def unconvert(encoding,S): ...@@ -78,10 +100,7 @@ def unconvert(encoding,S):
else: else:
return eval("'" + S.replace('\n', '') + "'") return eval("'" + S.replace('\n', '') + "'")
ppml.unconvert = unconvert
class Global: class Global:
def __init__(self, module, name, mapping): def __init__(self, module, name, mapping):
self.module=module self.module=module
self.name=name self.name=name
...@@ -95,10 +114,14 @@ class Global: ...@@ -95,10 +114,14 @@ class Global:
return '%s<%s%s name="%s" module="%s"/>\n' % ( return '%s<%s%s name="%s" module="%s"/>\n' % (
' '*indent, name, id, self.name, self.module) ' '*indent, name, id, self.name, self.module)
ppml.Global = Global class Immutable:
def __init__(self, value):
self.value = value
class Scalar: def getValue(self):
return self.value
class Scalar:
def __init__(self, v, mapping): def __init__(self, v, mapping):
self._v=v self._v=v
self.mapping = mapping self.mapping = mapping
...@@ -116,17 +139,14 @@ class Scalar: ...@@ -116,17 +139,14 @@ class Scalar:
self.mapping.setImmutable(self.id, Immutable(value = result)) self.mapping.setImmutable(self.id, Immutable(value = result))
return result return result
ppml.Scalar = Scalar class Long(Scalar):
def value(self):
class Immutable: result = str(self._v)
def __init__(self, value): if result[-1:] == 'L':
self.value = value return result[:-1]
return result
def getValue(self):
return self.value
class String(Scalar): class String(Scalar):
encoding = None encoding = None
def __init__(self, v, mapping, encoding=''): def __init__(self, v, mapping, encoding=''):
...@@ -160,16 +180,11 @@ class String(Scalar): ...@@ -160,16 +180,11 @@ class String(Scalar):
self.mapping.setImmutable(self.id, Immutable(value = result)) self.mapping.setImmutable(self.id, Immutable(value = result))
return '%s%s\n' % (' '*indent, result) return '%s%s\n' % (' '*indent, result)
ppml.String = String
class Unicode(String): class Unicode(String):
def value(self): def value(self):
return self._v.encode('utf-8') return self._v.encode('utf-8')
ppml.Unicode = Unicode
class Wrapper: class Wrapper:
def __init__(self, v, mapping): def __init__(self, v, mapping):
self._v=v self._v=v
self.mapping = mapping self.mapping = mapping
...@@ -189,10 +204,7 @@ class Wrapper: ...@@ -189,10 +204,7 @@ class Wrapper:
v=v.__str__(indent+2) v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name) return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
ppml.Wrapper = Wrapper
class Collection: class Collection:
def __init__(self, mapping): def __init__(self, mapping):
self.mapping = mapping self.mapping = mapping
...@@ -208,14 +220,15 @@ class Collection: ...@@ -208,14 +220,15 @@ class Collection:
else: else:
return '%s<%s%s/>\n' % (i, name, id) return '%s<%s%s/>\n' % (i, name, id)
ppml.Collection = Collection
class Dictionary(Collection): class Dictionary(Collection):
def __init__(self, mapping): def __init__(self, mapping):
self.mapping = mapping self.mapping = mapping
self._d=[] self._d=[]
def __len__(self): return len(self._d) def __len__(self): return len(self._d)
def __setitem__(self, k, v): self._d.append((k,v)) def __setitem__(self, k, v): self._d.append((k,v))
def value(self, indent): def value(self, indent):
#self._d.sort(lambda a, b: cmp(a[0]._v, b[0]._v)) # Sort the sequence by key JPS Improvement #self._d.sort(lambda a, b: cmp(a[0]._v, b[0]._v)) # Sort the sequence by key JPS Improvement
ind = ' ' * indent ind = ' ' * indent
...@@ -230,10 +243,7 @@ class Dictionary(Collection): ...@@ -230,10 +243,7 @@ class Dictionary(Collection):
for i in self._d for i in self._d
) )
ppml.Dictionary = Dictionary
class Sequence(Collection): class Sequence(Collection):
def __init__(self, mapping, v=None): def __init__(self, mapping, v=None):
if not v: v=[] if not v: v=[]
self._subs=v self._subs=v
...@@ -251,26 +261,10 @@ class Sequence(Collection): ...@@ -251,26 +261,10 @@ class Sequence(Collection):
v.__str__(indent) for v in v.__str__(indent) for v in
self._subs) self._subs)
ppml.Sequence = Sequence class none:
class Persistent(Wrapper):
def __str__(self, indent=0): def __str__(self, indent=0):
id = '' return ' ' * indent + '<none/>\n'
if hasattr(self, 'id'): none = none()
if self.mapping.isMarked(self.id): id=' id="%s"' % self.mapping[self.id]
name=self.__class__.__name__.lower()
v=self._v
i=' '*indent
if isinstance(v,String):
return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name)
elif isinstance(v,Scalar):
return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name)
else:
v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
ppml.Persistent = Persistent
class Reference(Scalar): class Reference(Scalar):
def __init__(self, v, mapping): def __init__(self, v, mapping):
...@@ -288,9 +282,7 @@ class Reference(Scalar): ...@@ -288,9 +282,7 @@ class Reference(Scalar):
value = '<%s id="%s"/>' % (name, self.mapping[v]) value = '<%s id="%s"/>' % (name, self.mapping[v])
return '%s%s\n' % (' '*indent, value) return '%s%s\n' % (' '*indent, value)
ppml.Reference = Reference
Get = Reference Get = Reference
ppml.Get = Get
class Object(Sequence): class Object(Sequence):
def __init__(self, klass, args, mapping): def __init__(self, klass, args, mapping):
...@@ -299,10 +291,33 @@ class Object(Sequence): ...@@ -299,10 +291,33 @@ class Object(Sequence):
def __setstate__(self, v): self.append(State(v, self.mapping)) def __setstate__(self, v): self.append(State(v, self.mapping))
ppml.Object = Object class Int(Scalar): pass
class Float(Scalar): pass
class List(Sequence): pass
class Tuple(Sequence): pass
class Key(Wrapper): pass
class Value(Wrapper): pass
class Klass(Wrapper): pass
class State(Wrapper): pass
class Pickle(Wrapper): pass
class Persistent(Wrapper):
def __str__(self, indent=0):
id = ''
if hasattr(self, 'id'):
if self.mapping.isMarked(self.id): id=' id="%s"' % self.mapping[self.id]
name=self.__class__.__name__.lower()
v=self._v
i=' '*indent
if isinstance(v,String):
return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name)
elif isinstance(v,Scalar):
return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name)
else:
v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
blanck_line_expression = re.compile('^ +$') blanck_line_expression = re.compile('^ +$')
class NoBlanks: class NoBlanks:
""" """
This allows to ignore at least whitespaces between elements and also This allows to ignore at least whitespaces between elements and also
...@@ -350,10 +365,7 @@ class NoBlanks: ...@@ -350,10 +365,7 @@ class NoBlanks:
self.append(data) self.append(data)
ppml.NoBlanks = NoBlanks
class IdentityMapping: class IdentityMapping:
def __init__(self): def __init__(self):
self.resetMapping() self.resetMapping()
self.immutable = {} self.immutable = {}
...@@ -385,11 +397,7 @@ class IdentityMapping: ...@@ -385,11 +397,7 @@ class IdentityMapping:
def hasImmutable(self, k): def hasImmutable(self, k):
return self.immutable.has_key(k) return self.immutable.has_key(k)
ppml.IdentityMapping = IdentityMapping
class MinimalMapping(IdentityMapping): class MinimalMapping(IdentityMapping):
def resetMapping(self): def resetMapping(self):
self.mapped_id = {} self.mapped_id = {}
self.mapped_core_id = {} self.mapped_core_id = {}
...@@ -443,30 +451,7 @@ class MinimalMapping(IdentityMapping): ...@@ -443,30 +451,7 @@ class MinimalMapping(IdentityMapping):
def __str__(self, a): def __str__(self, a):
return "Error here" return "Error here"
ppml.MinimalMapping = MinimalMapping
class List(Sequence): pass
class Tuple(Sequence): pass
class Klass(Wrapper): pass
class State(Wrapper): pass
class Pickle(Wrapper): pass
class Int(Scalar): pass
class Float(Scalar): pass
class Key(Wrapper): pass
class Value(Wrapper): pass
class Long(Scalar):
def value(self):
result = str(self._v)
if result[-1:] == 'L':
return result[:-1]
return result
class ToXMLUnpickler(Unpickler): class ToXMLUnpickler(Unpickler):
def load(self, id_mapping=None): def load(self, id_mapping=None):
if id_mapping is None: if id_mapping is None:
self.id_mapping = IdentityMapping() self.id_mapping = IdentityMapping()
...@@ -657,7 +642,58 @@ class ToXMLUnpickler(Unpickler): ...@@ -657,7 +642,58 @@ class ToXMLUnpickler(Unpickler):
#for code in dispatch.keys(): #for code in dispatch.keys():
# dispatch[code] = LogCall(dispatch[code]) # dispatch[code] = LogCall(dispatch[code])
ppml.ToXMLUnpickler = ToXMLUnpickler def ToXMLload(file):
return ToXMLUnpickler(file).load()
def ToXMLloads(str):
from StringIO import StringIO
file = StringIO(str)
return ToXMLUnpickler(file).load()
def name(self, tag, data):
return ''.join(data[2:]).strip()
def start_pickle(self, tag, attrs):
self._pickleids = {}
return [tag, attrs]
def save_int(self, tag, data):
if self.binary:
v = int(name(self, tag, data))
if v >= 0:
if v <= 0xff:
return BININT1 + chr(v)
if v <= 0xffff:
return '%c%c%c' % (BININT2, v & 0xff, v >> 8)
hb = v >> 31
if hb == 0 or hb == -1:
return BININT + struct.pack('<i', v)
return INT + name(self, tag, data) + '\n'
def save_float(self, tag, data):
if self.binary:
return BINFLOAT + struct.pack('>d', float(name(self, tag, data)))
else:
return FLOAT + name(self, tag, data) + '\n'
def save_put(self, v, attrs):
id = attrs.get('id', '')
if id:
prefix = id.rfind('.')
if prefix >= 0:
id = id[prefix + 1:]
elif id[0] == 'i':
id = id[1:]
if self.binary:
id = int(id)
if id < 256:
id = BINPUT + chr(id)
else:
id = LONG_BINPUT + struct.pack('<i', id)
else:
id = PUT + repr(id) + '\n'
return v + id
return v
def save_string(self, tag, data): def save_string(self, tag, data):
binary=self.binary binary=self.binary
...@@ -680,8 +716,6 @@ def save_string(self, tag, data): ...@@ -680,8 +716,6 @@ def save_string(self, tag, data):
else: v="S'"+v+"'\012" else: v="S'"+v+"'\012"
return save_put(self, v, a) return save_put(self, v, a)
ppml.save_string = save_string
def save_unicode(self, tag, data): def save_unicode(self, tag, data):
binary=self.binary binary=self.binary
v='' v=''
...@@ -699,7 +733,50 @@ def save_unicode(self, tag, data): ...@@ -699,7 +733,50 @@ def save_unicode(self, tag, data):
else: v=UNICODE+"'"+v+"'\012" else: v=UNICODE+"'"+v+"'\012"
return save_put(self, v, a) return save_put(self, v, a)
ppml.save_unicode = save_unicode def save_tuple(self, tag, data):
T = data[2:]
if not T:
return EMPTY_TUPLE
return save_put(self, MARK + ''.join(T) + TUPLE, data[1])
def save_list(self, tag, data):
L = data[2:]
if self.binary:
v = save_put(self, EMPTY_LIST, data[1])
if L:
v = v + MARK + ''.join(L) + APPENDS
else:
v = save_put(self, MARK + LIST, data[1])
if L:
v = APPEND.join(L) + APPEND
return v
def save_dict(self, tag, data):
D = data[2:]
if self.binary:
v = save_put(self, EMPTY_DICT, data[1])
if D:
v = v + MARK + ''.join(D) + SETITEMS
else:
v = save_put(self, MARK + DICT, data[1])
if D:
v = v + SETITEM.join(D) + SETITEM
return v
def save_reference(self, tag, data):
a = data[1]
id = a['id']
prefix = id.rfind('.')
if prefix >= 0:
id = id[prefix + 1:]
if self.binary:
id = int(id)
if id < 256:
return BINGET + chr(id)
else:
return LONG_BINGET + struct.pack('<i', i)
else:
return GET + repr(id) + '\n'
def save_object(self, tag, data): def save_object(self, tag, data):
if len(data)==5: if len(data)==5:
...@@ -720,7 +797,16 @@ def save_object(self, tag, data): ...@@ -720,7 +797,16 @@ def save_object(self, tag, data):
v=v+'R' v=v+'R'
return v return v
ppml.save_object = save_object def save_global(self, tag, data):
a = data[1]
return save_put(self, GLOBAL + a['module'] + '\n' + a['name'] + '\n', a)
def save_persis(self, tag, data):
v = data[2]
if self.binary:
return v + BINPERSID
else:
return PERSID + v
def save_pickle_start(self, tag, attrs): def save_pickle_start(self, tag, attrs):
return [tag, attrs] return [tag, attrs]
...@@ -778,41 +864,3 @@ class xmlPickler(NoBlanks, xyap): ...@@ -778,41 +864,3 @@ class xmlPickler(NoBlanks, xyap):
'persistent': save_persis, 'persistent': save_persis,
} }
# FIXME: Leo: Do we still need to replace ppml.xmlPickler now that we're
# using our own xmlPickler in our own importXML function below? Do we support
# any other use of xmlPickler except for Business Template export/import?
ppml.xmlPickler = xmlPickler
class Tuple(Sequence): pass
ppml.Tuple = Tuple
# Copied from OFS.XMLExportImport.importXML (of Zope 2.12)
# Imported and used directly by Products.ERP5.Document.BusinessTemplate
from OFS.XMLExportImport import save_record, save_zopedata, start_zopedata
from tempfile import TemporaryFile
import xml.parsers.expat
def importXML(jar, file, clue=''):
if type(file) is str:
file=open(file, 'rb')
outfile=TemporaryFile()
data=file.read()
F=xmlPickler()
F.end_handlers['record'] = save_record
F.end_handlers['ZopeData'] = save_zopedata
F.start_handlers['ZopeData'] = start_zopedata
F.binary=1
F.file=outfile
# <patch>
# Our BTs XML files don't declare encoding but have accented chars in them
# So we have to declare an encoding but not use unicode, so the unpickler
# can deal with the utf-8 strings directly
p=xml.parsers.expat.ParserCreate('utf-8')
p.returns_unicode = False
# </patch>
p.CharacterDataHandler=F.handle_data
p.StartElementHandler=F.unknown_starttag
p.EndElementHandler=F.unknown_endtag
r=p.Parse(data)
outfile.seek(0)
return jar.importFile(outfile,clue)
"""Yet another XML parser
This is meant to be very simple:
- stack based
- The parser has a table of start handlers and end handlers.
- start tag handlers are called with the parser instance, tag names
and attributes. The result is placed on the stack. The default
handler places a special object on the stack (uh, a list, with the
tag name and attributes as the first two elements. ;)
- end tag handlers are called with the object on the parser, the tag
name, and top of the stack right after it has been removed. The
result is appended to the object on the top of the stack.
Note that namespace attributes should recieve some special handling.
Oh well.
"""
import string
import xml.parsers.expat
class xyap:
start_handlers = {}
end_handlers = {}
def __init__(self):
top = []
self._stack = _stack = [top]
self.push = _stack.append
self.append = top.append
def handle_data(self, data):
self.append(data)
def unknown_starttag(self, tag, attrs):
if isinstance(attrs, list):
attrs = dict(attrs)
start = self.start_handlers
if tag in start:
tag = start[tag](self, tag, attrs)
else:
tag = [tag, attrs]
self.push(tag)
self.append = tag.append
def unknown_endtag(self, tag):
_stack = self._stack
top = _stack.pop()
append = self.append = _stack[-1].append
end = self.end_handlers
if tag in end:
top = end[tag](self, tag, top)
append(top)
class NoBlanks:
def handle_data(self, data):
if data.strip():
self.append(data)
def struct(self, tag, data):
r = {}
for k, v in data[2:]:
r[k] = v
return r
_nulljoin = "".join
def name(self, tag, data):
return _nulljoin(data[2:]).strip()
def tuplef(self, tag, data):
return tuple(data[2:])
class XYap(xyap):
def __init__(self):
self._parser = xml.parsers.expat.ParserCreate()
self._parser.StartElementHandler = self.unknown_starttag
self._parser.EndElementHandler = self.unknown_endtag
self._parser.CharacterDataHandler = self.handle_data
xyap.__init__(self)
class xmlrpc(NoBlanks, XYap):
end_handlers = {
'methodCall': tuplef,
'methodName': name,
'params': tuplef,
'param': lambda self, tag, data: data[2],
'value': lambda self, tag, data: data[2],
'i4':
lambda self, tag, data, atoi=string.atoi, name=name:
atoi(name(self, tag, data)),
'int':
lambda self, tag, data, atoi=string.atoi, name=name:
atoi(name(self, tag, data)),
'boolean':
lambda self, tag, data, atoi=string.atoi, name=name:
atoi(name(self, tag, data)),
'string': lambda self, tag, data, join=string.join:
join(data[2:], ''),
'double':
lambda self, tag, data, atof=string.atof, name=name:
atof(name(self, tag, data)),
'float':
lambda self, tag, data, atof=string.atof, name=name:
atof(name(self, tag, data)),
'struct': struct,
'member': tuplef,
'name': name,
'array': lambda self, tag, data: data[2],
'data': lambda self, tag, data: data[2:],
}
...@@ -40,8 +40,6 @@ if WITH_LEGACY_WORKFLOW: ...@@ -40,8 +40,6 @@ if WITH_LEGACY_WORKFLOW:
from Products.ERP5Type.patches import WorkflowTool from Products.ERP5Type.patches import WorkflowTool
from Products.ERP5Type.patches import WorkflowTool from Products.ERP5Type.patches import WorkflowTool
from Products.ERP5Type.patches import DynamicType from Products.ERP5Type.patches import DynamicType
from Products.ERP5Type.patches import XMLExportImport
from Products.ERP5Type.patches import ppml
from Products.ERP5Type.patches import Expression from Products.ERP5Type.patches import Expression
from Products.ERP5Type.patches import sqltest from Products.ERP5Type.patches import sqltest
from Products.ERP5Type.patches import sqlvar from Products.ERP5Type.patches import sqlvar
......
...@@ -36,7 +36,7 @@ except ImportError: ...@@ -36,7 +36,7 @@ except ImportError:
warnings.warn("Please install xmltodict, it is needed by json_representable mixin", warnings.warn("Please install xmltodict, it is needed by json_representable mixin",
DeprecationWarning) DeprecationWarning)
import zope.interface import zope.interface
from OFS import XMLExportImport from Products.ERP5Type import XMLExportImport
from StringIO import StringIO from StringIO import StringIO
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5Type.interfaces.json_representable import IJSONRepresentable from Products.ERP5Type.interfaces.json_representable import IJSONRepresentable
......
...@@ -12,10 +12,13 @@ ...@@ -12,10 +12,13 @@
# #
############################################################################## ##############################################################################
# Import: add rename feature and make _importObjectFromFile return the object from Products.ERP5Type.XMLExportImport import magic, importXML
from OFS.ObjectManager import ObjectManager, customImporters customImporters = {magic: importXML}
from App.version_txt import getZopeVersion
import OFS.ObjectManager
OFS.ObjectManager.customImporters = customImporters
# Import: add rename feature and make _importObjectFromFile return the object
def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id=None, suppress_events=False): def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id=None, suppress_events=False):
#LOG('_importObjectFromFile, filepath',0,filepath) #LOG('_importObjectFromFile, filepath',0,filepath)
# locate a valid connection # locate a valid connection
...@@ -41,4 +44,4 @@ def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id ...@@ -41,4 +44,4 @@ def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id
ob.manage_changeOwnershipType(explicit=0) ob.manage_changeOwnershipType(explicit=0)
return ob return ob
ObjectManager._importObjectFromFile=ObjectManager_importObjectFromFile OFS.ObjectManager.ObjectManager._importObjectFromFile=ObjectManager_importObjectFromFile
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
# Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Make sure the xml export will be ordered
from functools import partial
from inspect import getargspec
from ZODB.utils import u64, p64
from Shared.DC.xml import ppml
from base64 import encodestring
from cStringIO import StringIO
from ZODB.serialize import referencesf
from ZODB.ExportImport import TemporaryFile
from types import TupleType
from OFS import ObjectManager, XMLExportImport
from ..XMLExportImport import OrderedPickler
from logging import getLogger
log = getLogger(__name__)
def reorderPickle(jar, p):
try:
from ZODB._compat import Unpickler, Pickler
except ImportError: # BBB: ZODB 3.10
from ZODB.ExportImport import Unpickler, Pickler
from ZODB.ExportImport import Ghost, persistent_id
oids = {}
storage = jar._storage
new_oid = storage.new_oid
store = storage.store
def persistent_load(ooid,
Ghost=Ghost,
oids=oids, wrote_oid=oids.has_key,
new_oid=storage.new_oid):
"Remap a persistent id to an existing ID and create a ghost for it."
if type(ooid) is TupleType: ooid, klass = ooid
else: klass=None
try:
Ghost=Ghost()
Ghost.oid=ooid
except TypeError:
Ghost=Ghost(ooid)
return Ghost
# Reorder pickle by doing I/O
pfile = StringIO(p)
unpickler=Unpickler(pfile)
unpickler.persistent_load=persistent_load
newp=StringIO()
pickler=OrderedPickler(newp,1)
pickler.persistent_id=persistent_id
classdef = unpickler.load()
obj = unpickler.load()
pickler.dump(classdef)
pickler.dump(obj)
p=newp.getvalue()
return obj, p
def _mapOid(id_mapping, oid):
idprefix = str(u64(oid))
id = id_mapping[idprefix]
old_aka = encodestring(oid)[:-1]
aka=encodestring(p64(long(id)))[:-1] # Rebuild oid based on mapped id
id_mapping.setConvertedAka(old_aka, aka)
return idprefix+'.', id, aka
def XMLrecord(oid, plen, p, id_mapping):
# Proceed as usual
q=ppml.ToXMLUnpickler
f=StringIO(p)
u=q(f)
u.idprefix, id, aka = _mapOid(id_mapping, oid)
p=u.load(id_mapping=id_mapping).__str__(4)
if f.tell() < plen:
p=p+u.load(id_mapping=id_mapping).__str__(4)
String=' <record id="%s" aka="%s">\n%s </record>\n' % (id, aka, p)
return String
XMLExportImport.XMLrecord = XMLrecord
def exportXML(jar, oid, file=None):
# For performance reasons, exportXML does not use 'XMLrecord' anymore to map
# oids. This requires to initialize MinimalMapping.marked_reference before
# any string output, i.e. in ppml.Reference.__init__
# This also fixed random failures when DemoStorage is used, because oids
# can have values that have a shorter representation in 'repr' instead of
# 'base64' (see ppml.convert) and ppml.String does not support this.
load = jar._storage.load
if 'version' in getargspec(load).args: # BBB: ZODB<5 (TmpStore)
load = partial(load, version='')
pickle_dict = {oid: None}
max_cache = [1e7] # do not cache more than 10MB of pickle data
def getReorderedPickle(oid):
p = pickle_dict[oid]
if p is None:
p = load(oid)[0]
p = reorderPickle(jar, p)[1]
if len(p) < max_cache[0]:
max_cache[0] -= len(p)
pickle_dict[oid] = p
return p
# Sort records and initialize id_mapping
id_mapping = ppml.MinimalMapping()
reordered_oid_list = [oid]
for oid in reordered_oid_list:
_mapOid(id_mapping, oid)
for oid in referencesf(getReorderedPickle(oid)):
if oid not in pickle_dict:
pickle_dict[oid] = None
reordered_oid_list.append(oid)
# Do real export
if file is None:
file = TemporaryFile()
elif isinstance(file, basestring):
file = open(file, 'w+b')
write = file.write
write('<?xml version="1.0"?>\n<ZopeData>\n')
for oid in reordered_oid_list:
p = getReorderedPickle(oid)
write(XMLrecord(oid, len(p), p, id_mapping))
write('</ZopeData>\n')
return file
ObjectManager.exportXML = XMLExportImport.exportXML = exportXML
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment