zope4: XML Export/Import feature was removed from Zope4.

Only ZEXP Export/Import is possible. These modules (namely OFS.XMLExportImport
and Shared.DC.xml.*) were heavily monkey-patched anyway and are only used for
BusinessTemplates.

* ERP5Type/XMLExportImport.py  => ERP5Type/XMLExportImport/__init__.py
* OFS/XMLExportImport.py       => ERP5Type/XMLExportImport/__init__.py
* Shared/DC/xml/{xyap,ppml}.py => ERP5Type/XMLExportImport/{xyap,ppml}.py

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py

@@ -31,7 +31,7 @@ import pickle
 import re
 import xml.parsers.pyexpat
 from StringIO import StringIO
-from Shared.DC.xml import ppml
+from Products.ERP5Type.XMLExportImport import ppml
 
 
 class DummyClass:

product/ERP5/Document/BusinessTemplate.py

@@ -71,16 +71,13 @@ from Products.ERP5Type.dynamic.portal_type_class import synchronizeDynamicModule
 from Products.ERP5Type.Core.PropertySheet import PropertySheet as PropertySheetDocument
 from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
 from OFS.Traversable import NotFound
-from OFS import SimpleItem, XMLExportImport
+from OFS import SimpleItem
 from OFS.Image import Pdata
 from cStringIO import StringIO
 from copy import deepcopy
 from zExceptions import BadRequest
-import OFS.XMLExportImport
-from Products.ERP5Type.patches.ppml import importXML
-customImporters={
-    XMLExportImport.magic: importXML,
-    }
+from Products.ERP5Type.XMLExportImport import exportXML
+from OFS.ObjectManager import customImporters
 from Products.ERP5Type.Workflow import WorkflowHistoryList
 from zLOG import LOG, WARNING, INFO
 from warnings import warn
@@ -858,7 +855,7 @@ class ObjectTemplateItem(BaseTemplateItem):
           transaction.savepoint(optimistic=True)
 
         f = StringIO()
-        XMLExportImport.exportXML(obj._p_jar, obj._p_oid, f)
+        exportXML(obj._p_jar, obj._p_oid, f)
         bta.addObject(f, key, path=path)
         
       if catalog_method_template_item:
@@ -1015,8 +1012,8 @@ class ObjectTemplateItem(BaseTemplateItem):
       pass
 
     #LOG('Business Template', 0, 'Compiling %s...' % (name,))
-    from Shared.DC.xml import ppml
-    from OFS.XMLExportImport import start_zopedata, save_record, save_zopedata
+    from Products.ERP5Type.XMLExportImport import (ppml,
+      start_zopedata, save_record, save_zopedata)
     import xml.parsers.expat
     outfile=StringIO()
     try:
@@ -1069,10 +1066,10 @@ class ObjectTemplateItem(BaseTemplateItem):
       new_object = self._objects[path]
       new_io = StringIO()
       old_io = StringIO()
-      OFS.XMLExportImport.exportXML(new_object._p_jar, new_object._p_oid, new_io)
+      exportXML(new_object._p_jar, new_object._p_oid, new_io)
       new_obj_xml = new_io.getvalue()
       try:
-        OFS.XMLExportImport.exportXML(old_object._p_jar, old_object._p_oid, old_io)
+        exportXML(old_object._p_jar, old_object._p_oid, old_io)
         old_obj_xml = old_io.getvalue()
       except (ImportError, UnicodeDecodeError), e: # module is already
                                                    # removed etc.
@@ -6167,8 +6164,8 @@ Business Template is a set of definitions, such as skins, portal types and categ
         new_object = new_item.removeProperties(new_object, 1)
         installed_object = installed_item.removeProperties(installed_object, 1)
         # XML Export in memory
-        OFS.XMLExportImport.exportXML(new_object._p_jar, new_object._p_oid, f1)
-        OFS.XMLExportImport.exportXML(installed_object._p_jar,
+        exportXML(new_object._p_jar, new_object._p_oid, f1)
+        exportXML(installed_object._p_jar,
                                       installed_object._p_oid, f2)
         new_obj_xml = f1.getvalue()
         f1.close()
@@ -6503,6 +6500,8 @@ Business Template is a set of definitions, such as skins, portal types and categ
       'Products.ERP5Type.interfaces.json_representable',
       'Products.ERP5Type.mixin.json_representable',
       'Products.ERP5Type.XMLExportImport',
+      'Products.ERP5Type.XMLExportImport.ppml',
+      'Products.ERP5Type.XMLExportImport.xyap',
       'Products.ERP5Type.mixin.property_translatable',
       'Products.ERP5Type.Error',
       'Products.ERP5Type.Errors',

product/ERP5/tests/utils.py

@@ -32,7 +32,7 @@ import xml.dom.minidom
 from urllib import url2pathname
 from ZODB.DemoStorage import DemoStorage
 from ZODB import DB
-from OFS.XMLExportImport import importXML
+from Products.ERP5Type.XMLExportImport import importXML
 
 if int(os.environ.get('erp5_report_new_simulation_failures') or 0):
   newSimulationExpectedFailure = lambda test: test

product/ERP5Type/XMLExportImport/OFSXMLExportImport.py

+##############################################################################
+#
+# Copyright (c) 2001,2002 Zope Foundation and Contributors.
+# Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################

product/ERP5Type/XMLExportImport.py → product/ERP5Type/XMLExportImport/__init__.py

 # -*- coding: utf-8 -*-
 ##############################################################################
 #
-# Copyright (c) 2002-2003 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2001,2002 Zope Foundation and Contributors.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+# Copyright (c) 2002-2005 Nexedi SARL and Contributors. All Rights Reserved.
 #                    Sebastien Robin <seb@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
@@ -27,8 +36,8 @@
 #
 ##############################################################################
 
+## The code below was initially in ERP5Type/XMLExportImport.py
 from Acquisition import aq_base, aq_inner
-
 from collections import OrderedDict
 from cStringIO import StringIO
 from zodbpickle.pickle import Pickler
@@ -38,10 +47,9 @@ from lxml import etree
 from lxml.etree import Element, SubElement
 from xml_marshaller.xml_marshaller import Marshaller
 from OFS.Image import Pdata
-from zLOG import LOG
 from base64 import standard_b64encode
-
 from hashlib import sha1
+#from zLOG import LOG
 
 MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller'
 marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI,
@@ -207,3 +215,191 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None):
 
   return etree.tostring(root, encoding='utf-8',
                         xml_declaration=xml_declaration, pretty_print=True)
+
+## The code below was initially from OFS.XMLExportImport
+from base64 import encodestring
+from ZODB.serialize import referencesf
+from ZODB.ExportImport import TemporaryFile, export_end_marker
+from ZODB.utils import p64
+from ZODB.utils import u64
+from functools import partial
+from inspect import getargspec
+from types import TupleType
+from OFS import ObjectManager
+from . import ppml
+
+magic='<?xm' # importXML(jar, file, clue)}
+
+def reorderPickle(jar, p):
+    try:
+        from ZODB._compat import Unpickler, Pickler
+    except ImportError: # BBB: ZODB 3.10
+        from ZODB.ExportImport import Unpickler, Pickler
+    from ZODB.ExportImport import Ghost, persistent_id
+
+    oids = {}
+    storage = jar._storage
+    new_oid = storage.new_oid
+    store = storage.store
+
+    def persistent_load(ooid,
+                        Ghost=Ghost,
+                        oids=oids, wrote_oid=oids.has_key,
+                        new_oid=storage.new_oid):
+
+        "Remap a persistent id to an existing ID and create a ghost for it."
+
+        if type(ooid) is TupleType: ooid, klass = ooid
+        else: klass=None
+
+        try:
+          Ghost=Ghost()
+          Ghost.oid=ooid
+        except TypeError:
+          Ghost=Ghost(ooid)
+        return Ghost
+
+    # Reorder pickle by doing I/O
+    pfile = StringIO(p)
+    unpickler=Unpickler(pfile)
+    unpickler.persistent_load=persistent_load
+
+    newp=StringIO()
+    pickler=OrderedPickler(newp,1)
+    pickler.persistent_id=persistent_id
+
+    classdef = unpickler.load()
+    obj = unpickler.load()
+    pickler.dump(classdef)
+    pickler.dump(obj)
+    p=newp.getvalue()
+    return obj, p
+
+def _mapOid(id_mapping, oid):
+    idprefix = str(u64(oid))
+    id = id_mapping[idprefix]
+    old_aka = encodestring(oid)[:-1]
+    aka=encodestring(p64(long(id)))[:-1]  # Rebuild oid based on mapped id
+    id_mapping.setConvertedAka(old_aka, aka)
+    return idprefix+'.', id, aka
+
+def XMLrecord(oid, plen, p, id_mapping):
+    # Proceed as usual
+    q=ppml.ToXMLUnpickler
+    f=StringIO(p)
+    u=q(f)
+    u.idprefix, id, aka = _mapOid(id_mapping, oid)
+    p=u.load(id_mapping=id_mapping).__str__(4)
+    if f.tell() < plen:
+        p=p+u.load(id_mapping=id_mapping).__str__(4)
+    String='  <record id="%s" aka="%s">\n%s  </record>\n' % (id, aka, p)
+    return String
+
+def exportXML(jar, oid, file=None):
+    # For performance reasons, exportXML does not use 'XMLrecord' anymore to map
+    # oids. This requires to initialize MinimalMapping.marked_reference before
+    # any string output, i.e. in ppml.Reference.__init__
+    # This also fixed random failures when DemoStorage is used, because oids
+    # can have values that have a shorter representation in 'repr' instead of
+    # 'base64' (see ppml.convert) and ppml.String does not support this.
+    load = jar._storage.load
+    if 'version' in getargspec(load).args: # BBB: ZODB<5 (TmpStore)
+        load = partial(load, version='')
+    pickle_dict = {oid: None}
+    max_cache = [1e7] # do not cache more than 10MB of pickle data
+    def getReorderedPickle(oid):
+        p = pickle_dict[oid]
+        if p is None:
+            p = load(oid)[0]
+            p = reorderPickle(jar, p)[1]
+            if len(p) < max_cache[0]:
+                max_cache[0] -= len(p)
+                pickle_dict[oid] = p
+        return p
+
+    # Sort records and initialize id_mapping
+    id_mapping = ppml.MinimalMapping()
+    reordered_oid_list = [oid]
+    for oid in reordered_oid_list:
+        _mapOid(id_mapping, oid)
+        for oid in referencesf(getReorderedPickle(oid)):
+            if oid not in pickle_dict:
+                pickle_dict[oid] = None
+                reordered_oid_list.append(oid)
+
+    # Do real export
+    if file is None:
+        file = TemporaryFile()
+    elif isinstance(file, basestring):
+        file = open(file, 'w+b')
+    write = file.write
+    write('<?xml version="1.0"?>\n<ZopeData>\n')
+    for oid in reordered_oid_list:
+        p = getReorderedPickle(oid)
+        write(XMLrecord(oid, len(p), p, id_mapping))
+    write('</ZopeData>\n')
+    return file
+
+class zopedata:
+    def __init__(self, parser, tag, attrs):
+        self.file=parser.file
+        write=self.file.write
+        write('ZEXP')
+
+    def append(self, data):
+        file=self.file
+        write=file.write
+        pos=file.tell()
+        file.seek(pos)
+        write(data)
+
+def start_zopedata(parser, tag, data):
+    return zopedata(parser, tag, data)
+
+def save_zopedata(parser, tag, data):
+    file=parser.file
+    write=file.write
+    pos=file.tell()
+    file.seek(pos)
+    write(export_end_marker)
+
+def save_record(parser, tag, data):
+    file=parser.file
+    write=file.write
+    pos=file.tell()
+    file.seek(pos)
+    a=data[1]
+    if a.has_key('id'): oid=a['id']
+    oid=p64(int(oid))
+    v=''
+    for x in data[2:]:
+        v=v+x
+    l=p64(len(v))
+    v=oid+l+v
+    return v
+
+import xml.parsers.expat
+def importXML(jar, file, clue=''):
+    if type(file) is str:
+        file=open(file, 'rb')
+    outfile=TemporaryFile()
+    data=file.read()
+    F=ppml.xmlPickler()
+    F.end_handlers['record'] = save_record
+    F.end_handlers['ZopeData'] = save_zopedata
+    F.start_handlers['ZopeData'] = start_zopedata
+    F.binary=1
+    F.file=outfile
+    # <patch>
+    # Our BTs XML files don't declare encoding but have accented chars in them
+    # So we have to declare an encoding but not use unicode, so the unpickler
+    # can deal with the utf-8 strings directly
+    p=xml.parsers.expat.ParserCreate('utf-8')
+    p.returns_unicode = False
+    # </patch>
+    p.CharacterDataHandler=F.handle_data
+    p.StartElementHandler=F.unknown_starttag
+    p.EndElementHandler=F.unknown_endtag
+    r=p.Parse(data)
+    outfile.seek(0)
+    return jar.importFile(outfile,clue)

product/ERP5Type/patches/ppml.py → product/ERP5Type/XMLExportImport/ppml.py

-# -*- coding: utf-8 -*-
 ##############################################################################
 #
-# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
+# Copyright (c) 2001,2002 Zope Corporation and Contributors. All Rights Reserved.
 # Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
+"""Provide conversion between Python pickles and XML
+"""
 
+from pickle import *
+import struct
+import base64
 import re
+from marshal import loads as mloads
+from .xyap import NoBlanks
+from .xyap import xyap
 
-# Import everything right now, not after
-# or new patch will not work
-from Shared.DC.xml.ppml import *
-from Shared.DC.xml import ppml
-
+import re
 from marshal import dumps as mdumps
-from zLOG import LOG
+#from zLOG import LOG
+
+binary = re.compile('[^\x1f-\x7f]').search
+
+def escape(s, encoding='repr'):
+    if binary(s) and isinstance(s, str):
+        s = base64.encodestring(s)[:-1]
+        encoding = 'base64'
+    elif '>' in s or '<' in s or '&' in s:
+        if not ']]>' in s:
+            s = '<![CDATA[' + s + ']]>'
+            encoding = 'cdata'
+        else:
+            s = s.replace('&', '&amp;')
+            s = s.replace('>', '&gt;')
+            s = s.replace('<', '&lt;')
+    return encoding, s
+
+def unescape(s, encoding):
+    if encoding == 'base64':
+        return base64.decodestring(s)
+    else:
+        s = s.replace('&lt;', '<')
+        s = s.replace('&gt;', '>')
+        return s.replace('&amp;', '&')
 
 # For converting to a more readable expression.
 reprs = {}
@@ -38,15 +65,12 @@ reprs2={}
 reprs2['<'] = "\\074"
 reprs2['>'] = "\\076"
 reprs2['&'] = "\\046"
-
 reprs_re = re.compile('|'.join(re.escape(k) for k in reprs.keys()))
 def sub_reprs(m):
   return reprs[m.group(0)]
-
 reprs2_re = re.compile('|'.join(re.escape(k) for k in reprs2.keys()))
 def sub_reprs2(m):
   return reprs2[m.group(0)]
-
 def convert(S):
     new = ''
     ### patch begin: if the input string is a valid utf8 string, only
@@ -69,8 +93,6 @@ def convert(S):
             return 'repr', reprs2_re.sub(sub_reprs2, new)
     return 'repr', new
 
-ppml.convert = convert
-
 # For optimization.
 def unconvert(encoding,S):
     if encoding == 'base64':
@@ -78,10 +100,7 @@ def unconvert(encoding,S):
     else:
         return eval("'" + S.replace('\n', '') + "'")
 
-ppml.unconvert = unconvert
-
 class Global:
-
     def __init__(self, module, name, mapping):
         self.module=module
         self.name=name
@@ -95,10 +114,14 @@ class Global:
         return '%s<%s%s name="%s" module="%s"/>\n' % (
             ' '*indent, name, id, self.name, self.module)
 
-ppml.Global = Global
+class Immutable:
+    def __init__(self, value):
+        self.value = value
 
-class Scalar:
+    def getValue(self):
+        return self.value
 
+class Scalar:
     def __init__(self, v, mapping):
         self._v=v
         self.mapping = mapping
@@ -116,17 +139,14 @@ class Scalar:
             self.mapping.setImmutable(self.id, Immutable(value = result))
         return result
 
-ppml.Scalar = Scalar
-
-class Immutable:
-    def __init__(self, value):
-        self.value = value
-
-    def getValue(self):
-        return self.value
+class Long(Scalar):
+    def value(self):
+        result = str(self._v)
+        if result[-1:] == 'L':
+            return result[:-1]
+        return result
 
 class String(Scalar):
-
     encoding = None
 
     def __init__(self, v, mapping, encoding=''):
@@ -160,16 +180,11 @@ class String(Scalar):
             self.mapping.setImmutable(self.id, Immutable(value = result))
         return '%s%s\n' % (' '*indent, result)
 
-ppml.String = String
-
 class Unicode(String):
     def value(self):
         return self._v.encode('utf-8')
 
-ppml.Unicode = Unicode
-
 class Wrapper:
-
     def __init__(self, v, mapping):
         self._v=v
         self.mapping = mapping
@@ -189,10 +204,7 @@ class Wrapper:
             v=v.__str__(indent+2)
             return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
 
-ppml.Wrapper = Wrapper
-
 class Collection:
-
     def __init__(self, mapping):
         self.mapping = mapping
 
@@ -208,14 +220,15 @@ class Collection:
         else:
             return '%s<%s%s/>\n' % (i, name, id)
 
-ppml.Collection = Collection
-
 class Dictionary(Collection):
     def __init__(self, mapping):
         self.mapping = mapping
         self._d=[]
+
     def __len__(self): return len(self._d)
+
     def __setitem__(self, k, v): self._d.append((k,v))
+
     def value(self, indent):
         #self._d.sort(lambda a, b: cmp(a[0]._v, b[0]._v)) # Sort the sequence by key JPS Improvement
         ind = ' ' * indent
@@ -230,10 +243,7 @@ class Dictionary(Collection):
                  for i in self._d
             )
 
-ppml.Dictionary = Dictionary
-
 class Sequence(Collection):
-
     def __init__(self, mapping, v=None):
         if not v: v=[]
         self._subs=v
@@ -251,26 +261,10 @@ class Sequence(Collection):
             v.__str__(indent) for v in
             self._subs)
 
-ppml.Sequence = Sequence
-
-class Persistent(Wrapper):
-
+class none:
     def __str__(self, indent=0):
-        id = ''
-        if hasattr(self, 'id'):
-            if self.mapping.isMarked(self.id): id=' id="%s"' % self.mapping[self.id]
-        name=self.__class__.__name__.lower()
-        v=self._v
-        i=' '*indent
-        if isinstance(v,String):
-            return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name)
-        elif isinstance(v,Scalar):
-            return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name)
-        else:
-            v=v.__str__(indent+2)
-            return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
-
-ppml.Persistent = Persistent
+        return ' ' * indent + '<none/>\n'
+none = none()
 
 class Reference(Scalar):
     def __init__(self, v, mapping):
@@ -288,9 +282,7 @@ class Reference(Scalar):
           value = '<%s id="%s"/>' % (name, self.mapping[v])
         return '%s%s\n' % (' '*indent, value)
 
-ppml.Reference = Reference
 Get = Reference
-ppml.Get = Get
 
 class Object(Sequence):
     def __init__(self, klass, args, mapping):
@@ -299,10 +291,33 @@ class Object(Sequence):
 
     def __setstate__(self, v): self.append(State(v, self.mapping))
 
-ppml.Object = Object
+class Int(Scalar): pass
+class Float(Scalar): pass
+class List(Sequence): pass
+class Tuple(Sequence): pass
+class Key(Wrapper): pass
+class Value(Wrapper): pass
+class Klass(Wrapper): pass
+class State(Wrapper): pass
+class Pickle(Wrapper): pass
+
+class Persistent(Wrapper):
+    def __str__(self, indent=0):
+        id = ''
+        if hasattr(self, 'id'):
+            if self.mapping.isMarked(self.id): id=' id="%s"' % self.mapping[self.id]
+        name=self.__class__.__name__.lower()
+        v=self._v
+        i=' '*indent
+        if isinstance(v,String):
+            return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name)
+        elif isinstance(v,Scalar):
+            return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name)
+        else:
+            v=v.__str__(indent+2)
+            return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
 
 blanck_line_expression = re.compile('^ +$')
-
 class NoBlanks:
     """
     This allows to ignore at least whitespaces between elements and also
@@ -350,10 +365,7 @@ class NoBlanks:
 
                 self.append(data)
 
-ppml.NoBlanks = NoBlanks
-
 class IdentityMapping:
-
     def __init__(self):
       self.resetMapping()
       self.immutable = {}
@@ -385,11 +397,7 @@ class IdentityMapping:
     def hasImmutable(self, k):
       return self.immutable.has_key(k)
 
-
-ppml.IdentityMapping = IdentityMapping
-
 class MinimalMapping(IdentityMapping):
-
     def resetMapping(self):
       self.mapped_id = {}
       self.mapped_core_id = {}
@@ -443,30 +451,7 @@ class MinimalMapping(IdentityMapping):
     def __str__(self, a):
       return "Error here"
 
-ppml.MinimalMapping = MinimalMapping
-
-class List(Sequence): pass
-class Tuple(Sequence): pass
-
-class Klass(Wrapper): pass
-class State(Wrapper): pass
-class Pickle(Wrapper): pass
-
-class Int(Scalar): pass
-class Float(Scalar): pass
-
-class Key(Wrapper): pass
-class Value(Wrapper): pass
-
-class Long(Scalar):
-    def value(self):
-        result = str(self._v)
-        if result[-1:] == 'L':
-            return result[:-1]
-        return result
-
 class ToXMLUnpickler(Unpickler):
-
     def load(self, id_mapping=None):
       if id_mapping is None:
         self.id_mapping = IdentityMapping()
@@ -657,7 +642,58 @@ class ToXMLUnpickler(Unpickler):
     #for code in dispatch.keys():
     #  dispatch[code] = LogCall(dispatch[code])
 
-ppml.ToXMLUnpickler = ToXMLUnpickler
+def ToXMLload(file):
+    return ToXMLUnpickler(file).load()
+
+def ToXMLloads(str):
+    from StringIO import StringIO
+    file = StringIO(str)
+    return ToXMLUnpickler(file).load()
+
+def name(self, tag, data):
+    return ''.join(data[2:]).strip()
+
+def start_pickle(self, tag, attrs):
+    self._pickleids = {}
+    return [tag, attrs]
+
+def save_int(self, tag, data):
+    if self.binary:
+        v = int(name(self, tag, data))
+        if v >= 0:
+            if v <= 0xff:
+                return BININT1 + chr(v)
+            if v <= 0xffff:
+                return '%c%c%c' % (BININT2, v & 0xff, v >> 8)
+        hb = v >> 31
+        if hb == 0 or hb == -1:
+            return BININT + struct.pack('<i', v)
+    return INT + name(self, tag, data) + '\n'
+
+def save_float(self, tag, data):
+    if self.binary:
+        return BINFLOAT + struct.pack('>d', float(name(self, tag, data)))
+    else:
+        return FLOAT + name(self, tag, data) + '\n'
+
+def save_put(self, v, attrs):
+    id = attrs.get('id', '')
+    if id:
+        prefix = id.rfind('.')
+        if prefix >= 0:
+            id = id[prefix + 1:]
+        elif id[0] == 'i':
+            id = id[1:]
+        if self.binary:
+            id = int(id)
+            if id < 256:
+                id = BINPUT + chr(id)
+            else:
+                id = LONG_BINPUT + struct.pack('<i', id)
+        else:
+            id = PUT + repr(id) + '\n'
+        return v + id
+    return v
 
 def save_string(self, tag, data):
     binary=self.binary
@@ -680,8 +716,6 @@ def save_string(self, tag, data):
     else: v="S'"+v+"'\012"
     return save_put(self, v, a)
 
-ppml.save_string = save_string
-
 def save_unicode(self, tag, data):
     binary=self.binary
     v=''
@@ -699,7 +733,50 @@ def save_unicode(self, tag, data):
     else: v=UNICODE+"'"+v+"'\012"
     return save_put(self, v, a)
 
-ppml.save_unicode = save_unicode
+def save_tuple(self, tag, data):
+    T = data[2:]
+    if not T:
+        return EMPTY_TUPLE
+    return save_put(self, MARK + ''.join(T) + TUPLE, data[1])
+
+def save_list(self, tag, data):
+    L = data[2:]
+    if self.binary:
+        v = save_put(self, EMPTY_LIST, data[1])
+        if L:
+            v = v + MARK + ''.join(L) + APPENDS
+    else:
+        v = save_put(self, MARK + LIST, data[1])
+        if L:
+            v = APPEND.join(L) + APPEND
+    return v
+
+def save_dict(self, tag, data):
+    D = data[2:]
+    if self.binary:
+        v = save_put(self, EMPTY_DICT, data[1])
+        if D:
+            v = v + MARK + ''.join(D) + SETITEMS
+    else:
+        v = save_put(self, MARK + DICT, data[1])
+        if D:
+            v = v + SETITEM.join(D) + SETITEM
+    return v
+
+def save_reference(self, tag, data):
+    a = data[1]
+    id = a['id']
+    prefix = id.rfind('.')
+    if prefix >= 0:
+        id = id[prefix + 1:]
+    if self.binary:
+        id = int(id)
+        if id < 256:
+            return BINGET + chr(id)
+        else:
+            return LONG_BINGET + struct.pack('<i', i)
+    else:
+        return GET + repr(id) + '\n'
 
 def save_object(self, tag, data):
     if len(data)==5:
@@ -720,7 +797,16 @@ def save_object(self, tag, data):
         v=v+'R'
         return v
 
-ppml.save_object = save_object
+def save_global(self, tag, data):
+    a = data[1]
+    return save_put(self, GLOBAL + a['module'] + '\n' + a['name'] + '\n', a)
+
+def save_persis(self, tag, data):
+    v = data[2]
+    if self.binary:
+        return v + BINPERSID
+    else:
+        return PERSID + v
 
 def save_pickle_start(self, tag, attrs):
     return [tag, attrs]
@@ -778,41 +864,3 @@ class xmlPickler(NoBlanks, xyap):
         'persistent': save_persis,
         }
 
-# FIXME: Leo: Do we still need to replace ppml.xmlPickler now that we're
-# using our own xmlPickler in our own importXML function below? Do we support
-# any other use of xmlPickler except for Business Template export/import?
-ppml.xmlPickler = xmlPickler
-
-class Tuple(Sequence): pass
-
-ppml.Tuple = Tuple
-
-# Copied from OFS.XMLExportImport.importXML (of Zope 2.12)
-# Imported and used directly by Products.ERP5.Document.BusinessTemplate
-from OFS.XMLExportImport import save_record, save_zopedata, start_zopedata
-from tempfile import TemporaryFile
-import xml.parsers.expat
-def importXML(jar, file, clue=''):
-    if type(file) is str:
-        file=open(file, 'rb')
-    outfile=TemporaryFile()
-    data=file.read()
-    F=xmlPickler()
-    F.end_handlers['record'] = save_record
-    F.end_handlers['ZopeData'] = save_zopedata
-    F.start_handlers['ZopeData'] = start_zopedata
-    F.binary=1
-    F.file=outfile
-    # <patch>
-    # Our BTs XML files don't declare encoding but have accented chars in them
-    # So we have to declare an encoding but not use unicode, so the unpickler
-    # can deal with the utf-8 strings directly
-    p=xml.parsers.expat.ParserCreate('utf-8')
-    p.returns_unicode = False
-    # </patch>
-    p.CharacterDataHandler=F.handle_data
-    p.StartElementHandler=F.unknown_starttag
-    p.EndElementHandler=F.unknown_endtag
-    r=p.Parse(data)
-    outfile.seek(0)
-    return jar.importFile(outfile,clue)

product/ERP5Type/XMLExportImport/xyap.py

+"""Yet another XML parser
+
+This is meant to be very simple:
+
+  - stack based
+
+  - The parser has a table of start handlers and end handlers.
+
+  - start tag handlers are called with the parser instance, tag names
+    and attributes.  The result is placed on the stack.  The default
+    handler places a special object on the stack (uh, a list, with the
+    tag name and attributes as the first two elements. ;)
+
+  - end tag handlers are called with the object on the parser, the tag
+    name, and top of the stack right after it has been removed.  The
+    result is appended to the object on the top of the stack.
+
+Note that namespace attributes should recieve some special handling.
+Oh well.
+"""
+
+import string
+import xml.parsers.expat
+
+
+class xyap:
+    start_handlers = {}
+    end_handlers = {}
+
+    def __init__(self):
+        top = []
+        self._stack = _stack = [top]
+        self.push = _stack.append
+        self.append = top.append
+
+    def handle_data(self, data):
+        self.append(data)
+
+    def unknown_starttag(self, tag, attrs):
+        if isinstance(attrs, list):
+            attrs = dict(attrs)
+        start = self.start_handlers
+        if tag in start:
+            tag = start[tag](self, tag, attrs)
+        else:
+            tag = [tag, attrs]
+        self.push(tag)
+        self.append = tag.append
+
+    def unknown_endtag(self, tag):
+        _stack = self._stack
+        top = _stack.pop()
+        append = self.append = _stack[-1].append
+        end = self.end_handlers
+        if tag in end:
+            top = end[tag](self, tag, top)
+        append(top)
+
+class NoBlanks:
+
+    def handle_data(self, data):
+        if data.strip():
+            self.append(data)
+
+
+def struct(self, tag, data):
+    r = {}
+    for k, v in data[2:]:
+        r[k] = v
+    return r
+
+_nulljoin = "".join
+
+def name(self, tag, data):
+    return _nulljoin(data[2:]).strip()
+
+def tuplef(self, tag, data):
+    return tuple(data[2:])
+
+class XYap(xyap):
+    def __init__(self):
+        self._parser = xml.parsers.expat.ParserCreate()
+        self._parser.StartElementHandler = self.unknown_starttag
+        self._parser.EndElementHandler = self.unknown_endtag
+        self._parser.CharacterDataHandler = self.handle_data
+        xyap.__init__(self)
+
+class xmlrpc(NoBlanks, XYap):
+    end_handlers = {
+        'methodCall': tuplef,
+        'methodName': name,
+        'params': tuplef,
+        'param': lambda self, tag, data: data[2],
+        'value': lambda self, tag, data: data[2],
+        'i4':
+        lambda self, tag, data, atoi=string.atoi, name=name:
+        atoi(name(self, tag, data)),
+        'int':
+        lambda self, tag, data, atoi=string.atoi, name=name:
+            atoi(name(self, tag, data)),
+        'boolean':
+        lambda self, tag, data, atoi=string.atoi, name=name:
+            atoi(name(self, tag, data)),
+        'string': lambda self, tag, data, join=string.join:
+            join(data[2:], ''),
+        'double':
+        lambda self, tag, data, atof=string.atof, name=name:
+            atof(name(self, tag, data)),
+        'float':
+        lambda self, tag, data, atof=string.atof, name=name:
+            atof(name(self, tag, data)),
+        'struct': struct,
+        'member': tuplef,
+        'name': name,
+        'array': lambda self, tag, data: data[2],
+        'data': lambda self, tag, data: data[2:],
+        }

product/ERP5Type/ZopePatch.py

@@ -40,8 +40,6 @@ if WITH_LEGACY_WORKFLOW:
   from Products.ERP5Type.patches import WorkflowTool
 from Products.ERP5Type.patches import WorkflowTool
 from Products.ERP5Type.patches import DynamicType
-from Products.ERP5Type.patches import XMLExportImport
-from Products.ERP5Type.patches import ppml
 from Products.ERP5Type.patches import Expression
 from Products.ERP5Type.patches import sqltest
 from Products.ERP5Type.patches import sqlvar

product/ERP5Type/mixin/json_representable.py

@@ -36,7 +36,7 @@ except ImportError:
   warnings.warn("Please install xmltodict, it is needed by json_representable mixin",
                  DeprecationWarning)
 import zope.interface
-from OFS import XMLExportImport
+from Products.ERP5Type import XMLExportImport
 from StringIO import StringIO
 from AccessControl import ClassSecurityInfo
 from Products.ERP5Type.interfaces.json_representable import IJSONRepresentable

product/ERP5Type/patches/ObjectManager.py

@@ -12,10 +12,13 @@
 #
 ##############################################################################
 
-# Import: add rename feature and make _importObjectFromFile return the object
-from OFS.ObjectManager import ObjectManager, customImporters
-from App.version_txt import getZopeVersion
+from Products.ERP5Type.XMLExportImport import magic, importXML
+customImporters = {magic: importXML}
+
+import OFS.ObjectManager
+OFS.ObjectManager.customImporters = customImporters
 
+# Import: add rename feature and make _importObjectFromFile return the object
 def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id=None, suppress_events=False):
     #LOG('_importObjectFromFile, filepath',0,filepath)
     # locate a valid connection
@@ -41,4 +44,4 @@ def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id
       ob.manage_changeOwnershipType(explicit=0)
     return ob
 
-ObjectManager._importObjectFromFile=ObjectManager_importObjectFromFile
+OFS.ObjectManager.ObjectManager._importObjectFromFile=ObjectManager_importObjectFromFile

product/ERP5Type/patches/XMLExportImport.py

-##############################################################################
-#
-# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
-# Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Make sure the xml export will be ordered
-
-from functools import partial
-from inspect import getargspec
-from ZODB.utils import u64, p64
-from Shared.DC.xml import ppml
-from base64 import encodestring
-from cStringIO import StringIO
-from ZODB.serialize import referencesf
-from ZODB.ExportImport import TemporaryFile
-from types import TupleType
-from OFS import ObjectManager, XMLExportImport
-from ..XMLExportImport import OrderedPickler
-
-from logging import getLogger
-log = getLogger(__name__)
-
-def reorderPickle(jar, p):
-    try:
-        from ZODB._compat import Unpickler, Pickler
-    except ImportError: # BBB: ZODB 3.10
-        from ZODB.ExportImport import Unpickler, Pickler
-    from ZODB.ExportImport import Ghost, persistent_id
-
-    oids = {}
-    storage = jar._storage
-    new_oid = storage.new_oid
-    store = storage.store
-
-    def persistent_load(ooid,
-                        Ghost=Ghost,
-                        oids=oids, wrote_oid=oids.has_key,
-                        new_oid=storage.new_oid):
-
-        "Remap a persistent id to an existing ID and create a ghost for it."
-
-        if type(ooid) is TupleType: ooid, klass = ooid
-        else: klass=None
-
-        try:
-          Ghost=Ghost()
-          Ghost.oid=ooid
-        except TypeError:
-          Ghost=Ghost(ooid)
-        return Ghost
-
-    # Reorder pickle by doing I/O
-    pfile = StringIO(p)
-    unpickler=Unpickler(pfile)
-    unpickler.persistent_load=persistent_load
-
-    newp=StringIO()
-    pickler=OrderedPickler(newp,1)
-    pickler.persistent_id=persistent_id
-
-    classdef = unpickler.load()
-    obj = unpickler.load()
-    pickler.dump(classdef)
-    pickler.dump(obj)
-    p=newp.getvalue()
-    return obj, p
-
-def _mapOid(id_mapping, oid):
-    idprefix = str(u64(oid))
-    id = id_mapping[idprefix]
-    old_aka = encodestring(oid)[:-1]
-    aka=encodestring(p64(long(id)))[:-1]  # Rebuild oid based on mapped id
-    id_mapping.setConvertedAka(old_aka, aka)
-    return idprefix+'.', id, aka
-
-def XMLrecord(oid, plen, p, id_mapping):
-    # Proceed as usual
-    q=ppml.ToXMLUnpickler
-    f=StringIO(p)
-    u=q(f)
-    u.idprefix, id, aka = _mapOid(id_mapping, oid)
-    p=u.load(id_mapping=id_mapping).__str__(4)
-    if f.tell() < plen:
-        p=p+u.load(id_mapping=id_mapping).__str__(4)
-    String='  <record id="%s" aka="%s">\n%s  </record>\n' % (id, aka, p)
-    return String
-
-XMLExportImport.XMLrecord = XMLrecord
-
-def exportXML(jar, oid, file=None):
-    # For performance reasons, exportXML does not use 'XMLrecord' anymore to map
-    # oids. This requires to initialize MinimalMapping.marked_reference before
-    # any string output, i.e. in ppml.Reference.__init__
-    # This also fixed random failures when DemoStorage is used, because oids
-    # can have values that have a shorter representation in 'repr' instead of
-    # 'base64' (see ppml.convert) and ppml.String does not support this.
-    load = jar._storage.load
-    if 'version' in getargspec(load).args: # BBB: ZODB<5 (TmpStore)
-        load = partial(load, version='')
-    pickle_dict = {oid: None}
-    max_cache = [1e7] # do not cache more than 10MB of pickle data
-    def getReorderedPickle(oid):
-        p = pickle_dict[oid]
-        if p is None:
-            p = load(oid)[0]
-            p = reorderPickle(jar, p)[1]
-            if len(p) < max_cache[0]:
-                max_cache[0] -= len(p)
-                pickle_dict[oid] = p
-        return p
-
-    # Sort records and initialize id_mapping
-    id_mapping = ppml.MinimalMapping()
-    reordered_oid_list = [oid]
-    for oid in reordered_oid_list:
-        _mapOid(id_mapping, oid)
-        for oid in referencesf(getReorderedPickle(oid)):
-            if oid not in pickle_dict:
-                pickle_dict[oid] = None
-                reordered_oid_list.append(oid)
-
-    # Do real export
-    if file is None:
-        file = TemporaryFile()
-    elif isinstance(file, basestring):
-        file = open(file, 'w+b')
-    write = file.write
-    write('<?xml version="1.0"?>\n<ZopeData>\n')
-    for oid in reordered_oid_list:
-        p = getReorderedPickle(oid)
-        write(XMLrecord(oid, len(p), p, id_mapping))
-    write('</ZopeData>\n')
-    return file
-
-ObjectManager.exportXML = XMLExportImport.exportXML = exportXML