py2/py3: data property is for binary, thus getData() should return bytes.

bt5/erp5_base/DocumentTemplateItem/portal_components/document.erp5.Image.py

@@ -32,8 +32,7 @@
 
 import os
 import subprocess
-from six.moves import cStringIO as StringIO
-
+from io import BytesIO
 from AccessControl import ClassSecurityInfo
 from Acquisition import aq_base
 
@@ -117,7 +116,7 @@ class Image(TextConvertableMixin, File, OFSImage):
     content_type, width, height = getImageInfo(self.data)
     if not content_type:
       try:
-        image = PIL.Image.open(StringIO(str(self.data)))
+        image = PIL.Image.open(BytesIO(bytes(self.data)))
       except IOError:
         width = height = -1
         content_type = 'application/unknown'
@@ -381,7 +380,7 @@ class Image(TextConvertableMixin, File, OFSImage):
     else:
       parameter_list.append('-')
 
-    data = str(self.getData())
+    data = bytes(self.getData())
     if self.getContentType() == "image/svg+xml":
       data = transformUrlToDataURI(data)
 
@@ -401,7 +400,7 @@ class Image(TextConvertableMixin, File, OFSImage):
     finally:
       del process
     if image:
-      return StringIO(image)
+      return BytesIO(image)
     raise ConversionError('Image conversion failed (%s).' % err)
 
   def _getContentTypeAndImageData(

bt5/erp5_base/DocumentTemplateItem/portal_components/document.erp5.OOoDocument.py

@@ -28,7 +28,7 @@
 ##############################################################################
 
 import re, zipfile
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 from warnings import warn
 from AccessControl import ClassSecurityInfo
 from OFS.Image import Pdata
@@ -39,7 +39,7 @@ from Products.ERP5Type.Cache import CachingMethod
 from erp5.component.document.File import File
 from erp5.component.document.Document import Document, \
        VALID_IMAGE_FORMAT_LIST, ConversionError, NotConvertedError
-from Products.ERP5Type.Utils import fill_args_from_request
+from Products.ERP5Type.Utils import bytes2str, fill_args_from_request, str2bytes
 
 # Mixin Import
 from erp5.component.mixin.BaseConvertableFileMixin import BaseConvertableFileMixin
@@ -199,10 +199,10 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       raise NotConvertedError()
     if format == 'text-content':
       # Extract text from the ODF file
-      cs = cStringIO.StringIO()
-      cs.write(str(self.getBaseData()))
+      cs = BytesIO()
+      cs.write(self.getBaseData())
       z = zipfile.ZipFile(cs)
-      s = z.read('content.xml')
+      s = bytes2str(z.read('content.xml'))
       s = self.rx_strip.sub(" ", s) # strip xml
       s = self.rx_compr.sub(" ", s) # compress multiple spaces
       cs.close()
@@ -211,7 +211,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
     server_proxy = DocumentConversionServerProxy(self)
     orig_format = self.getBaseContentType()
     generate_result = server_proxy.run_generate(self.getId(),
-                                       enc(str(self.getBaseData())),
+                                       bytes2str(enc(bytes(self.getBaseData()))),
                                        None,
                                        format,
                                        orig_format)
@@ -223,7 +223,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       response_dict = generate_result
 
     # XXX: handle possible OOOd server failure
-    return response_dict['mime'], Pdata(dec(response_dict['data']))
+    return response_dict['mime'], Pdata(dec(str2bytes(response_dict['data'])))
 
   # Conversion API
   def _convert(self, format, frame=0, **kw): #  pylint: disable=redefined-builtin
@@ -259,7 +259,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
     original_format = format
     allowed_format_list = self.getTargetFormatList()
     if format == 'base-data':
-      return self.getBaseContentType(), str(self.getBaseData())
+      return self.getBaseContentType(), self.getBaseData()
     if format == 'pdf':
       format_list = [x for x in allowed_format_list
                                           if x.endswith('pdf')]
@@ -302,8 +302,8 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       if is_html:
         # Extra processing required since
         # we receive a zip file
-        cs = cStringIO.StringIO()
-        cs.write(str(data))
+        cs = BytesIO()
+        cs.write(data)
         z = zipfile.ZipFile(cs) # A disk file would be more RAM efficient
         for f in z.infolist():
           fn = f.filename
@@ -325,7 +325,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
         # create temporary image and use it to resize accordingly
         temp_image = self.portal_contributions.newContent(
                                        portal_type='Image',
-                                       file=cStringIO.StringIO(),
+                                       file=BytesIO(),
                                        filename=self.getId(),
                                        temp_object=1)
         temp_image._setData(data)
@@ -347,8 +347,8 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       format_list = [x for x in self.getTargetFormatList()
                                 if x.startswith('html') or x.endswith('html')]
       mime, data = self._getConversionFromProxyServer(format_list[0])
-      archive_file = cStringIO.StringIO()
-      archive_file.write(str(data))
+      archive_file = BytesIO()
+      archive_file.write(data)
       zip_file = zipfile.ZipFile(archive_file)
       must_close = 1
     else:
@@ -385,13 +385,13 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
     server_proxy = DocumentConversionServerProxy(self)
     response_code, response_dict, response_message = server_proxy.run_convert(
                                       self.getFilename() or self.getId(),
-                                      enc(str(self.getData())),
+                                      bytes2str(enc(bytes(self.getData()))),
                                       None,
                                       None,
                                       self.getContentType())
     if response_code == 200:
       # sucessfully converted document
-      self._setBaseData(dec(response_dict['data']))
+      self._setBaseData(dec(str2bytes(response_dict['data'])))
       metadata = response_dict['meta']
       self._base_metadata = metadata
       if metadata.get('MIMEType', None) is not None:
@@ -424,11 +424,11 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
     server_proxy = DocumentConversionServerProxy(self)
     response_code, response_dict, response_message = \
           server_proxy.run_setmetadata(self.getId(),
-                                       enc(str(self.getBaseData())),
+                                       bytes2str(enc(bytes(self.getBaseData()))),
                                        kw)
     if response_code == 200:
       # successful meta data extraction
-      self._setBaseData(dec(response_dict['data']))
+      self._setBaseData(dec(str2bytes(response_dict['data'])))
       self.updateFileMetadata() # record in workflow history # XXX must put appropriate comments.
     else:
       # Explicitly raise the exception!

bt5/erp5_corporate_identity/SkinTemplateItem/portal_skins/erp5_corporate_identity/Presentation_viewAsSlideshow.py

@@ -29,7 +29,7 @@ if context.getPortalType() in ["Presentation"]:
   portal = context.getPortalObject()
   mimetype = 'text/html'
   content_type = context.getContentType()
-  raw_data = portal.portal_transforms.convertToData(mimetype, str(context.getData() or ""), context=context, mimetype=content_type)
+  raw_data = portal.portal_transforms.convertToData(mimetype, bytes(context.getData() or b""), context=context, mimetype=content_type)
   if raw_data is None:
     raise ValueError("Failed to convert to %r" % mimetype)
   if context.REQUEST is not None:

bt5/erp5_dms/DocumentTemplateItem/portal_components/document.erp5.PDFDocument.py

@@ -32,13 +32,16 @@ import zope.interface
 from AccessControl import ClassSecurityInfo
 
 from Products.ERP5Type import Permissions, PropertySheet
+from Products.ERP5Type.Utils import bytes2str
 from erp5.component.interface.IWatermarkable import IWatermarkable
 from erp5.component.document.Image import Image
 from erp5.component.document.Document import ConversionError
 from subprocess import Popen, PIPE
 from zLOG import LOG, INFO, PROBLEM
 import errno
-from StringIO import StringIO
+from io import BytesIO
+from six.moves import range
+import six
 
 @zope.interface.implementer(IWatermarkable)
 class PDFDocument(Image):
@@ -90,8 +93,8 @@ class PDFDocument(Image):
         raise ValueError("watermark_data cannot not be empty")
       if not self.hasData():
         raise ValueError("Cannot watermark an empty document")
-      self_reader = PdfFileReader(StringIO(self.getData()))
-      watermark_reader = PdfFileReader(StringIO(watermark_data))
+      self_reader = PdfFileReader(BytesIO(self.getData()))
+      watermark_reader = PdfFileReader(BytesIO(watermark_data))
       watermark_page_count = watermark_reader.getNumPages()
 
       output = PdfFileWriter()
@@ -109,7 +112,7 @@ class PDFDocument(Image):
             self_page.mergePage(watermark_page)
         output.addPage(self_page)
 
-      outputStream = StringIO()
+      outputStream = BytesIO()
       output.write(outputStream)
       return outputStream.getvalue()
 
@@ -171,7 +174,7 @@ class PDFDocument(Image):
     """
     if not self.hasData():
       return ''
-    data = str(self.getData())
+    data = bytes(self.getData())
     try:
       from PyPDF2 import PdfFileReader
       from PyPDF2.utils import PdfReadError
@@ -179,7 +182,7 @@ class PDFDocument(Image):
       pass
     else:
       try:
-        if PdfFileReader(StringIO(data)).isEncrypted:
+        if PdfFileReader(BytesIO(data)).isEncrypted:
           return ''
       except PdfReadError:
         return ''
@@ -344,7 +347,7 @@ class PDFDocument(Image):
         raise
 
       result = {}
-      for line in command_result.splitlines():
+      for line in bytes2str(command_result).splitlines():
         item_list = line.split(':')
         key = item_list[0].strip()
         value = ':'.join(item_list[1:]).strip()
@@ -360,9 +363,9 @@ class PDFDocument(Image):
       else:
         try:
           pdf_file = PdfFileReader(tmp)
-          for info_key, info_value in (pdf_file.getDocumentInfo() or {}).iteritems():
+          for info_key, info_value in six.iteritems(pdf_file.getDocumentInfo() or {}):
             info_key = info_key.lstrip("/")
-            if isinstance(info_value, unicode):
+            if six.PY2 and isinstance(info_value, six.text_type):
               info_value = info_value.encode("utf-8")
 
             # Ignore values that cannot be pickled ( such as AAPL:Keywords )

bt5/erp5_officejs_appstore_base/SkinTemplateItem/portal_skins/erp5_officejs_appstore_base/SoftwarePublication_submitSoftwarePublication.py

@@ -60,12 +60,12 @@ if not zip_file:
   rejectSoftwarePublication(software_publication)
   return
 
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 import zipfile
 from zipfile import BadZipfile
 
-zipbuffer = StringIO()
-zipbuffer.write(str(zip_file.getData()))
+zipbuffer = BytesIO()
+zipbuffer.write(bytes(zip_file.getData()))
 try:
   zip_reader = zipfile.ZipFile(zipbuffer)
 except BadZipfile:

product/ERP5/bootstrap/erp5_core/DocumentTemplateItem/portal_components/document.erp5.File.py

@@ -37,7 +37,7 @@ from erp5.component.document.Document import ConversionError
 from Products.ERP5Type.Base import Base, removeIContentishInterface
 from OFS.Image import File as OFS_File
 from Products.ERP5Type.Utils import deprecated
-
+import six
 
 _MARKER = object()
 
@@ -67,7 +67,7 @@ class File(Document, OFS_File):
   security.declareObjectProtected(Permissions.AccessContentsInformation)
 
   # Default global values
-  data = '' # A hack required to use OFS.Image.index_html without calling OFS.Image.__init__
+  data = b'' # A hack required to use OFS.Image.index_html without calling OFS.Image.__init__
 
   # Default Properties
   property_sheets = ( PropertySheet.Base
@@ -186,6 +186,8 @@ class File(Document, OFS_File):
     if data is None:
       return None
     else:
+      if six.PY3 and isinstance(data, str):
+        return bytes(data, self._get_encoding())
       return bytes(data)
 
   # DAV Support

product/ERP5/bootstrap/erp5_core/MixinTemplateItem/portal_components/mixin.erp5.BaseConvertableFileMixin.py

@@ -33,6 +33,7 @@ from Products.ERP5Type.Message import translateString
 from Products.ERP5Type import Permissions
 from OFS.Image import Pdata
 from io import BytesIO
+import six
 _MARKER = object()
 
 class BaseConvertableFileMixin:

product/ERP5/bootstrap/erp5_core/MixinTemplateItem/portal_components/mixin.erp5.CachedConvertableMixin.py

@@ -37,6 +37,7 @@ from AccessControl import ClassSecurityInfo
 from Products.ERP5Type.Globals import InitializeClass
 from Products.ERP5Type import Permissions
 from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
+from Products.ERP5Type.Utils import str2bytes
 from OFS.Image import Pdata, Image as OFSImage
 from DateTime import DateTime
 
@@ -129,17 +130,20 @@ class CachedConvertableMixin:
       conversion_md5 = None
       size = 0
     elif isinstance(data, Pdata):
-      cached_value = aq_base(data)
-      size = str(cached_value) # not a size but avoids a 'del' statement
-      conversion_md5 = md5(size).hexdigest()
-      size = len(size)
+      cached_value = bytes(aq_base(data))
+      conversion_md5 = md5(cached_value).hexdigest()
+      size = len(cached_value)
     elif isinstance(data, OFSImage):
       warn('Passing an OFS.Image to setConversion is deprecated', stacklevel=1)
+      cached_value = bytes(data)
+      conversion_md5 = md5(cached_value).hexdigest()
+      size = len(cached_value)
+    elif isinstance(data, bytes):
       cached_value = data
-      conversion_md5 = md5(str(data.data)).hexdigest()
-      size = len(data.data)
+      conversion_md5 = md5(cached_value).hexdigest()
+      size = len(cached_value)
     elif isinstance(data, six.string_types):
-      cached_value = data
+      cached_value = str2bytes(data)
       conversion_md5 = md5(cached_value).hexdigest()
       size = len(cached_value)
     elif isinstance(data, dict):

product/ERP5/bootstrap/erp5_property_sheets/PropertySheetTemplateItem/portal_property_sheets/Data/data_property.xml

@@ -49,7 +49,7 @@
         </item>
         <item>
             <key> <string>property_default</string> </key>
-            <value> <string>python: \'\'</string> </value>
+            <value> <string>python: b\'\'</string> </value>
         </item>
       </dictionary>
     </pickle>

product/ERP5/bootstrap/erp5_property_sheets/PropertySheetTemplateItem/portal_property_sheets/Document/base_data_property.xml

@@ -49,7 +49,7 @@
         </item>
         <item>
             <key> <string>property_default</string> </key>
-            <value> <string>python: \'\'</string> </value>
+            <value> <string>python: b\'\'</string> </value>
         </item>
       </dictionary>
     </pickle>

product/ERP5OOo/FormPrintout.py

@@ -37,6 +37,7 @@ from Products.ERP5OOo.OOoUtils import OOoBuilder
 from Products.CMFCore.exceptions import AccessControl_Unauthorized
 from Acquisition import Implicit, aq_base
 from Products.ERP5Type.Globals import InitializeClass, DTMLFile, Persistent
+from Products.ERP5Type.Utils import bytes2str, str2bytes
 from AccessControl import ClassSecurityInfo
 from OFS.role import RoleManager
 from OFS.SimpleItem import Item
@@ -290,11 +291,11 @@ class FormPrintout(Implicit, Persistent, RoleManager, Item, PropertyManager):
     from erp5.component.document.Document import DocumentConversionServerProxy, enc, dec
     server_proxy = DocumentConversionServerProxy(self)
     extension = guess_extension(content_type).strip('.')
-    printout = dec(server_proxy.convertFile(enc(printout),
+    printout = dec(str2bytes(server_proxy.convertFile(bytes2str(enc(printout)),
                                         extension, # source_format
                                         extension, # destination_format
                                         False, # zip
-                                        True)) # refresh
+                                        True))) # refresh
     # End of temporary implementation
     if not format:
       if REQUEST is not None and not batch_mode:
@@ -317,7 +318,7 @@ class FormPrintout(Implicit, Persistent, RoleManager, Item, PropertyManager):
       REQUEST.RESPONSE.setHeader('Content-type', mime)
       REQUEST.RESPONSE.setHeader('Content-disposition',
           'attachment;filename="%s.%s"' % (filename, format))
-    return str(data)
+    return bytes(data)
 
 InitializeClass(FormPrintout)
 
@@ -574,7 +575,7 @@ class ODFStrategy(Implicit):
     path = image_field.get_value('default')
     image_node = image_list[0]
     image_frame = image_node.getparent()
-    if path is not None:
+    if six.PY2 and path is not None:
       path = path.encode()
     picture = self.getPortalObject().restrictedTraverse(path)
     picture_data = getattr(aq_base(picture), 'data', None)
@@ -816,19 +817,20 @@ class ODFStrategy(Implicit):
       \n -> line-breaks
       DateTime -> Y-m-d
     """
-    assert six.PY2 # TODO-py3
     if value is None:
       value = ''
-    translated_value = str(value)
     if isinstance(value, DateTime):
       translated_value = value.strftime('%Y-%m-%d')
+    elif isinstance(value, bytes):
+      translated_value = value.decode('utf-8')
+    else:
+      translated_value = str(value)
     translated_value = escape(translated_value)
     tab_element_str = '<text:tab xmlns:text="%s"/>' % TEXT_URI
     line_break_element_str ='<text:line-break xmlns:text="%s"/>' % TEXT_URI
     translated_value = translated_value.replace('\t', tab_element_str)
     translated_value = translated_value.replace('\r', '')
     translated_value = translated_value.replace('\n', line_break_element_str)
-    translated_value = unicode(str(translated_value),'utf-8')
     # create a paragraph
     template = '<text:p xmlns:text="%s">%s</text:p>'
     fragment_element_tree = etree.XML(template % (TEXT_URI, translated_value))

product/ERP5OOo/OOoTemplate.py

@@ -44,10 +44,11 @@ from Acquisition import aq_base
 from AccessControl import ClassSecurityInfo
 from .OOoUtils import OOoBuilder
 from zipfile import ZipFile, ZIP_DEFLATED
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 import re
 import itertools
 import six
+from Products.ERP5Type.Utils import bytes2str
 
 try:
   from zExceptions import ResourceLockedError
@@ -226,7 +227,7 @@ class OOoTemplate(ZopePageTemplate):
         self.OLE_documents_zipstring = None
       # create a zip archive and store it
       if attached_files_list:
-        memory_file = StringIO()
+        memory_file = BytesIO()
         try:
           zf = ZipFile(memory_file, mode='w', compression=ZIP_DEFLATED)
         except RuntimeError:
@@ -439,8 +440,8 @@ class OOoTemplate(ZopePageTemplate):
       draw_object.attrib.update({'{%s}href' % xml_doc.nsmap.get('xlink'): new_path})
       draw_object.attrib.update(dict(office_include.attrib))
       office_include.getparent().replace(office_include, draw_object)
-    text = etree.tostring(xml_doc, encoding='utf-8', xml_declaration=True,
-                          pretty_print=False)
+    text = bytes2str(etree.tostring(xml_doc, encoding='utf-8', xml_declaration=True,
+                                    pretty_print=False))
     text = re.sub('<\s*office:include_img\s+(.*?)\s*/\s*>(?s)', replaceIncludesImg, text)
 
     return (text, attached_files_dict)

product/ERP5Type/Accessor/TypeDefinition.py

@@ -44,6 +44,12 @@ ATTRIBUTE_PREFIX = ''
 def identity(value):
   return value
 
+
+def asData(value):
+  assert not isinstance(value, six.text_type)
+  return value
+
+
 def asFloat(value):
   """
     Return the value as a float or a type-specific default value if it fails.
@@ -101,7 +107,7 @@ def asString(value):
       if six.PY2 and isinstance(value, six.text_type):
         result = value.encode('utf-8')
       elif six.PY3 and isinstance(value, bytes):
-        result = value.decode('utf-8')
+        result = value.decode('utf-8', 'surrogateescape')
       else:
         result = str(value)
   except TypeError:
@@ -195,8 +201,8 @@ type_definition = {
                            # which intention is store large data
                            # such as files of BLOBs. It uses pdata
                            # structure.
-    'data'               : { 'cast'    : identity,
-                             'null'    : ('', 'None', None,),
+    'data'               : { 'cast'    : asData,
+                             'null'    : (b'', b'None', None,),
                              'default' : None,
                              'isList'  : 0,
                            },