py2/py3: data property is for binary, thus getData() should return bytes.

bt5/erp5_base/DocumentTemplateItem/portal_components/document.erp5.Image.py

@@ -32,8 +32,7 @@
 
 import os
 import subprocess
-from six.moves import cStringIO as StringIO
-
+from io import BytesIO
 from AccessControl import ClassSecurityInfo
 from Acquisition import aq_base
 
@@ -117,7 +116,7 @@ class Image(TextConvertableMixin, File, OFSImage):
     content_type, width, height = getImageInfo(self.data)
     if not content_type:
       try:
-        image = PIL.Image.open(StringIO(str(self.data)))
+        image = PIL.Image.open(BytesIO(bytes(self.data)))
       except IOError:
         width = height = -1
         content_type = 'application/unknown'
@@ -381,7 +380,7 @@ class Image(TextConvertableMixin, File, OFSImage):
     else:
       parameter_list.append('-')
 
-    data = str(self.getData())
+    data = bytes(self.getData())
     if self.getContentType() == "image/svg+xml":
       data = transformUrlToDataURI(data)
 
@@ -401,7 +400,7 @@ class Image(TextConvertableMixin, File, OFSImage):
     finally:
       del process
     if image:
-      return StringIO(image)
+      return BytesIO(image)
     raise ConversionError('Image conversion failed (%s).' % err)
 
   def _getContentTypeAndImageData(

bt5/erp5_base/DocumentTemplateItem/portal_components/document.erp5.OOoDocument.py

@@ -28,7 +28,7 @@
 ##############################################################################
 
 import re, zipfile
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 from warnings import warn
 from AccessControl import ClassSecurityInfo
 from OFS.Image import Pdata
@@ -39,7 +39,7 @@ from Products.ERP5Type.Cache import CachingMethod
 from erp5.component.document.File import File
 from erp5.component.document.Document import Document, \
        VALID_IMAGE_FORMAT_LIST, ConversionError, NotConvertedError
-from Products.ERP5Type.Utils import fill_args_from_request
+from Products.ERP5Type.Utils import bytes2str, fill_args_from_request, str2bytes
 
 # Mixin Import
 from erp5.component.mixin.BaseConvertableFileMixin import BaseConvertableFileMixin
@@ -199,10 +199,10 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       raise NotConvertedError()
     if format == 'text-content':
       # Extract text from the ODF file
-      cs = cStringIO.StringIO()
-      cs.write(str(self.getBaseData()))
+      cs = BytesIO()
+      cs.write(self.getBaseData())
       z = zipfile.ZipFile(cs)
-      s = z.read('content.xml')
+      s = bytes2str(z.read('content.xml'))
       s = self.rx_strip.sub(" ", s) # strip xml
       s = self.rx_compr.sub(" ", s) # compress multiple spaces
       cs.close()
@@ -211,7 +211,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
     server_proxy = DocumentConversionServerProxy(self)
     orig_format = self.getBaseContentType()
     generate_result = server_proxy.run_generate(self.getId(),
-                                       enc(str(self.getBaseData())),
+                                       bytes2str(enc(bytes(self.getBaseData()))),
                                        None,
                                        format,
                                        orig_format)
@@ -223,7 +223,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       response_dict = generate_result
 
     # XXX: handle possible OOOd server failure
-    return response_dict['mime'], Pdata(dec(response_dict['data']))
+    return response_dict['mime'], Pdata(dec(str2bytes(response_dict['data'])))
 
   # Conversion API
   def _convert(self, format, frame=0, **kw): #  pylint: disable=redefined-builtin
@@ -259,7 +259,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
     original_format = format
     allowed_format_list = self.getTargetFormatList()
     if format == 'base-data':
-      return self.getBaseContentType(), str(self.getBaseData())
+      return self.getBaseContentType(), self.getBaseData()
     if format == 'pdf':
       format_list = [x for x in allowed_format_list
                                           if x.endswith('pdf')]
@@ -302,8 +302,8 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       if is_html:
         # Extra processing required since
         # we receive a zip file
-        cs = cStringIO.StringIO()
-        cs.write(str(data))
+        cs = BytesIO()
+        cs.write(data)
         z = zipfile.ZipFile(cs) # A disk file would be more RAM efficient
         for f in z.infolist():
           fn = f.filename
@@ -325,7 +325,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
         # create temporary image and use it to resize accordingly
         temp_image = self.portal_contributions.newContent(
                                        portal_type='Image',
-                                       file=cStringIO.StringIO(),
+                                       file=BytesIO(),
                                        filename=self.getId(),
                                        temp_object=1)
         temp_image._setData(data)
@@ -347,8 +347,8 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       format_list = [x for x in self.getTargetFormatList()
                                 if x.startswith('html') or x.endswith('html')]
       mime, data = self._getConversionFromProxyServer(format_list[0])
-      archive_file = cStringIO.StringIO()
-      archive_file.write(str(data))
+      archive_file = BytesIO()
+      archive_file.write(data)
       zip_file = zipfile.ZipFile(archive_file)
       must_close = 1
     else:
@@ -385,13 +385,13 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
     server_proxy = DocumentConversionServerProxy(self)
     response_code, response_dict, response_message = server_proxy.run_convert(
                                       self.getFilename() or self.getId(),
-                                      enc(str(self.getData())),
+                                      bytes2str(enc(bytes(self.getData()))),
                                       None,
                                       None,
                                       self.getContentType())
     if response_code == 200:
       # sucessfully converted document
-      self._setBaseData(dec(response_dict['data']))
+      self._setBaseData(dec(str2bytes(response_dict['data'])))
       metadata = response_dict['meta']
       self._base_metadata = metadata
       if metadata.get('MIMEType', None) is not None:
@@ -424,11 +424,11 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
     server_proxy = DocumentConversionServerProxy(self)
     response_code, response_dict, response_message = \
           server_proxy.run_setmetadata(self.getId(),
-                                       enc(str(self.getBaseData())),
+                                       bytes2str(enc(bytes(self.getBaseData()))),
                                        kw)
     if response_code == 200:
       # successful meta data extraction
-      self._setBaseData(dec(response_dict['data']))
+      self._setBaseData(dec(str2bytes(response_dict['data'])))
       self.updateFileMetadata() # record in workflow history # XXX must put appropriate comments.
     else:
       # Explicitly raise the exception!

bt5/erp5_corporate_identity/SkinTemplateItem/portal_skins/erp5_corporate_identity/Presentation_viewAsSlideshow.py

@@ -29,7 +29,7 @@ if context.getPortalType() in ["Presentation"]:
   portal = context.getPortalObject()
   mimetype = 'text/html'
   content_type = context.getContentType()
-  raw_data = portal.portal_transforms.convertToData(mimetype, str(context.getData() or ""), context=context, mimetype=content_type)
+  raw_data = portal.portal_transforms.convertToData(mimetype, bytes(context.getData() or b""), context=context, mimetype=content_type)
   if raw_data is None:
     raise ValueError("Failed to convert to %r" % mimetype)
   if context.REQUEST is not None:

bt5/erp5_crm/TestTemplateItem/portal_components/test.erp5.testCRM.py

@@ -727,7 +727,7 @@ class TestCRMMailIngestion(BaseTestCRM):
     self.assertEqual(
       'Mail Message',
       self.portal.portal_contribution_registry.findPortalTypeName(
-      filename='postfix_mail.eml', content_type='message/rfc822', data='Test'
+      filename='postfix_mail.eml', content_type='message/rfc822', data=b'Test'
       ))
 
   def test_Base_getEntityListFromFromHeader(self):
@@ -844,13 +844,13 @@ class TestCRMMailIngestion(BaseTestCRM):
     event = self.portal.event_module.newContent(
         portal_type='Mail Message',
         destination_value=organisation,
-        data='\r\n'.join(textwrap.dedent('''
+        data=('\r\n'.join(textwrap.dedent('''
         From: Source <source@example.com>
         To: destination <destination@example.com>
         Subject: mail subject
 
         content
-        ''').splitlines()[1:]))
+        ''').splitlines()[1:])).encode())
 
     property_dict = event.getPropertyDictFromContent()
     # destination is set on the event. In this case it is kept as is.
@@ -1059,7 +1059,8 @@ class TestCRMMailIngestion(BaseTestCRM):
       file_path = '%s/test_data/%s' % (
         os.path.dirname(Products.ERP5.tests.__file__),
         filename)
-      event.setData(open(file_path).read())
+      with open(file_path, 'rb') as f:
+        event.setData(f.read())
       self.assertTrue(event.getTextContent().startswith('<'))
 
 
@@ -1811,7 +1812,7 @@ class TestCRMMailSend(BaseTestCRM):
     self.assertEqual(event.getTitle(), dummy_title)
     self.assertEqual(event.getTextContent(), dummy_content)
 
-    event.setData('Subject: %s\r\n\r\n%s' % (real_title, real_content))
+    event.setData(('Subject: %s\r\n\r\n%s' % (real_title, real_content)).encode())
     self.assertTrue(event.hasFile(), '%r has no file' % (event,))
     self.assertEqual(event.getTitle(), real_title)
     self.assertEqual(event.getTextContent(), real_content)

bt5/erp5_crm/TestTemplateItem/portal_components/test.erp5.testEditorField.py

@@ -215,7 +215,7 @@ class TestEditorField(ERP5TypeTestCase, ZopeTestCase.Functional):
 
     # Set a fake file on Event and make sure no more editor is displayed
     # and that instead a div with page CSS style appears with stripped HTML
-    event.setData('fake')
+    event.setData(b'fake')
     self.assertFalse(event.Event_view.my_text_content.get_value('editable'))
     html_text = event.view()
     self.assertTrue(self._isReadOnlyEditor(html_text, event, 'fake'))
@@ -244,7 +244,7 @@ class TestEditorField(ERP5TypeTestCase, ZopeTestCase.Functional):
 
     # Set a fake file on Event and make sure no more editor is displayed
     # and that instead a div with page CSS style appears with stripped HTML
-    event.setData('fake')
+    event.setData(b'fake')
     self.assertFalse(event.Event_view.my_text_content.get_value('editable'))
     html_text = event.view()
     self.assertTrue(self._isReadOnlyEditor(html_text, event, 'fake'))
@@ -272,7 +272,7 @@ class TestEditorField(ERP5TypeTestCase, ZopeTestCase.Functional):
 
     # Set a fake file on Event and make sure no more editor is displayed
     # and that instead a div with page CSS style appears with stripped HTML
-    event.setData('fake')
+    event.setData(b'fake')
     self.assertFalse(event.Event_view.my_text_content.get_value('editable'))
     html_text = event.view()
     self.assertTrue(self._isReadOnlyEditor(html_text, event, 'fake'))
@@ -300,7 +300,7 @@ class TestEditorField(ERP5TypeTestCase, ZopeTestCase.Functional):
 
     # Set a fake file on Event and make sure no more editor is displayed
     # and that instead a div with page CSS style appears with stripped HTML
-    event.setData('fake')
+    event.setData(b'fake')
     self.assertFalse(event.Event_view.my_text_content.get_value('editable'))
     html_text = event.view()
     self.assertTrue(self._isReadOnlyEditor(html_text, event, 'fake'))

bt5/erp5_dms/DocumentTemplateItem/portal_components/document.erp5.PDFDocument.py

@@ -32,13 +32,16 @@ import zope.interface
 from AccessControl import ClassSecurityInfo
 
 from Products.ERP5Type import Permissions, PropertySheet
+from Products.ERP5Type.Utils import bytes2str
 from erp5.component.interface.IWatermarkable import IWatermarkable
 from erp5.component.document.Image import Image
 from erp5.component.document.Document import ConversionError
 from subprocess import Popen, PIPE
 from zLOG import LOG, INFO, PROBLEM
 import errno
-from StringIO import StringIO
+from io import BytesIO
+from six.moves import range
+import six
 
 @zope.interface.implementer(IWatermarkable)
 class PDFDocument(Image):
@@ -90,8 +93,8 @@ class PDFDocument(Image):
         raise ValueError("watermark_data cannot not be empty")
       if not self.hasData():
         raise ValueError("Cannot watermark an empty document")
-      self_reader = PdfFileReader(StringIO(self.getData()))
-      watermark_reader = PdfFileReader(StringIO(watermark_data))
+      self_reader = PdfFileReader(BytesIO(self.getData()))
+      watermark_reader = PdfFileReader(BytesIO(watermark_data))
       watermark_page_count = watermark_reader.getNumPages()
 
       output = PdfFileWriter()
@@ -109,7 +112,7 @@ class PDFDocument(Image):
             self_page.mergePage(watermark_page)
         output.addPage(self_page)
 
-      outputStream = StringIO()
+      outputStream = BytesIO()
       output.write(outputStream)
       return outputStream.getvalue()
 
@@ -171,7 +174,7 @@ class PDFDocument(Image):
     """
     if not self.hasData():
       return ''
-    data = str(self.getData())
+    data = bytes(self.getData())
     try:
       from PyPDF2 import PdfFileReader
       from PyPDF2.utils import PdfReadError
@@ -179,7 +182,7 @@ class PDFDocument(Image):
       pass
     else:
       try:
-        if PdfFileReader(StringIO(data)).isEncrypted:
+        if PdfFileReader(BytesIO(data)).isEncrypted:
           return ''
       except PdfReadError:
         return ''
@@ -344,7 +347,7 @@ class PDFDocument(Image):
         raise
 
       result = {}
-      for line in command_result.splitlines():
+      for line in bytes2str(command_result).splitlines():
         item_list = line.split(':')
         key = item_list[0].strip()
         value = ':'.join(item_list[1:]).strip()
@@ -360,9 +363,9 @@ class PDFDocument(Image):
       else:
         try:
           pdf_file = PdfFileReader(tmp)
-          for info_key, info_value in (pdf_file.getDocumentInfo() or {}).iteritems():
+          for info_key, info_value in six.iteritems(pdf_file.getDocumentInfo() or {}):
             info_key = info_key.lstrip("/")
-            if isinstance(info_value, unicode):
+            if six.PY2 and isinstance(info_value, six.text_type):
               info_value = info_value.encode("utf-8")
 
             # Ignore values that cannot be pickled ( such as AAPL:Keywords )

bt5/erp5_dms/TestTemplateItem/portal_components/test.erp5.testDms.py

@@ -2900,7 +2900,7 @@ return 1
     document.setReference('TEST')
     request = self.app.REQUEST
     download_file = document.index_html(REQUEST=request, format=None)
-    self.assertEqual(download_file, 'foo\n')
+    self.assertEqual(download_file, b'foo\n')
     document_format = None
     self.assertEqual('TEST-001-en.dummy', document.getStandardFilename(
                       document_format))

bt5/erp5_officejs_appstore_base/SkinTemplateItem/portal_skins/erp5_officejs_appstore_base/SoftwarePublication_submitSoftwarePublication.py

@@ -60,12 +60,12 @@ if not zip_file:
   rejectSoftwarePublication(software_publication)
   return
 
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 import zipfile
 from zipfile import BadZipfile
 
-zipbuffer = StringIO()
-zipbuffer.write(str(zip_file.getData()))
+zipbuffer = BytesIO()
+zipbuffer.write(bytes(zip_file.getData()))
 try:
   zip_reader = zipfile.ZipFile(zipbuffer)
 except BadZipfile:

bt5/erp5_payroll_l10n_fr/TestTemplateItem/portal_components/test.erp5.testSimplifiedPayslipReport.py

@@ -181,7 +181,7 @@ class TestSimplifiedPayslipReport(ERP5TypeTestCase):
     image_source_pdf_doc.setData(pdf_data)
     self.tic()
     _, png = image_source_pdf_doc.convert("png", frame=0, quality=100)
-    self.assertImageRenderingEquals(str(png), str(expected_image.getData()))
+    self.assertImageRenderingEquals(bytes(png), bytes(expected_image.getData()))
 
   def test_03_payslip_holiday(self):
     for i in self.portal.portal_catalog(
@@ -257,5 +257,3 @@ class TestSimplifiedPayslipReport(ERP5TypeTestCase):
     self.assertEqual(payslip_data["report_data"]["total_holiday_this_year"], 2)
     self.assertEqual(payslip_data["report_data"]["taken_holiday"], 2)
     self.assertEqual(payslip_data["report_data"]["total_holiday_year_before"], 0)
-
-

bt5/erp5_web_shacache/TestTemplateItem/portal_components/test.erp5.ShaCacheMixin.py

@@ -52,5 +52,5 @@ class ShaCacheMixin(object):
     }
     self.shacache_url = self.shacache.absolute_url()
     self.tic()
-    self.data = 'Random Content. %s' % str(random.random())
+    self.data = 'Random Content. %s' % str(random.random()).encode()
     self.key = hashlib.sha512(self.data).hexdigest()

bt5/networkcache_erp5/TestTemplateItem/portal_components/test.erp5.testShaDirSecurity.py

@@ -216,7 +216,7 @@ class TestShaDirSecurity(ShaDirMixin, ShaSecurityMixin, SecurityTestCase):
     data_set = self.portal.data_set_module.newContent(portal_type='Data Set')
     document = self.portal.portal_contributions.newContent(
                                     filename='test.txt',
-                                    data='test content',
+                                    data=b'test content',
                                     reference='test-reference',
                                     discover_metadata=False,
                                     follow_up_list=[data_set.getRelativeUrl()])

product/ERP5/bootstrap/erp5_core/DocumentTemplateItem/portal_components/document.erp5.File.py

@@ -37,7 +37,7 @@ from erp5.component.document.Document import ConversionError
 from Products.ERP5Type.Base import Base, removeIContentishInterface
 from OFS.Image import File as OFS_File
 from Products.ERP5Type.Utils import deprecated
-
+import six
 
 _MARKER = object()
 
@@ -67,7 +67,7 @@ class File(Document, OFS_File):
   security.declareObjectProtected(Permissions.AccessContentsInformation)
 
   # Default global values
-  data = '' # A hack required to use OFS.Image.index_html without calling OFS.Image.__init__
+  data = b'' # A hack required to use OFS.Image.index_html without calling OFS.Image.__init__
 
   # Default Properties
   property_sheets = ( PropertySheet.Base
@@ -186,6 +186,8 @@ class File(Document, OFS_File):
     if data is None:
       return None
     else:
+      if six.PY3 and isinstance(data, str):
+        return bytes(data, self._get_encoding())
       return bytes(data)
 
   # DAV Support

product/ERP5/bootstrap/erp5_core/MixinTemplateItem/portal_components/mixin.erp5.BaseConvertableFileMixin.py

@@ -33,6 +33,7 @@ from Products.ERP5Type.Message import translateString
 from Products.ERP5Type import Permissions
 from OFS.Image import Pdata
 from io import BytesIO
+import six
 _MARKER = object()
 
 class BaseConvertableFileMixin:

product/ERP5/bootstrap/erp5_core/MixinTemplateItem/portal_components/mixin.erp5.CachedConvertableMixin.py

@@ -37,6 +37,7 @@ from AccessControl import ClassSecurityInfo
 from Products.ERP5Type.Globals import InitializeClass
 from Products.ERP5Type import Permissions
 from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
+from Products.ERP5Type.Utils import str2bytes
 from OFS.Image import Pdata, Image as OFSImage
 from DateTime import DateTime
 
@@ -132,17 +133,20 @@ class CachedConvertableMixin:
       conversion_md5 = None
       size = 0
     elif isinstance(data, Pdata):
-      cached_value = aq_base(data)
-      size = str(cached_value) # not a size but avoids a 'del' statement
-      conversion_md5 = md5(size).hexdigest()
-      size = len(size)
+      cached_value = bytes(aq_base(data))
+      conversion_md5 = md5(cached_value).hexdigest()
+      size = len(cached_value)
     elif isinstance(data, OFSImage):
       warn('Passing an OFS.Image to setConversion is deprecated', stacklevel=1)
+      cached_value = bytes(data)
+      conversion_md5 = md5(cached_value).hexdigest()
+      size = len(cached_value)
+    elif isinstance(data, bytes):
       cached_value = data
-      conversion_md5 = md5(str(data.data)).hexdigest()
-      size = len(data.data)
+      conversion_md5 = md5(cached_value).hexdigest()
+      size = len(cached_value)
     elif isinstance(data, six.string_types):
-      cached_value = data
+      cached_value = str2bytes(data)
       conversion_md5 = md5(cached_value).hexdigest()
       size = len(cached_value)
     elif isinstance(data, dict):

product/ERP5/bootstrap/erp5_property_sheets/PropertySheetTemplateItem/portal_property_sheets/Data/data_property.xml

@@ -49,7 +49,7 @@
         </item>
         <item>
             <key> <string>property_default</string> </key>
-            <value> <string>python: \'\'</string> </value>
+            <value> <string>python: b\'\'</string> </value>
         </item>
       </dictionary>
     </pickle>

product/ERP5/bootstrap/erp5_property_sheets/PropertySheetTemplateItem/portal_property_sheets/Document/base_data_property.xml

@@ -49,7 +49,7 @@
         </item>
         <item>
             <key> <string>property_default</string> </key>
-            <value> <string>python: \'\'</string> </value>
+            <value> <string>python: b\'\'</string> </value>
         </item>
       </dictionary>
     </pickle>

product/ERP5OOo/FormPrintout.py

@@ -37,6 +37,7 @@ from Products.ERP5OOo.OOoUtils import OOoBuilder
 from Products.CMFCore.exceptions import AccessControl_Unauthorized
 from Acquisition import Implicit, aq_base
 from Products.ERP5Type.Globals import InitializeClass, DTMLFile, Persistent
+from Products.ERP5Type.Utils import bytes2str, str2bytes
 from AccessControl import ClassSecurityInfo
 from OFS.role import RoleManager
 from OFS.SimpleItem import Item
@@ -290,11 +291,11 @@ class FormPrintout(Implicit, Persistent, RoleManager, Item, PropertyManager):
     from erp5.component.document.Document import DocumentConversionServerProxy, enc, dec
     server_proxy = DocumentConversionServerProxy(self)
     extension = guess_extension(content_type).strip('.')
-    printout = dec(server_proxy.convertFile(enc(printout),
+    printout = dec(str2bytes(server_proxy.convertFile(bytes2str(enc(printout)),
                                         extension, # source_format
                                         extension, # destination_format
                                         False, # zip
-                                        True)) # refresh
+                                        True))) # refresh
     # End of temporary implementation
     if not format:
       if REQUEST is not None and not batch_mode:
@@ -317,7 +318,7 @@ class FormPrintout(Implicit, Persistent, RoleManager, Item, PropertyManager):
       REQUEST.RESPONSE.setHeader('Content-type', mime)
       REQUEST.RESPONSE.setHeader('Content-disposition',
           'attachment;filename="%s.%s"' % (filename, format))
-    return str(data)
+    return bytes(data)
 
 InitializeClass(FormPrintout)
 
@@ -574,7 +575,7 @@ class ODFStrategy(Implicit):
     path = image_field.get_value('default')
     image_node = image_list[0]
     image_frame = image_node.getparent()
-    if path is not None:
+    if six.PY2 and path is not None:
       path = path.encode()
     picture = self.getPortalObject().restrictedTraverse(path)
     picture_data = getattr(aq_base(picture), 'data', None)
@@ -816,19 +817,20 @@ class ODFStrategy(Implicit):
       \n -> line-breaks
       DateTime -> Y-m-d
     """
-    assert six.PY2 # TODO-py3
     if value is None:
       value = ''
-    translated_value = str(value)
     if isinstance(value, DateTime):
       translated_value = value.strftime('%Y-%m-%d')
+    elif isinstance(value, bytes):
+      translated_value = value.decode('utf-8')
+    else:
+      translated_value = str(value)
     translated_value = escape(translated_value)
     tab_element_str = '<text:tab xmlns:text="%s"/>' % TEXT_URI
     line_break_element_str ='<text:line-break xmlns:text="%s"/>' % TEXT_URI
     translated_value = translated_value.replace('\t', tab_element_str)
     translated_value = translated_value.replace('\r', '')
     translated_value = translated_value.replace('\n', line_break_element_str)
-    translated_value = unicode(str(translated_value),'utf-8')
     # create a paragraph
     template = '<text:p xmlns:text="%s">%s</text:p>'
     fragment_element_tree = etree.XML(template % (TEXT_URI, translated_value))

product/ERP5OOo/OOoTemplate.py

@@ -44,10 +44,11 @@ from Acquisition import aq_base
 from AccessControl import ClassSecurityInfo
 from .OOoUtils import OOoBuilder
 from zipfile import ZipFile, ZIP_DEFLATED
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 import re
 import itertools
 import six
+from Products.ERP5Type.Utils import bytes2str
 
 try:
   from zExceptions import ResourceLockedError
@@ -226,7 +227,7 @@ class OOoTemplate(ZopePageTemplate):
         self.OLE_documents_zipstring = None
       # create a zip archive and store it
       if attached_files_list:
-        memory_file = StringIO()
+        memory_file = BytesIO()
         try:
           zf = ZipFile(memory_file, mode='w', compression=ZIP_DEFLATED)
         except RuntimeError:
@@ -439,8 +440,8 @@ class OOoTemplate(ZopePageTemplate):
       draw_object.attrib.update({'{%s}href' % xml_doc.nsmap.get('xlink'): new_path})
       draw_object.attrib.update(dict(office_include.attrib))
       office_include.getparent().replace(office_include, draw_object)
-    text = etree.tostring(xml_doc, encoding='utf-8', xml_declaration=True,
-                          pretty_print=False)
+    text = bytes2str(etree.tostring(xml_doc, encoding='utf-8', xml_declaration=True,
+                                    pretty_print=False))
     text = re.sub('<\s*office:include_img\s+(.*?)\s*/\s*>(?s)', replaceIncludesImg, text)
 
     return (text, attached_files_dict)

product/ERP5OOo/tests/TestFormPrintoutMixin.py

@@ -32,7 +32,6 @@
 
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from AccessControl.SecurityManagement import newSecurityManager
-from six import StringIO
 
 class TestFormPrintoutMixin(ERP5TypeTestCase):
   run_all_test = 1
@@ -55,7 +54,7 @@ class TestFormPrintoutMixin(ERP5TypeTestCase):
     '''return odf document from the printout
     '''
     document_file = getattr(self.portal, printout_form.template, None)
-    document_file = StringIO(document_file).read()
+    document_file = bytes(document_file)
     if document_file is not None:
       return document_file
     raise ValueError ('%s template not found' % printout_form.template)

product/ERP5OOo/tests/testFormPrintoutAsODT.py

@@ -37,6 +37,7 @@ from Products.MimetypesRegistry.mime_types.magic import guessMime
 from Products.ERP5OOo.OOoUtils import OOoBuilder
 from Products.ERP5OOo.tests.utils import Validator
 from Products.ERP5Type.tests.utils import FileUpload
+from Products.ERP5Type.Utils import bytes2str
 from DateTime import DateTime
 from lxml import etree
 import os
@@ -148,7 +149,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("Foo title!") > 0)
+    self.assertTrue(bytes2str(content_xml).find("Foo title!") > 0)
     self.assertEqual(request.RESPONSE.getHeader('content-type'),
                      'application/vnd.oasis.opendocument.text')
     self.assertEqual(request.RESPONSE.getHeader('content-disposition'),
@@ -167,7 +168,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("Changed Title!") > 0)
+    self.assertTrue(bytes2str(content_xml).find("Changed Title!") > 0)
     self._validate(odf_document)
 
     # 3. False case: change the field name
@@ -179,7 +180,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertFalse(content_xml.find("you cannot find") > 0)
+    self.assertFalse(bytes2str(content_xml).find("you cannot find") > 0)
     self._validate(odf_document)
     # put back
     foo_form.manage_renameObject('xxx_title', 'my_title', REQUEST=request)
@@ -199,7 +200,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("call!") > 0)
+    self.assertTrue(bytes2str(content_xml).find("call!") > 0)
     # Zope4 add charset=utf-8
     self.assertTrue('text/html' in request.RESPONSE.getHeader('content-type'))
     self._validate(odf_document)
@@ -210,7 +211,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("Français") > 0)
+    self.assertTrue(bytes2str(content_xml).find("Français") > 0)
     self._validate(odf_document)
 
     # 6. Normal case: unicode string
@@ -219,7 +220,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("Français test2") > 0)
+    self.assertTrue(bytes2str(content_xml).find("Français test2") > 0)
     self._validate(odf_document)
 
     # 7. Change Filename of downloadable file
@@ -307,8 +308,8 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     #test_output.write(odf_document)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("ZZZ test here ZZZ") > 0)
-    self.assertTrue(content_xml.find("test title") < 0)
+    self.assertTrue(bytes2str(content_xml).find("ZZZ test here ZZZ") > 0)
+    self.assertTrue(bytes2str(content_xml).find("test title") < 0)
     self._validate(odf_document)
 
   def test_02_Table_01_Normal(self):
@@ -360,7 +361,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("foo_title_1") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_title_1") > 0)
     self._validate(odf_document)
 
   def test_02_Table_02_SmallerThanListboxColumns(self):
@@ -401,8 +402,8 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertFalse(content_xml.find("foo_title_1") > 0)
-    self.assertTrue(content_xml.find("foo_title_2") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_title_1") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_title_2") > 0)
     self._validate(odf_document)
 
   def test_02_Table_03_ListboxColumnsLargerThanTable(self):
@@ -437,8 +438,8 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertFalse(content_xml.find("foo_title_2") > 0)
-    self.assertTrue(content_xml.find("foo_title_3") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_title_2") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_title_3") > 0)
     self._validate(odf_document)
 
   def test_02_Table_04_ListboxHasNotStat(self):
@@ -473,8 +474,8 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
     self.assertTrue(odf_document is not None)
-    self.assertFalse(content_xml.find("foo_title_3") > 0)
-    self.assertTrue(content_xml.find("foo_title_4") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_title_3") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_title_4") > 0)
 
     content = etree.XML(content_xml)
     table_row_xpath = '//table:table[@table:name="listbox"]/table:table-row'
@@ -542,7 +543,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     #Check that each listbox values are inside ODT table cells
     xpath_result_expression = '//table:table[@table:name="listbox2"]/table:table-row/table:table-cell/text:p/text()'
     self.assertEqual(['foo_1', 'foo_title_5', '0.0', 'foo_2', 'foo_2', '0.0', '1234.5'], content_tree.xpath(xpath_result_expression, namespaces=content_tree.nsmap))
-    self.assertFalse(content_xml.find("foo_title_4") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_title_4") > 0)
     self._validate(odf_document)
 
     # put back the field name
@@ -590,8 +591,8 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertFalse(content_xml.find("foo_title_5") > 0)
-    self.assertTrue(content_xml.find("foo_title_6") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_title_5") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_title_6") > 0)
     self._validate(odf_document)
 
     # put back the field name
@@ -627,8 +628,8 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertFalse(content_xml.find("foo_title_6") > 0)
-    self.assertTrue(content_xml.find("foo_title_7") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_title_6") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_title_7") > 0)
 
     content = etree.XML(content_xml)
     table_row_xpath = '//table:table[@table:name="listbox"]/table:table-row'
@@ -702,7 +703,7 @@ class TestFormPrintoutAsODT(TestFormPrintoutMixin):
 r"""
 line_index = kw['list_index']
 line_number = line_index + 1
-for n in xrange(6, 0, -1):
+for n in range(6, 0, -1):
   if line_number % n is 0:
     return "line" + str(n)
 """
@@ -729,7 +730,7 @@ for n in xrange(6, 0, -1):
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("foo_title_9") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_title_9") > 0)
 
     content = etree.XML(content_xml)
     table_row_xpath = '//table:table[@table:name="listbox4"]/table:table-row'
@@ -838,7 +839,7 @@ return report_section_list
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("foo_04_Iteration_1") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_04_Iteration_1") > 0)
     content = etree.XML(content_xml)
     frame_xpath = '//draw:frame[@draw:name="FooReport_getReportSectionList"]'
     frame_list = content.xpath(frame_xpath, namespaces=content.nsmap)
@@ -865,7 +866,7 @@ return []
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertFalse(content_xml.find("foo_04_Iteration") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_04_Iteration") > 0)
     content = etree.XML(content_xml)
     frame_xpath = '//draw:frame[@draw:name="FooReport_getReportSectionList"]'
     frame_list = content.xpath(frame_xpath, namespaces=content.nsmap)
@@ -942,7 +943,7 @@ return report_section_list
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("foo_04_Iteration_1") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_04_Iteration_1") > 0)
     content = etree.XML(content_xml)
     section_xpath = '//text:section[@text:name="FooReport_getReportSectionList"]'
     section_list = content.xpath(section_xpath, namespaces=content.nsmap)
@@ -969,7 +970,7 @@ return []
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertFalse(content_xml.find("foo_04_Iteration") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_04_Iteration") > 0)
     content = etree.XML(content_xml)
     section_xpath = '//text:section[@text:name="FooReport_getReportSectionList"]'
     section_list = content.xpath(section_xpath, namespaces=content.nsmap)
@@ -1054,7 +1055,7 @@ return report_section_list
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertTrue(content_xml.find("foo_04_Iteration_1") > 0)
+    self.assertTrue(bytes2str(content_xml).find("foo_04_Iteration_1") > 0)
     content = etree.XML(content_xml)
     section_xpath = '//text:section[@text:name="your_report_box1"]'
     section_list = content.xpath(section_xpath, namespaces=content.nsmap)
@@ -1081,7 +1082,7 @@ return []
     self.assertTrue(odf_document is not None)
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
-    self.assertFalse(content_xml.find("foo_04_Iteration") > 0)
+    self.assertFalse(bytes2str(content_xml).find("foo_04_Iteration") > 0)
     content = etree.XML(content_xml)
     section_xpath = '//text:section[@text:name="your_report_box1"]'
     section_list = content.xpath(section_xpath, namespaces=content.nsmap)
@@ -1142,7 +1143,7 @@ return []
     builder = OOoBuilder(odf_document)
     content_xml = builder.extract("content.xml")
     # confirming the image was removed
-    self.assertTrue(content_xml.find('<draw:image xlink:href') < 0)
+    self.assertTrue(bytes2str(content_xml).find('<draw:image xlink:href') < 0)
     self._validate(odf_document)
 
   def test_08_OOoConversion(self):

product/ERP5OOo/tests/utils.py

@@ -41,6 +41,7 @@ import zipfile
 import subprocess
 from six.moves import urllib
 from six.moves import cStringIO as StringIO
+from io import BytesIO
 
 try:
   import lxml
@@ -65,14 +66,14 @@ if lxml:
 
     def validate(self, odf_file_content):
       error_list = []
-      odf_file = StringIO(odf_file_content)
+      odf_file = BytesIO(odf_file_content)
       for f in ('content.xml', 'meta.xml', 'styles.xml', 'settings.xml'):
         error_list.extend(self._validateXML(odf_file, f))
       return error_list
 
     def _validateXML(self, odf_file, content_file_name):
       zfd = zipfile.ZipFile(odf_file)
-      doc = lxml.etree.parse(StringIO(zfd.read(content_file_name)))
+      doc = lxml.etree.parse(BytesIO(zfd.read(content_file_name)))
       return []
       # The following is the past implementation that validates with
       # RelaxNG schema. But recent LibreOffice uses extended odf

product/ERP5Type/Accessor/TypeDefinition.py

@@ -44,6 +44,12 @@ ATTRIBUTE_PREFIX = ''
 def identity(value):
   return value
 
+
+def asData(value):
+  assert not isinstance(value, six.text_type)
+  return value
+
+
 def asFloat(value):
   """
     Return the value as a float or a type-specific default value if it fails.
@@ -101,7 +107,7 @@ def asString(value):
       if six.PY2 and isinstance(value, six.text_type):
         result = value.encode('utf-8')
       elif six.PY3 and isinstance(value, bytes):
-        result = value.decode('utf-8')
+        result = value.decode('utf-8', 'surrogateescape')
       else:
         result = str(value)
   except TypeError:
@@ -195,8 +201,8 @@ type_definition = {
                            # which intention is store large data
                            # such as files of BLOBs. It uses pdata
                            # structure.
-    'data'               : { 'cast'    : identity,
-                             'null'    : ('', 'None', None,),
+    'data'               : { 'cast'    : asData,
+                             'null'    : (b'', b'None', None,),
                              'default' : None,
                              'isList'  : 0,
                            },