- in-place migration to unicode

- the ZMI edit view now *always* uses 'utf-8' and no longer uses self.output_encoding (which is only relevant for FTP/WebDAV)
- 'output_encoding' is now a property

lib/python/Products/PageTemplates/ZopePageTemplate.py

@@ -40,7 +40,7 @@ from Products.PageTemplates.PageTemplateFile import PageTemplateFile
 from Products.PageTemplates.PageTemplateFile import guess_type
 from Products.PageTemplates.Expressions import SecureModuleImporter
 
-from Products.PageTemplates.utils import encodingFromXMLPreamble, charsetFromMetaEquiv
+from Products.PageTemplates.utils import encodingFromXMLPreamble, charsetFromMetaEquiv, convertToUnicode
             
 
 preferred_encodings = ['utf-8', 'iso-8859-15']
@@ -70,6 +70,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
     __implements__ = (WriteLockInterface,)
 
     meta_type = 'Page Template'
+    output_encoding = 'iso-8859-15'  # provide default for old instances
 
     func_defaults = None
     func_code = FuncCode((), 0)
@@ -90,6 +91,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
 
     _properties=({'id':'title', 'type': 'ustring', 'mode': 'w'},
                  {'id':'content_type', 'type':'string', 'mode': 'w'},
+                 {'id':'output_encoding', 'type':'string', 'mode': 'w'},
                  {'id':'expand', 'type':'boolean', 'mode': 'w'},
                  )
 
@@ -183,7 +185,12 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
             raise ResourceLockedError("File is locked via WebDAV")
 
         self.expand = expand
-        self.pt_setTitle(title, self.output_encoding)
+
+        # The ZMI edit view uses utf-8! So we can safely assume
+        # that 'title' and 'text' are utf-8 encoded strings - hopefully
+
+        self.pt_setTitle(title, 'utf-8') 
+        text = unicode(text, 'utf-8')
 
         self.pt_edit(text, content_type, True)
         REQUEST.set('text', self.read()) # May not equal 'text'!
@@ -194,6 +201,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
                        % '<br>'.join(self._v_warnings))
         return self.pt_editForm(manage_tabs_message=message)
 
+
     security.declareProtected(change_page_templates, 'pt_setTitle')
     def pt_setTitle(self, title, encoding='utf-8'):
         if not isinstance(title, unicode):
@@ -394,6 +402,17 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
             # acquisition context, so we don't know where it is. :-(
             return None
 
+
+    def __setstate__(self, state):
+        # Perform on-the-fly migration to unicode.
+        # Perhaps it might be work with the 'generation' module here?
+        if not isinstance(state['_text'], unicode):
+            text, encoding = convertToUnicode(state['_text'], state['content_type'])
+            state['_text'] = text
+            state['output_encoding'] = encoding
+        self.__dict__.update(state) 
+
+
     def pt_render(self, source=False, extra_context={}):
         result = PageTemplate.pt_render(self, source, extra_context)
         assert isinstance(result, unicode)

lib/python/Products/PageTemplates/utils.py

@@ -56,4 +56,34 @@ def charsetFromMetaEquiv(html):
 
     return None
 
+                                                                          
 
+def convertToUnicode(source, content_type):
+    """ Convert 'source' to unicode.
+        Returns (unicode_str, source_encoding).
+    """
+
+    if content_type.startswith('text/xml'):
+        encoding = encodingFromXMLPreamble(source)
+        return unicode(source, encoding), encoding  
+
+    elif content_type.startswith('text/html'):
+
+        encoding = charsetFromMetaEquiv(source)
+
+        # Try to detect the encoding by converting it unicode without raising
+        # exceptions. There are some smarter Python-based sniffer methods
+        # available however we have to check their licenses first before
+        # including them into the Zope 2 core
+
+        if not encoding:
+            for enc in ('utf-8', 'iso-8859-15'):
+                try:
+                    return unicode(source, enc), enc
+                except UnicodeDecodeError:
+                    continue
+
+        raise TypeError('Could not auto-detect encoding')
+
+    else:
+        raise ValueError('Unsupported content-type: %s' % content_type) 

lib/python/Products/PageTemplates/www/ptEdit.zpt

-<h1 tal:replace="structure python:context.manage_page_header(management_page_charset=context.output_encoding)">Header</h1>
+<h1 tal:replace="structure python:context.manage_page_header(management_page_charset='utf-8')">Header</h1>
 <h2 tal:define="manage_tabs_message options/manage_tabs_message | nothing"
     tal:replace="structure context/manage_tabs">Tabs</h2>
 
@@ -44,19 +44,7 @@
     </td>
   </tr>
 
-  <tr>
-    <td align="left" valign="middle">
-      <div class="form-label">Output encoding</div>
-    </td>
-    <td align="left" valign="middle">
-      <div class="form-text" 
-           tal:content="context/output_encoding"
-      />
-    </td>
-  </tr>
-
   <tr tal:define="errors context/pt_errors" tal:condition="errors">
-    <tal:block define="global body python:context.document_src({'raw':1})" />
     <td align="left" valign="middle" class="form-label">Errors</td>
     <td align="left" valign="middle" style="background-color: #FFDDDD"
         colspan="3">