passing 'preferred_encodings' down to the encoding sniffer

in order to make the sniffing a bit configurable for instances
working with encodings other than iso-8859-15 or utf-8

doc/CHANGES.txt

@@ -39,7 +39,13 @@ Zope Changes
         until the late startup phase. This in in particular useful when running
         Zope behind a loadbalancer (patch by Patrick Gerken).
 
-      - the ZopePageTemplate implementation now uses unicode internally.        
+      - the ZopePageTemplate implementation now uses unicode internally.
+        Non-unicode instances are migrated on-the-fly to unicode. However this
+        will work only properly for ZPT instances formerly encoded as utf-8 or
+        ISO-8859-15. For other encodings you might set the environment variable
+        ZPT_REFERRED_ENCODING to insert your preferred encoding in front of utf-8 and
+        ISO-8859-15 within the encoding sniffer code.
+
 
     Bugs Fixed
 

lib/python/Products/PageTemplates/ZopePageTemplate.py

@@ -408,7 +408,9 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
         # Perform on-the-fly migration to unicode.
         # Perhaps it might be work with the 'generation' module here?
         if not isinstance(state['_text'], unicode):
-            text, encoding = convertToUnicode(state['_text'], state.get('content_type', 'text/html'))
+            text, encoding = convertToUnicode(state['_text'], 
+                                    state.get('content_type', 'text/html'), 
+                                    preferred_encodings)
             state['_text'] = text
             state['output_encoding'] = encoding
         self.__dict__.update(state) 
@@ -423,31 +425,6 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
     def wl_isLocked(self):
         return 0
 
-    def manage_convertUnicode(self, preferred_encodings=preferred_encodings,
-                              RESPONSE=None):
-        """Convert non-unicode templates to unicode"""
-        if not isinstance(self._text, unicode):
-            for encoding in preferred_encodings:
-                try:
-                    self._text = unicode(self._text, encoding)
-                    if RESPONSE:
-                        return RESPONSE.redirect(self.absolute_url() +
-                                                 '/pt_editForm?manage_tabs_message='
-                                                 'ZPT+successfully+converted')
-                    else:
-                        return
-                except UnicodeDecodeError:
-                    pass
-
-            raise RuntimeError('Pagetemplate could not be converted to unicode')
-
-        else:
-            if RESPONSE:
-                return RESPONSE.redirect(self.absolute_url() +
-                                         '/pt_editForm?manage_tabs_message='
-                                         'ZPT+already+converted')
-            else:
-                return
 
 InitializeClass(ZopePageTemplate)
 

lib/python/Products/PageTemplates/utils.py

@@ -58,7 +58,7 @@ def charsetFromMetaEquiv(html):
 
                                                                           
 
-def convertToUnicode(source, content_type):
+def convertToUnicode(source, content_type, preferred_encodings):
     """ Convert 'source' to unicode.
         Returns (unicode_str, source_encoding).
     """
@@ -77,7 +77,7 @@ def convertToUnicode(source, content_type):
         # including them into the Zope 2 core
 
         if not encoding:
-            for enc in ('utf-8', 'iso-8859-15'):
+            for enc in preferred_encodings:
                 try:
                     return unicode(source, enc), enc
                 except UnicodeDecodeError: