merge from trunk (PreferredResolver, more tests), fixes Collector #2180

020a6d18 · Andreas Jung · c3a84f48 · 020a6d18 · 020a6d18 · 020a6d18
Commit 020a6d18 authored Jan 09, 2007 by Andreas Jung
3 changed files
--- a/lib/python/Products/PageTemplates/configure.zcml
+++ b/lib/python/Products/PageTemplates/configure.zcml
@@ -2,7 +2,7 @@

  <utility
      provides="Products.PageTemplates.interfaces.IUnicodeEncodingConflictResolver"
-      component="Products.PageTemplates.unicodeconflictresolver.DefaultUnicodeEncodingConflictResolver"
+      component="Products.PageTemplates.unicodeconflictresolver.PreferredCharsetResolver"
      />

 </configure>
--- a/lib/python/Products/PageTemplates/tests/testZopePageTemplate.py
+++ b/lib/python/Products/PageTemplates/tests/testZopePageTemplate.py
@@ -11,19 +11,25 @@ Note: Tests require Zope >= 2.7
 import unittest
 import Zope2
 import transaction
+
 import zope.component.testing
-from zope.traversing.adapters import DefaultTraversable
+from zope.traversing.adapters import DefaultTraversable, Traverser
+from zope.publisher.http import HTTPCharsets 
+
 from Testing.makerequest import makerequest
 from Testing.ZopeTestCase import ZopeTestCase, installProduct
 from Products.PageTemplates.ZopePageTemplate import ZopePageTemplate, manage_addPageTemplate
 from Products.PageTemplates.utils import encodingFromXMLPreamble, charsetFromMetaEquiv
+from zope.component import provideUtility
+from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
+from Products.PageTemplates.unicodeconflictresolver import PreferredCharsetResolver


 ascii_str = '<html><body>hello world</body></html>'
 iso885915_str = '<html><body></body></html>'
 utf8_str = unicode(iso885915_str, 'iso-8859-15').encode('utf-8')

-xml_template = '''<?xml vesion="1.0" encoding="%s"?>
+xml_template = '''<?xml version="1.0" encoding="%s"?>
 <foo>

 </foo>
@@ -76,7 +82,51 @@ class ZPTUtilsTests(unittest.TestCase):
        self.assertEqual(extract('<html><META http-equiv="content-type" content="text/html; charset=iso-8859-15"></html>'), 'iso-8859-15')
        self.assertEqual(extract('<html><META http-equiv="content-type" content="text/html"></html>'), None)
        self.assertEqual(extract('<html>...<html>'), None)
-        
+
+
+class ZPTUnicodeEncodingConflictResolution(ZopeTestCase):
+
+    def setUp(self):
+        super(ZPTUnicodeEncodingConflictResolution, self).setUp()
+        zope.component.provideAdapter(DefaultTraversable, (None,))
+        zope.component.provideAdapter(HTTPCharsets, (None,))
+        provideUtility(PreferredCharsetResolver, IUnicodeEncodingConflictResolver)
+        transaction.begin()
+
+    def tearDown(self):
+        transaction.abort()
+        self.app._p_jar.close()
+
+    def testISO_8859_15(self):
+        manage_addPageTemplate(self.app, 'test', 
+                               text='<div tal:content="python: request.get(\'data\')" />', 
+                               encoding='ascii')
+        zpt = self.app['test']
+        self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'ISO-8859-15,utf-8')
+        self.app.REQUEST.set('data', '')
+        result = zpt.pt_render()
+        self.assertEqual(result.startswith(unicode('<div></div>', 'iso-8859-15')), True)
+
+    def testUTF8(self):
+        manage_addPageTemplate(self.app, 'test', 
+                               text='<div tal:content="python: request.get(\'data\')" />', 
+                               encoding='ascii')
+        zpt = self.app['test']
+        self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'utf-8,ISO-8859-15')
+        self.app.REQUEST.set('data', unicode('', 'iso-8859-15').encode('utf-8'))
+        result = zpt.pt_render()
+        self.assertEqual(result.startswith(unicode('<div></div>', 'iso-8859-15')), True)
+
+    def testUTF8WrongPreferredCharset(self):
+        manage_addPageTemplate(self.app, 'test', 
+                               text='<div tal:content="python: request.get(\'data\')" />', 
+                               encoding='ascii')
+        zpt = self.app['test']
+        self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'iso-8859-15')
+        self.app.REQUEST.set('data', unicode('', 'iso-8859-15').encode('utf-8'))
+        result = zpt.pt_render()
+        self.assertEqual(result.startswith(unicode('<div></div>', 'iso-8859-15')), False)
+

 class ZopePageTemplateFileTests(ZopeTestCase):

@@ -273,6 +323,7 @@ def test_suite():
    suite.addTests(unittest.makeSuite(ZPTUtilsTests))
    suite.addTests(unittest.makeSuite(ZPTMacros))
    suite.addTests(unittest.makeSuite(ZopePageTemplateFileTests))
+    suite.addTests(unittest.makeSuite(ZPTUnicodeEncodingConflictResolution))
    return suite

 if __name__ == '__main__':

--- a/lib/python/Products/PageTemplates/unicodeconflictresolver.py
+++ b/lib/python/Products/PageTemplates/unicodeconflictresolver.py
@@ -12,7 +12,10 @@
 ##############################################################################

 import sys
+
 from zope.interface import implements
+from zope.i18n.interfaces import IUserPreferredCharsets
+
 from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver

 default_encoding = sys.getdefaultencoding()
@@ -47,10 +50,47 @@ class Z2UnicodeEncodingConflictResolver:
        try:
            return unicode(text)
        except UnicodeDecodeError:
-            encoding = getattr(context, 'managment_page_charset', default_encoding)
+            encoding = getattr(context, 'management_page_charset', default_encoding)
            return unicode(text, encoding, self.mode)

+class PreferredCharsetResolver:
+    """ A resolver that tries use the encoding information
+        from the HTTP_ACCEPT_CHARSET header.
+    """
+
+    implements(IUnicodeEncodingConflictResolver)
+
+    def resolve(self, context, text, expression):
+
+        request = context.REQUEST
+
+        charsets = getattr(request, '__zpt_available_charsets', None)
+        if charsets is None:
+            charsets = IUserPreferredCharsets(request).getPreferredCharsets()
+
+            # add management_page_charset as one fallback
+            management_charset = getattr(context, 'management_page_charset', None)
+            if management_charset:
+                charsets.append(management_charset)
+
+            # add Python's default encoding as last fallback
+            charsets.append(default_encoding)               
+
+            # cache list of charsets
+            request.__zpt_available_charsets = charsets
+
+        for enc in charsets:
+            if enc == '*': continue
+
+            try:
+                return unicode(text, enc)
+            except UnicodeDecodeError:
+                pass
+
+        return text
+

 StrictUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('strict')
 ReplacingUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('replace')
 IgnoringUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('ignore')
+PreferredCharsetResolver = PreferredCharsetResolver()