Skip to content

Z
Zope
Commit cd36f0e3 authored by Andreas Jung's avatar Andreas Jung
Browse files
Options

Launchpad #254570: 

the PreferredCharset resolver now deals with situations
where 'context' has no REQUEST attribute
parent 736bc77c
Show whitespace changes
Inline Side-by-side
Showing with 151 additions and 41 deletions
lib/python/Products/PageTemplates/tests/testZopePageTemplate.py
View file @ cd36f0e3
...@@ -18,11 +18,14 @@ from zope.publisher.http import HTTPCharsets ...@@ -18,11 +18,14 @@ from zope.publisher.http import HTTPCharsets
from Testing.makerequest import makerequest from Testing.makerequest import makerequest
from Testing.ZopeTestCase import ZopeTestCase, installProduct from Testing.ZopeTestCase import ZopeTestCase, installProduct
from Products.PageTemplates.ZopePageTemplate import ZopePageTemplate, manage_addPageTemplate from Products.PageTemplates.ZopePageTemplate import ZopePageTemplate
from Products.PageTemplates.utils import encodingFromXMLPreamble, charsetFromMetaEquiv from Products.PageTemplates.ZopePageTemplate import manage_addPageTemplate
from Products.PageTemplates.utils import encodingFromXMLPreamble
from Products.PageTemplates.utils import charsetFromMetaEquiv
from zope.component import provideUtility from zope.component import provideUtility
from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
from Products.PageTemplates.unicodeconflictresolver import PreferredCharsetResolver from Products.PageTemplates.unicodeconflictresolver \
import PreferredCharsetResolver
ascii_str = '<html><body>hello world</body></html>' ascii_str = '<html><body>hello world</body></html>'
...@@ -50,7 +53,8 @@ html_template_w_header = ''' ...@@ -50,7 +53,8 @@ html_template_w_header = '''
''' '''
html_iso_8859_15_w_header = html_template_w_header % 'iso-8859-15' html_iso_8859_15_w_header = html_template_w_header % 'iso-8859-15'
html_utf8_w_header = unicode(html_template_w_header, 'iso-8859-15').encode('utf-8') % 'utf-8' html_utf8_w_header = unicode(html_template_w_header,
'iso-8859-15').encode('utf-8') % 'utf-8'
html_template_wo_header = ''' html_template_wo_header = '''
<html> <html>
...@@ -61,7 +65,21 @@ html_template_wo_header = ''' ...@@ -61,7 +65,21 @@ html_template_wo_header = '''
''' '''
html_iso_8859_15_wo_header = html_template_wo_header html_iso_8859_15_wo_header = html_template_wo_header
html_utf8_wo_header = unicode(html_template_wo_header, 'iso-8859-15').encode('utf-8') html_utf8_wo_header = unicode(html_template_wo_header,
'iso-8859-15').encode('utf-8')
xml_with_upper_attr = '''<?xml version="1.0"?>
<foo>
<bar ATTR="1" />
</foo>
'''
html_with_upper_attr = '''<html><body>
<foo>
<bar ATTR="1" />
</foo>
</body></html>
'''
installProduct('PageTemplates') installProduct('PageTemplates')
...@@ -71,16 +89,32 @@ class ZPTUtilsTests(unittest.TestCase): ...@@ -71,16 +89,32 @@ class ZPTUtilsTests(unittest.TestCase):
def testExtractEncodingFromXMLPreamble(self): def testExtractEncodingFromXMLPreamble(self):
extract = encodingFromXMLPreamble extract = encodingFromXMLPreamble
self.assertEqual(extract('<?xml version="1.0" ?>'), 'utf-8') self.assertEqual(extract('<?xml version="1.0" ?>'), 'utf-8')
self.assertEqual(extract('<?xml encoding="utf-8" version="1.0" ?>'), 'utf-8') self.assertEqual(extract('<?xml encoding="utf-8" '
self.assertEqual(extract('<?xml encoding="UTF-8" version="1.0" ?>'), 'utf-8') 'version="1.0" ?>'),
self.assertEqual(extract('<?xml encoding="ISO-8859-15" version="1.0" ?>'), 'iso-8859-15') 'utf-8')
self.assertEqual(extract('<?xml encoding="iso-8859-15" version="1.0" ?>'), 'iso-8859-15') self.assertEqual(extract('<?xml encoding="UTF-8" '
'version="1.0" ?>'),
'utf-8')
self.assertEqual(extract('<?xml encoding="ISO-8859-15" '
'version="1.0" ?>'),
'iso-8859-15')
self.assertEqual(extract('<?xml encoding="iso-8859-15" '
'version="1.0" ?>'),
'iso-8859-15')
def testExtractCharsetFromMetaHTTPEquivTag(self): def testExtractCharsetFromMetaHTTPEquivTag(self):
extract = charsetFromMetaEquiv extract = charsetFromMetaEquiv
self.assertEqual(extract('<html><META http-equiv="content-type" content="text/html; charset=UTF-8"></html>'), 'utf-8') self.assertEqual(extract('<html><META http-equiv="content-type" '
self.assertEqual(extract('<html><META http-equiv="content-type" content="text/html; charset=iso-8859-15"></html>'), 'iso-8859-15') 'content="text/html; '
self.assertEqual(extract('<html><META http-equiv="content-type" content="text/html"></html>'), None) 'charset=UTF-8"></html>'),
'utf-8')
self.assertEqual(extract('<html><META http-equiv="content-type" '
'content="text/html; '
'charset=iso-8859-15"></html>'),
'iso-8859-15')
self.assertEqual(extract('<html><META http-equiv="content-type" '
'content="text/html"></html>'),
None)
self.assertEqual(extract('<html>...<html>'), None) self.assertEqual(extract('<html>...<html>'), None)
...@@ -89,57 +123,73 @@ class ZPTUnicodeEncodingConflictResolution(ZopeTestCase): ...@@ -89,57 +123,73 @@ class ZPTUnicodeEncodingConflictResolution(ZopeTestCase):
def afterSetUp(self): def afterSetUp(self):
zope.component.provideAdapter(DefaultTraversable, (None,)) zope.component.provideAdapter(DefaultTraversable, (None,))
zope.component.provideAdapter(HTTPCharsets, (None,)) zope.component.provideAdapter(HTTPCharsets, (None,))
provideUtility(PreferredCharsetResolver, IUnicodeEncodingConflictResolver) provideUtility(PreferredCharsetResolver,
IUnicodeEncodingConflictResolver)
def testISO_8859_15(self): def testISO_8859_15(self):
manage_addPageTemplate(self.app, 'test', manage_addPageTemplate(self.app, 'test',
text='<div tal:content="python: request.get(\'data\')" />', text=('<div tal:content="python: '
'request.get(\'data\')" />'),
encoding='ascii') encoding='ascii')
zpt = self.app['test'] zpt = self.app['test']
self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'ISO-8859-15,utf-8') self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'ISO-8859-15,utf-8')
self.app.REQUEST.set('data', '') self.app.REQUEST.set('data', '')
result = zpt.pt_render() result = zpt.pt_render()
self.assertEqual(result.startswith(unicode('<div></div>', 'iso-8859-15')), True) self.failUnless(result.startswith(unicode('<div></div>',
'iso-8859-15')))
def testUTF8(self): def testUTF8(self):
manage_addPageTemplate(self.app, 'test', manage_addPageTemplate(self.app, 'test',
text='<div tal:content="python: request.get(\'data\')" />', text=('<div tal:content="python: '
'request.get(\'data\')" />'),
encoding='ascii') encoding='ascii')
zpt = self.app['test'] zpt = self.app['test']
self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'utf-8,ISO-8859-15') self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'utf-8,ISO-8859-15')
self.app.REQUEST.set('data', unicode('', 'iso-8859-15').encode('utf-8')) self.app.REQUEST.set('data',
unicode('', 'iso-8859-15').encode('utf-8'))
result = zpt.pt_render() result = zpt.pt_render()
self.assertEqual(result.startswith(unicode('<div></div>', 'iso-8859-15')), True) self.failUnless(result.startswith(unicode('<div></div>',
'iso-8859-15')))
def testUTF8WrongPreferredCharset(self): def testUTF8WrongPreferredCharset(self):
manage_addPageTemplate(self.app, 'test', manage_addPageTemplate(self.app, 'test',
text='<div tal:content="python: request.get(\'data\')" />', text=('<div tal:content="python: '
'request.get(\'data\')" />'),
encoding='ascii') encoding='ascii')
zpt = self.app['test'] zpt = self.app['test']
self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'iso-8859-15') self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'iso-8859-15')
self.app.REQUEST.set('data', unicode('', 'iso-8859-15').encode('utf-8')) self.app.REQUEST.set('data',
unicode('', 'iso-8859-15').encode('utf-8'))
result = zpt.pt_render() result = zpt.pt_render()
self.assertEqual(result.startswith(unicode('<div></div>', 'iso-8859-15')), False) self.failIf(result.startswith(unicode('<div></div>',
'iso-8859-15')))
def testStructureWithAccentedChars(self): def testStructureWithAccentedChars(self):
manage_addPageTemplate(self.app, 'test', manage_addPageTemplate(self.app, 'test',
text='<div tal:content="structure python: %s" />' % "''", text=('<div tal:content="structure '
'python: %s" />' % "''"),
encoding='iso-8859-15') encoding='iso-8859-15')
zpt = self.app['test'] zpt = self.app['test']
self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'iso-8859-15,utf-8') self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'iso-8859-15,utf-8')
self.app.REQUEST.set('data', unicode('', 'iso-8859-15').encode('utf-8')) self.app.REQUEST.set('data', unicode('',
'iso-8859-15').encode('utf-8'))
result = zpt.pt_render() result = zpt.pt_render()
self.assertEqual(result.startswith(unicode('<div></div>', 'iso-8859-15')), True) self.failUnless(result.startswith(unicode('<div></div>',
'iso-8859-15')))
def testBug151020(self): def testBug151020(self):
manage_addPageTemplate(self.app, 'test', manage_addPageTemplate(self.app, 'test',
text='<div tal:content="structure python: %s" />' % "''", text=('<div tal:content="structure '
'python: %s" />' % "''"),
encoding='iso-8859-15') encoding='iso-8859-15')
zpt = self.app['test'] zpt = self.app['test']
self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'x-user-defined, iso-8859-15,utf-8') self.app.REQUEST.set('HTTP_ACCEPT_CHARSET',
self.app.REQUEST.set('data', unicode('', 'iso-8859-15').encode('utf-8')) 'x-user-defined, iso-8859-15,utf-8')
self.app.REQUEST.set('data',
unicode('', 'iso-8859-15').encode('utf-8'))
result = zpt.pt_render() result = zpt.pt_render()
self.assertEqual(result.startswith(unicode('<div></div>', 'iso-8859-15')), True) self.failUnless(result.startswith(unicode('<div></div>',
'iso-8859-15')))
def test_bug_198274(self): def test_bug_198274(self):
# See https://bugs.launchpad.net/bugs/198274 # See https://bugs.launchpad.net/bugs/198274
...@@ -168,8 +218,14 @@ class ZPTUnicodeEncodingConflictResolution(ZopeTestCase): ...@@ -168,8 +218,14 @@ class ZPTUnicodeEncodingConflictResolution(ZopeTestCase):
class ZopePageTemplateFileTests(ZopeTestCase): class ZopePageTemplateFileTests(ZopeTestCase):
def test_class_conforms_to_IWriteLock(self):
from zope.interface.verify import verifyClass
from webdav.interfaces import IWriteLock
verifyClass(IWriteLock, ZopePageTemplate)
def testPT_RenderWithAscii(self): def testPT_RenderWithAscii(self):
manage_addPageTemplate(self.app, 'test', text=ascii_str, encoding='ascii') manage_addPageTemplate(self.app, 'test',
text=ascii_str, encoding='ascii')
zpt = self.app['test'] zpt = self.app['test']
result = zpt.pt_render() result = zpt.pt_render()
# use startswith() because the renderer appends a trailing \n # use startswith() because the renderer appends a trailing \n
...@@ -180,21 +236,25 @@ class ZopePageTemplateFileTests(ZopeTestCase): ...@@ -180,21 +236,25 @@ class ZopePageTemplateFileTests(ZopeTestCase):
# Check workaround for unicode incompatibility of ZRPythonExpr. # Check workaround for unicode incompatibility of ZRPythonExpr.
# See http://mail.zope.org/pipermail/zope/2007-February/170537.html # See http://mail.zope.org/pipermail/zope/2007-February/170537.html
manage_addPageTemplate(self.app, 'test', manage_addPageTemplate(self.app, 'test',
text='<span tal:content="python: unicode(\'\xfe\', \'iso-8859-15\')" />', text=('<span tal:content="python: '
'unicode(\'\xfe\', \'iso-8859-15\')" />'),
encoding='iso-8859-15') encoding='iso-8859-15')
zpt = self.app['test'] zpt = self.app['test']
result = zpt.pt_render() # should not raise a UnicodeDecodeError result = zpt.pt_render() # should not raise a UnicodeDecodeError
def testPT_RenderWithISO885915(self): def testPT_RenderWithISO885915(self):
manage_addPageTemplate(self.app, 'test', text=iso885915_str, encoding='iso-8859-15') manage_addPageTemplate(self.app, 'test',
text=iso885915_str, encoding='iso-8859-15')
zpt = self.app['test'] zpt = self.app['test']
result = zpt.pt_render() result = zpt.pt_render()
# use startswith() because the renderer appends a trailing \n # use startswith() because the renderer appends a trailing \n
self.assertEqual(result.encode('iso-8859-15').startswith(iso885915_str), True) self.failUnless(result.encode('iso-8859-15'
).startswith(iso885915_str))
self.assertEqual(zpt.output_encoding, 'iso-8859-15') self.assertEqual(zpt.output_encoding, 'iso-8859-15')
def testPT_RenderWithUTF8(self): def testPT_RenderWithUTF8(self):
manage_addPageTemplate(self.app, 'test', text=utf8_str, encoding='utf-8') manage_addPageTemplate(self.app, 'test',
text=utf8_str, encoding='utf-8')
zpt = self.app['test'] zpt = self.app['test']
result = zpt.pt_render() result = zpt.pt_render()
# use startswith() because the renderer appends a trailing \n # use startswith() because the renderer appends a trailing \n
...@@ -218,7 +278,8 @@ class ZopePageTemplateFileTests(ZopeTestCase): ...@@ -218,7 +278,8 @@ class ZopePageTemplateFileTests(ZopeTestCase):
self.assertEqual(zpt.read(), xml_unicode) self.assertEqual(zpt.read(), xml_unicode)
def _createZPT(self): def _createZPT(self):
manage_addPageTemplate(self.app, 'test', text=utf8_str, encoding='utf-8') manage_addPageTemplate(self.app, 'test',
text=utf8_str, encoding='utf-8')
zpt = self.app['test'] zpt = self.app['test']
return zpt return zpt
...@@ -266,6 +327,36 @@ class ZopePageTemplateFileTests(ZopeTestCase): ...@@ -266,6 +327,36 @@ class ZopePageTemplateFileTests(ZopeTestCase):
self.assertEqual(zpt.content_type, 'text/xml') self.assertEqual(zpt.content_type, 'text/xml')
result = zpt.pt_render() # should not raise an exception result = zpt.pt_render() # should not raise an exception
def testXMLAttrsMustNotBeLowercased(self):
zpt = self._put(xml_with_upper_attr)
self.assertEqual(zpt.content_type, 'text/xml')
result = zpt.pt_render()
self.assertEqual('ATTR' in result, True)
def testHTMLAttrsAreLowerCased(self):
zpt = self._put(html_with_upper_attr)
self.content_type = 'text/html'
result = zpt.pt_render()
self.assertEqual('ATTR' in result, False)
class PreferredCharsetUnicodeResolverTests(unittest.TestCase):
def testPreferredCharsetResolverWithoutRequestAndWithoutEncoding(self):
# This test checks the edgecase where the unicode conflict resolver
# is called with a context object having no REQUEST
context = object()
result = PreferredCharsetResolver.resolve(context, '', None)
self.assertEqual(result, '')
def testPreferredCharsetResolverWithoutRequestAndWithEncoding(self):
# This test checks the edgecase where the unicode conflict resolver
# is called with a context object having no REQUEST
class ContextMock:
management_page_charset = 'iso-8859-15'
result = PreferredCharsetResolver.resolve(ContextMock(), '', None)
self.assertEqual(result, u'')
class ZPTRegressions(unittest.TestCase): class ZPTRegressions(unittest.TestCase):
def setUp(self): def setUp(self):
...@@ -389,6 +480,7 @@ def test_suite(): ...@@ -389,6 +480,7 @@ def test_suite():
suite.addTests(unittest.makeSuite(ZPTMacros)) suite.addTests(unittest.makeSuite(ZPTMacros))
suite.addTests(unittest.makeSuite(ZopePageTemplateFileTests)) suite.addTests(unittest.makeSuite(ZopePageTemplateFileTests))
suite.addTests(unittest.makeSuite(ZPTUnicodeEncodingConflictResolution)) suite.addTests(unittest.makeSuite(ZPTUnicodeEncodingConflictResolution))
suite.addTests(unittest.makeSuite(PreferredCharsetUnicodeResolverTests))
return suite return suite
if __name__ == '__main__': if __name__ == '__main__':
......
lib/python/Products/PageTemplates/unicodeconflictresolver.py
View file @ cd36f0e3
...@@ -62,25 +62,43 @@ class PreferredCharsetResolver: ...@@ -62,25 +62,43 @@ class PreferredCharsetResolver:
def resolve(self, context, text, expression): def resolve(self, context, text, expression):
request = context.REQUEST request = getattr(context, 'REQUEST', None)
charsets = getattr(request, '__zpt_available_charsets', None) # Deal with the fact that a REQUEST is not always available.
if charsets is None: # In this case fall back to the encoding of the ZMI and the
charsets = IUserPreferredCharsets(request).getPreferredCharsets() # Python default encoding.
# add management_page_charset as one fallback if request is None:
charsets = [default_encoding]
management_charset = getattr(context, 'management_page_charset', None) management_charset = getattr(context, 'management_page_charset', None)
if management_charset: if management_charset:
charsets.append(management_charset) charsets.insert(0, management_charset)
else:
# charsets might by cached within the request
charsets = getattr(request, '__zpt_available_charsets', None)
# No uncached charsets found: investigate the HTTP_ACCEPT_CHARSET
# header. This code is only called if 'context' has a request
# object. The condition is true because otherwise 'charsets' contains
# at least the default encoding of Python.
if charsets is None:
charsets = list()
# add Python's default encoding as last fallback # add Python's default encoding as last fallback
charsets.append(default_encoding) charsets.append(default_encoding)
# include the charsets based on the HTTP_ACCEPT_CHARSET
# header
charsets = IUserPreferredCharsets(request).getPreferredCharsets() +\
charsets
# cache list of charsets # cache list of charsets
request.__zpt_available_charsets = charsets request.__zpt_available_charsets = charsets
for enc in charsets: for enc in charsets:
if enc == '*': continue if enc == '*':
continue
try: try:
return unicode(text, enc) return unicode(text, enc)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7