Commit e6d702c5 authored by Andreas Jung's avatar Andreas Jung

- in-place migration to unicode

- the ZMI edit view now *always* uses 'utf-8' and no longer uses self.output_encoding (which is only relevant for FTP/WebDAV)
- 'output_encoding' is now a property
parent c161fc80
...@@ -40,7 +40,7 @@ from Products.PageTemplates.PageTemplateFile import PageTemplateFile ...@@ -40,7 +40,7 @@ from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from Products.PageTemplates.PageTemplateFile import guess_type from Products.PageTemplates.PageTemplateFile import guess_type
from Products.PageTemplates.Expressions import SecureModuleImporter from Products.PageTemplates.Expressions import SecureModuleImporter
from Products.PageTemplates.utils import encodingFromXMLPreamble, charsetFromMetaEquiv from Products.PageTemplates.utils import encodingFromXMLPreamble, charsetFromMetaEquiv, convertToUnicode
preferred_encodings = ['utf-8', 'iso-8859-15'] preferred_encodings = ['utf-8', 'iso-8859-15']
...@@ -70,6 +70,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable, ...@@ -70,6 +70,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
__implements__ = (WriteLockInterface,) __implements__ = (WriteLockInterface,)
meta_type = 'Page Template' meta_type = 'Page Template'
output_encoding = 'iso-8859-15' # provide default for old instances
func_defaults = None func_defaults = None
func_code = FuncCode((), 0) func_code = FuncCode((), 0)
...@@ -90,6 +91,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable, ...@@ -90,6 +91,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
_properties=({'id':'title', 'type': 'ustring', 'mode': 'w'}, _properties=({'id':'title', 'type': 'ustring', 'mode': 'w'},
{'id':'content_type', 'type':'string', 'mode': 'w'}, {'id':'content_type', 'type':'string', 'mode': 'w'},
{'id':'output_encoding', 'type':'string', 'mode': 'w'},
{'id':'expand', 'type':'boolean', 'mode': 'w'}, {'id':'expand', 'type':'boolean', 'mode': 'w'},
) )
...@@ -183,7 +185,12 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable, ...@@ -183,7 +185,12 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
raise ResourceLockedError("File is locked via WebDAV") raise ResourceLockedError("File is locked via WebDAV")
self.expand = expand self.expand = expand
self.pt_setTitle(title, self.output_encoding)
# The ZMI edit view uses utf-8! So we can safely assume
# that 'title' and 'text' are utf-8 encoded strings - hopefully
self.pt_setTitle(title, 'utf-8')
text = unicode(text, 'utf-8')
self.pt_edit(text, content_type, True) self.pt_edit(text, content_type, True)
REQUEST.set('text', self.read()) # May not equal 'text'! REQUEST.set('text', self.read()) # May not equal 'text'!
...@@ -194,6 +201,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable, ...@@ -194,6 +201,7 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
% '<br>'.join(self._v_warnings)) % '<br>'.join(self._v_warnings))
return self.pt_editForm(manage_tabs_message=message) return self.pt_editForm(manage_tabs_message=message)
security.declareProtected(change_page_templates, 'pt_setTitle') security.declareProtected(change_page_templates, 'pt_setTitle')
def pt_setTitle(self, title, encoding='utf-8'): def pt_setTitle(self, title, encoding='utf-8'):
if not isinstance(title, unicode): if not isinstance(title, unicode):
...@@ -394,6 +402,17 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable, ...@@ -394,6 +402,17 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
# acquisition context, so we don't know where it is. :-( # acquisition context, so we don't know where it is. :-(
return None return None
def __setstate__(self, state):
# Perform on-the-fly migration to unicode.
# Perhaps it might be work with the 'generation' module here?
if not isinstance(state['_text'], unicode):
text, encoding = convertToUnicode(state['_text'], state['content_type'])
state['_text'] = text
state['output_encoding'] = encoding
self.__dict__.update(state)
def pt_render(self, source=False, extra_context={}): def pt_render(self, source=False, extra_context={}):
result = PageTemplate.pt_render(self, source, extra_context) result = PageTemplate.pt_render(self, source, extra_context)
assert isinstance(result, unicode) assert isinstance(result, unicode)
......
...@@ -56,4 +56,34 @@ def charsetFromMetaEquiv(html): ...@@ -56,4 +56,34 @@ def charsetFromMetaEquiv(html):
return None return None
def convertToUnicode(source, content_type):
""" Convert 'source' to unicode.
Returns (unicode_str, source_encoding).
"""
if content_type.startswith('text/xml'):
encoding = encodingFromXMLPreamble(source)
return unicode(source, encoding), encoding
elif content_type.startswith('text/html'):
encoding = charsetFromMetaEquiv(source)
# Try to detect the encoding by converting it unicode without raising
# exceptions. There are some smarter Python-based sniffer methods
# available however we have to check their licenses first before
# including them into the Zope 2 core
if not encoding:
for enc in ('utf-8', 'iso-8859-15'):
try:
return unicode(source, enc), enc
except UnicodeDecodeError:
continue
raise TypeError('Could not auto-detect encoding')
else:
raise ValueError('Unsupported content-type: %s' % content_type)
<h1 tal:replace="structure python:context.manage_page_header(management_page_charset=context.output_encoding)">Header</h1> <h1 tal:replace="structure python:context.manage_page_header(management_page_charset='utf-8')">Header</h1>
<h2 tal:define="manage_tabs_message options/manage_tabs_message | nothing" <h2 tal:define="manage_tabs_message options/manage_tabs_message | nothing"
tal:replace="structure context/manage_tabs">Tabs</h2> tal:replace="structure context/manage_tabs">Tabs</h2>
...@@ -44,19 +44,7 @@ ...@@ -44,19 +44,7 @@
</td> </td>
</tr> </tr>
<tr>
<td align="left" valign="middle">
<div class="form-label">Output encoding</div>
</td>
<td align="left" valign="middle">
<div class="form-text"
tal:content="context/output_encoding"
/>
</td>
</tr>
<tr tal:define="errors context/pt_errors" tal:condition="errors"> <tr tal:define="errors context/pt_errors" tal:condition="errors">
<tal:block define="global body python:context.document_src({'raw':1})" />
<td align="left" valign="middle" class="form-label">Errors</td> <td align="left" valign="middle" class="form-label">Errors</td>
<td align="left" valign="middle" style="background-color: #FFDDDD" <td align="left" valign="middle" style="background-color: #FFDDDD"
colspan="3"> colspan="3">
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment