Commit 702c4880 authored by 's avatar

merged yuppie-tal-backports branch:

- made sure custom strings like MessageID are not converted by ustr
- synced TALInterpreter code and tests with zope.tal where possible without changing behavior
- backported MessageID support from zope.tal
parent 1bd27dd7
......@@ -26,6 +26,11 @@ Zope Changes
Bugs Fixed
- TAL: MassageIDs are now handled the same way as in zope.tal.
- DocumentTemplate: ustr no longer mangles MassageIDs.
Custom string types are now returned unchanged.
- As developed in a long thread starting at
http://mail.zope.org/pipermail/zope/2005-July/160433.html
there appears to be a race bug in the Microsoft Windows socket
......
......@@ -7,20 +7,18 @@
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Document Template Tests
"""
"""ustr unit tests.
__rcs_id__='$Id$'
__version__='$Revision: 1.3 $'[11:-2]
$Id$
"""
import sys, os
import unittest
from DocumentTemplate.ustr import ustr
from ExtensionClass import Base
class force_str:
# A class whose string representation is not always a plain string:
......@@ -29,7 +27,18 @@ class force_str:
def __str__(self):
return self.s
class UnicodeTests (unittest.TestCase):
class Foo(str):
pass
class Bar(unicode):
pass
class UnicodeTests(unittest.TestCase):
def testPlain(self):
a = ustr('hello')
......@@ -59,13 +68,17 @@ class UnicodeTests (unittest.TestCase):
a = ustr(ValueError(unichr(200)))
assert a==unichr(200), `a`
def testCustomStrings(self):
a = ustr(Foo('foo'))
self.failUnlessEqual(type(a), Foo)
a = ustr(Bar('bar'))
self.failUnlessEqual(type(a), Bar)
def test_suite():
suite = unittest.TestSuite()
suite.addTest( unittest.makeSuite( UnicodeTests ) )
return suite
def main():
unittest.TextTestRunner().run(test_suite())
if __name__ == '__main__':
main()
unittest.main(defaultTest='test_suite')
......@@ -7,11 +7,13 @@
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""ustr function.
from types import StringType, UnicodeType, InstanceType
$Id$
"""
nasty_exception_str = Exception.__str__.im_func
......@@ -20,8 +22,7 @@ def ustr(v):
minimising the chance of raising a UnicodeError. This
even works with uncooperative objects like Exceptions
"""
string_types = (StringType,UnicodeType)
if type(v) in string_types:
if isinstance(v, basestring):
return v
else:
fn = getattr(v,'__str__',None)
......@@ -41,7 +42,7 @@ def ustr(v):
else:
# Trust the object to do this right
v = fn()
if type(v) in string_types:
if isinstance(v, basestring):
return v
else:
raise ValueError('__str__ returned wrong type')
......@@ -59,4 +60,3 @@ def _exception_str(exc):
else:
return str(exc.args)
return str(exc)
......@@ -4,29 +4,32 @@
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""
Interpreter for a pre-compiled TAL program.
"""Interpreter for a pre-compiled TAL program.
$Id$
"""
import cgi
import sys
import getopt
import re
from types import ListType
from cgi import escape
# Do not use cStringIO here! It's not unicode aware. :(
from StringIO import StringIO
from DocumentTemplate.DT_Util import ustr
from ZODB.POSException import ConflictError
from TALDefs import TAL_VERSION, TALError, METALError, attrEscape
from TALDefs import isCurrentVersion, getProgramVersion, getProgramMode
from zope.i18nmessageid import MessageID
from TALDefs import attrEscape, TAL_VERSION, METALError
from TALDefs import isCurrentVersion
from TALDefs import getProgramVersion, getProgramMode
from TALGenerator import TALGenerator
from TranslationContext import TranslationContext
......@@ -41,11 +44,22 @@ BOOLEAN_HTML_ATTRS = [
"defer"
]
def _init():
d = {}
for s in BOOLEAN_HTML_ATTRS:
d[s] = 1
return d
BOOLEAN_HTML_ATTRS = _init()
_nulljoin = ''.join
_spacejoin = ' '.join
def normalize(text):
# Now we need to normalize the whitespace in implicit message ids and
# implicit $name substitution values by stripping leading and trailing
# whitespace, and folding all internal whitespace to a single space.
return ' '.join(text.split())
return _spacejoin(text.split())
NAME_RE = r"[a-zA-Z][a-zA-Z0-9_]*"
......@@ -159,7 +173,8 @@ class TALInterpreter:
self.scopeLevel, self.level, self.i18nContext)
def restoreState(self, state):
(self.position, self.col, self.stream, scopeLevel, level, i18n) = state
(self.position, self.col, self.stream,
scopeLevel, level, i18n) = state
self._stream_write = self.stream.write
assert self.level == level
while self.scopeLevel > scopeLevel:
......@@ -169,7 +184,8 @@ class TALInterpreter:
self.i18nContext = i18n
def restoreOutputState(self, state):
(dummy, self.col, self.stream, scopeLevel, level, i18n) = state
(dummy, self.col, self.stream,
scopeLevel, level, i18n) = state
self._stream_write = self.stream.write
assert self.level == level
assert self.scopeLevel == scopeLevel
......@@ -195,6 +211,16 @@ class TALInterpreter:
self._stream_write("\n")
self.col = 0
def interpretWithStream(self, program, stream):
oldstream = self.stream
self.stream = stream
self._stream_write = stream.write
try:
self.interpret(program)
finally:
self.stream = oldstream
self._stream_write = oldstream.write
def stream_write(self, s,
len=len):
self._stream_write(s)
......@@ -206,16 +232,6 @@ class TALInterpreter:
bytecode_handlers = {}
def interpretWithStream(self, program, stream):
oldstream = self.stream
self.stream = stream
self._stream_write = stream.write
try:
self.interpret(program)
finally:
self.stream = oldstream
self._stream_write = oldstream.write
def interpret(self, program):
oldlevel = self.level
self.level = oldlevel + 1
......@@ -269,6 +285,7 @@ class TALInterpreter:
# for start tags with no attributes; those are optimized down
# to rawtext events. Hence, there is no special "fast path"
# for that case.
self._currentTag = name
L = ["<", name]
append = L.append
col = self.col + _len(name) + 1
......@@ -303,9 +320,9 @@ class TALInterpreter:
col = col + 1 + slen
append(s)
append(end)
self._stream_write("".join(L))
col = col + endlen
finally:
self._stream_write(_nulljoin(L))
self.col = col
bytecode_handlers["startTag"] = do_startTag
......@@ -487,6 +504,9 @@ class TALInterpreter:
if text is self.Default:
self.interpret(stuff[1])
return
if isinstance(text, MessageID):
# Translate this now.
text = self.engine.translate(text.domain, text, text.mapping)
s = escape(text)
self._stream_write(s)
i = s.rfind('\n')
......@@ -505,7 +525,10 @@ class TALInterpreter:
try:
tmpstream = self.StringIO()
self.interpretWithStream(program, tmpstream)
value = normalize(tmpstream.getvalue())
if self.html and self._currentTag == "pre":
value = tmpstream.getvalue()
else:
value = normalize(tmpstream.getvalue())
finally:
self.restoreState(state)
else:
......@@ -516,8 +539,14 @@ class TALInterpreter:
else:
value = self.engine.evaluate(expression)
# evaluate() does not do any I18n, so we do it here.
if isinstance(value, MessageID):
# Translate this now.
value = self.engine.translate(value.domain, value,
value.mapping)
if not structure:
value = cgi.escape(str(value))
value = cgi.escape(ustr(value))
# Either the i18n:name tag is nested inside an i18n:translate in which
# case the last item on the stack has the i18n dictionary and string
......@@ -545,20 +574,29 @@ class TALInterpreter:
#
# Use a temporary stream to capture the interpretation of the
# subnodes, which should /not/ go to the output stream.
currentTag = self._currentTag
tmpstream = self.StringIO()
self.interpretWithStream(stuff[1], tmpstream)
default = tmpstream.getvalue()
# We only care about the evaluated contents if we need an implicit
# message id. All other useful information will be in the i18ndict on
# the top of the i18nStack.
if msgid == '':
msgid = normalize(default)
default = tmpstream.getvalue()
if not msgid:
if self.html and currentTag == "pre":
msgid = default
else:
msgid = normalize(default)
self.i18nStack.pop()
# See if there is was an i18n:data for msgid
if len(stuff) > 2:
obj = self.engine.evaluate(stuff[2])
xlated_msgid = self.translate(msgid, default, i18ndict, obj)
assert xlated_msgid is not None, self.position
# XXX I can't decide whether we want to cgi escape the translated
# string or not. OT1H not doing this could introduce a cross-site
# scripting vector by allowing translators to sneak JavaScript into
# translations. OTOH, for implicit interpolation values, we don't
# want to escape stuff like ${name} <= "<b>Timmy</b>".
assert xlated_msgid is not None
self._stream_write(xlated_msgid)
bytecode_handlers['insertTranslation'] = do_insertTranslation
......
# -*- coding: ISO-8859-1 -*-
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Tests for TALInterpreter."""
"""Tests for TALInterpreter.
$Id$
"""
import sys
from TAL.tests import utils
import unittest
from StringIO import StringIO
from TAL.TALDefs import METALError, I18NError
from TAL.HTMLTALParser import HTMLTALParser
from TAL.TALParser import TALParser
from TAL.TALInterpreter import TALInterpreter
from TAL.DummyEngine import DummyEngine, DummyTranslationService
from TAL.TALInterpreter import interpolate
from TAL.tests import utils
from zope.i18nmessageid import MessageID
class TestCaseBase(unittest.TestCase):
......@@ -53,10 +56,10 @@ class MacroErrorsTestCase(TestCaseBase):
else:
self.fail("Expected METALError")
def check_mode_error(self):
def test_mode_error(self):
self.macro[1] = ("mode", "duh")
def check_version_error(self):
def test_version_error(self):
self.macro[0] = ("version", "duh")
......@@ -64,7 +67,9 @@ class I18NCornerTestCase(TestCaseBase):
def setUp(self):
self.engine = DummyEngine()
self.engine.setLocal('foo', MessageID('FoOvAlUe', 'default'))
self.engine.setLocal('bar', 'BaRvAlUe')
self.engine.setLocal('raw', ' \tRaW\n ')
def _check(self, program, expected):
result = StringIO()
......@@ -73,11 +78,57 @@ class I18NCornerTestCase(TestCaseBase):
self.interpreter()
self.assertEqual(expected, result.getvalue())
def test_simple_messageid_translate(self):
# This test is mainly here to make sure our DummyEngine works
# correctly.
program, macros = self._compile('<span tal:content="foo"/>')
self._check(program, '<span>FOOVALUE</span>\n')
program, macros = self._compile('<span tal:replace="foo"/>')
self._check(program, 'FOOVALUE\n')
def test_replace_with_messageid_and_i18nname(self):
program, macros = self._compile(
'<div i18n:translate="" >'
'<span tal:replace="foo" i18n:name="foo_name"/>'
'</div>')
self._check(program, '<div>FOOVALUE</div>\n')
def test_pythonexpr_replace_with_messageid_and_i18nname(self):
program, macros = self._compile(
'<div i18n:translate="" >'
'<span tal:replace="python: foo" i18n:name="foo_name"/>'
'</div>')
self._check(program, '<div>FOOVALUE</div>\n')
def test_structure_replace_with_messageid_and_i18nname(self):
program, macros = self._compile(
'<div i18n:translate="" >'
'<span tal:replace="structure foo" i18n:name="foo_name"/>'
'</div>')
self._check(program, '<div>FOOVALUE</div>\n')
def test_complex_replace_with_messageid_and_i18nname(self):
program, macros = self._compile(
'<div i18n:translate="" >'
'<em i18n:name="foo_name">'
'<span tal:replace="foo"/>'
'</em>'
'</div>')
self._check(program, '<div>FOOVALUE</div>\n')
def test_content_with_messageid_and_i18nname(self):
program, macros = self._compile(
'<div i18n:translate="" >'
'<span tal:content="foo" i18n:name="foo_name"/>'
'</div>')
self._check(program, '<div><span>FOOVALUE</span></div>\n')
def test_content_with_messageid_and_i18nname_and_i18ntranslate(self):
# Let's tell the user this is incredibly silly!
self.assertRaises(
I18NError, self._compile,
'<span i18n:translate="" tal:content="bar" i18n:name="bar_name"/>')
'<span i18n:translate="" tal:content="foo" i18n:name="foo_name"/>')
def test_content_with_plaintext_and_i18nname_and_i18ntranslate(self):
# Let's tell the user this is incredibly silly!
......@@ -126,20 +177,23 @@ class I18NCornerTestCase(TestCaseBase):
self._check(program,
'<div>THIS IS TEXT FOR <span>BARVALUE</span>.</div>\n')
def test_for_correct_msgids(self):
def _getCollectingTranslationDomain(self):
class CollectingTranslationService(DummyTranslationService):
data = []
def translate(self, domain, msgid, mapping=None,
context=None, target_language=None, default=None):
self.data.append(msgid)
self.data.append((msgid, mapping))
return DummyTranslationService.translate(
self,
domain, msgid, mapping, context, target_language, default)
xlatsvc = CollectingTranslationService()
self.engine.translationService = xlatsvc
return xlatsvc
def test_for_correct_msgids(self):
xlatdmn = self._getCollectingTranslationDomain()
result = StringIO()
program, macros = self._compile(
'<div i18n:translate="">This is text for '
......@@ -148,13 +202,90 @@ class I18NCornerTestCase(TestCaseBase):
self.interpreter = TALInterpreter(program, {}, self.engine,
stream=result)
self.interpreter()
self.assert_('BaRvAlUe' in xlatsvc.data)
self.assert_('This is text for ${bar_name}.' in
xlatsvc.data)
msgids = list(xlatdmn.data)
msgids.sort()
self.assertEqual(2, len(msgids))
self.assertEqual('BaRvAlUe', msgids[0][0])
self.assertEqual('This is text for ${bar_name}.', msgids[1][0])
self.assertEqual({'bar_name': '<span>BARVALUE</span>'}, msgids[1][1])
self.assertEqual(
'<div>THIS IS TEXT FOR <span>BARVALUE</span>.</div>\n',
result.getvalue())
def test_for_raw_msgids(self):
# Test for Issue 314: i18n:translate removes line breaks from
# <pre>...</pre> contents
# HTML mode
xlatdmn = self._getCollectingTranslationDomain()
result = StringIO()
program, macros = self._compile(
'<div i18n:translate=""> This is text\n'
' \tfor\n div. </div>'
'<pre i18n:translate=""> This is text\n'
' <b>\tfor</b>\n pre. </pre>')
self.interpreter = TALInterpreter(program, {}, self.engine,
stream=result)
self.interpreter()
msgids = list(xlatdmn.data)
msgids.sort()
self.assertEqual(2, len(msgids))
self.assertEqual(' This is text\n <b>\tfor</b>\n pre. ', msgids[0][0])
self.assertEqual('This is text for div.', msgids[1][0])
self.assertEqual(
'<div>THIS IS TEXT FOR DIV.</div>'
'<pre> THIS IS TEXT\n <B>\tFOR</B>\n PRE. </pre>\n',
result.getvalue())
# XML mode
xlatdmn = self._getCollectingTranslationDomain()
result = StringIO()
parser = TALParser()
parser.parseString(
'<?xml version="1.0"?>\n'
'<pre xmlns:i18n="http://xml.zope.org/namespaces/i18n"'
' i18n:translate=""> This is text\n'
' <b>\tfor</b>\n barvalue. </pre>')
program, macros = parser.getCode()
self.interpreter = TALInterpreter(program, {}, self.engine,
stream=result)
self.interpreter()
msgids = list(xlatdmn.data)
msgids.sort()
self.assertEqual(1, len(msgids))
self.assertEqual('This is text <b> for</b> barvalue.', msgids[0][0])
self.assertEqual(
'<?xml version="1.0"?>\n'
'<pre>THIS IS TEXT <B> FOR</B> BARVALUE.</pre>\n',
result.getvalue())
def test_raw_msgids_and_i18ntranslate_i18nname(self):
xlatdmn = self._getCollectingTranslationDomain()
result = StringIO()
program, macros = self._compile(
'<div i18n:translate=""> This is text\n \tfor\n'
'<pre tal:content="raw" i18n:name="raw"'
' i18n:translate=""></pre>.</div>')
self.interpreter = TALInterpreter(program, {}, self.engine,
stream=result)
self.interpreter()
msgids = list(xlatdmn.data)
msgids.sort()
self.assertEqual(2, len(msgids))
self.assertEqual(' \tRaW\n ', msgids[0][0])
self.assertEqual('This is text for ${raw}.', msgids[1][0])
self.assertEqual({'raw': '<pre> \tRAW\n </pre>'}, msgids[1][1])
self.assertEqual(
u'<div>THIS IS TEXT FOR <pre> \tRAW\n </pre>.</div>\n',
result.getvalue())
def test_for_handling_unicode_vars(self):
# Make sure that non-ASCII Unicode is substituted correctly.
# http://collector.zope.org/Zope3-dev/264
program, macros = self._compile(
"<div i18n:translate='' tal:define='bar python:unichr(0xC0)'>"
"Foo <span tal:replace='bar' i18n:name='bar' /></div>")
self._check(program, u"<div>FOO \u00C0</div>\n")
class I18NErrorsTestCase(TestCaseBase):
......@@ -187,7 +318,7 @@ class I18NErrorsTestCase(TestCaseBase):
class OutputPresentationTestCase(TestCaseBase):
def check_attribute_wrapping(self):
def test_attribute_wrapping(self):
# To make sure the attribute-wrapping code is invoked, we have to
# include at least one TAL/METAL attribute to avoid having the start
# tag optimized into a rawtext instruction.
......@@ -202,17 +333,17 @@ class OutputPresentationTestCase(TestCaseBase):
</html>''' "\n"
self.compare(INPUT, EXPECTED)
def check_unicode_content(self):
def test_unicode_content(self):
INPUT = """<p tal:content="python:u'dj-vu'">para</p>"""
EXPECTED = u"""<p>dj-vu</p>""" "\n"
self.compare(INPUT, EXPECTED)
def check_unicode_structure(self):
def test_unicode_structure(self):
INPUT = """<p tal:replace="structure python:u'dj-vu'">para</p>"""
EXPECTED = u"""dj-vu""" "\n"
self.compare(INPUT, EXPECTED)
def check_i18n_replace_number(self):
def test_i18n_replace_number(self):
INPUT = """
<p i18n:translate="foo ${bar}">
<span tal:replace="python:123" i18n:name="bar">para</span>
......@@ -221,13 +352,13 @@ class OutputPresentationTestCase(TestCaseBase):
<p>FOO 123</p>""" "\n"
self.compare(INPUT, EXPECTED)
def check_entities(self):
INPUT = ('<img tal:attributes="alt default" '
def test_entities(self):
INPUT = ('<img tal:define="foo nothing" '
'alt="&a; &#1; &#x0a; &a &#45 &; &#0a; <>" />')
EXPECTED = ('<img alt="&a; &#1; &#x0a; '
'&amp;a &amp;#45 &amp;; &amp;#0a; &lt;&gt;" />\n')
self.compare(INPUT, EXPECTED)
def compare(self, INPUT, EXPECTED):
program, macros = self._compile(INPUT)
sio = StringIO()
......@@ -236,43 +367,44 @@ class OutputPresentationTestCase(TestCaseBase):
self.assertEqual(sio.getvalue(), EXPECTED)
class InterpolateTestCase(TestCaseBase):
def check_syntax_ok(self):
def test_syntax_ok(self):
text = "foo ${bar_0MAN} $baz_zz bee"
mapping = {'bar_0MAN': 'fish', 'baz_zz': 'moo'}
expected = "foo fish moo bee"
self.assertEqual(interpolate(text, mapping), expected)
def check_syntax_bad(self):
def test_syntax_bad(self):
text = "foo $_bar_man} $ ${baz bee"
mapping = {'_bar_man': 'fish', 'baz': 'moo'}
expected = text
self.assertEqual(interpolate(text, mapping), expected)
def check_missing(self):
def test_missing(self):
text = "foo ${bar} ${baz}"
mapping = {'bar': 'fish'}
expected = "foo fish ${baz}"
self.assertEqual(interpolate(text, mapping), expected)
def check_redundant(self):
def test_redundant(self):
text = "foo ${bar}"
mapping = {'bar': 'fish', 'baz': 'moo'}
expected = "foo fish"
self.assertEqual(interpolate(text, mapping), expected)
def check_numeric(self):
def test_numeric(self):
text = "foo ${bar}"
mapping = {'bar': 123}
expected = "foo 123"
self.assertEqual(interpolate(text, mapping), expected)
def check_unicode(self):
def test_unicode(self):
text = u"foo ${bar}"
mapping = {u'bar': u'baz'}
expected = u"foo baz"
self.assertEqual(interpolate(text, mapping), expected)
def check_unicode_mixed_unknown_encoding(self):
def test_unicode_mixed_unknown_encoding(self):
# This test assumes that sys.getdefaultencoding is ascii...
text = u"foo ${bar}"
mapping = {u'bar': 'd\xe9j\xe0'}
......@@ -280,13 +412,13 @@ class InterpolateTestCase(TestCaseBase):
self.assertEqual(interpolate(text, mapping), expected)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(MacroErrorsTestCase, "check_"))
suite.addTest(unittest.makeSuite(OutputPresentationTestCase, "check_"))
suite.addTest(unittest.makeSuite(InterpolateTestCase, "check_"))
suite = unittest.makeSuite(I18NErrorsTestCase)
suite.addTest(unittest.makeSuite(MacroErrorsTestCase))
suite.addTest(unittest.makeSuite(OutputPresentationTestCase))
suite.addTest(unittest.makeSuite(I18NCornerTestCase))
return suite
suite.addTest(unittest.makeSuite(InterpolateTestCase))
return suite
if __name__ == "__main__":
errs = utils.run_suite(test_suite())
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment