Merge of the Zope-2_6-i18n-branch into HEAD.

Impacted code:
- TAL: merge of the 2.7 i18n stuff, unicode fixes, tests.
- PageTemplates: addition of a global translation service and of its use
  by the TALES engine, unicode fixes, tests.
- StructuredText: unicode fixes, tests.

lib/python/Products/PageTemplates/Expressions.py

@@ -17,7 +17,7 @@ Page Template-specific implementation of TALES, with handlers
 for Python expressions, string literals, and paths.
 """
 
-__version__='$Revision: 1.37 $'[11:-2]
+__version__='$Revision: 1.38 $'[11:-2]
 
 import re, sys
 from TALES import Engine, CompilerError, _valid_name, NAME_RE, \
@@ -246,7 +246,10 @@ class NotExpr:
         self._c = compiler.compile(expr)
 
     def __call__(self, econtext):
-        return not econtext.evaluateBoolean(self._c)
+        # We use the (not x) and 1 or 0 formulation to avoid changing
+        # the representation of the result in Python 2.3, where the
+        # result of "not" becomes an instance of bool.
+        return (not econtext.evaluateBoolean(self._c)) and 1 or 0
 
     def __repr__(self):
         return 'not:%s' % `self._s`

lib/python/Products/PageTemplates/GlobalTranslationService.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Global Translation Service for providing I18n to Page Templates.
+
+$Id: GlobalTranslationService.py,v 1.2 2002/09/18 15:12:46 efge Exp $
+"""
+
+class DummyTranslationService:
+    """Translation service that does nothing and returns the message id."""
+    def translate(self, domain, msgid, mapping=None,
+                  context=None, target_language=None):
+        return msgid
+    # XXX Not all of Zope.I18n.ITranslationService is implemented.
+
+translationService = DummyTranslationService()
+
+def setGlobalTranslationService(service):
+    """Sets the global translation service, and returns the previous one."""
+    global translationService
+    old_service = translationService
+    translationService = service
+    return old_service
+
+def getGlobalTranslationService():
+    """Returns the global translation service."""
+    return translationService

lib/python/Products/PageTemplates/PageTemplate.py

@@ -15,7 +15,7 @@
 HTML- and XML-based template objects using TAL, TALES, and METAL.
 """
 
-__version__='$Revision: 1.27 $'[11:-2]
+__version__='$Revision: 1.28 $'[11:-2]
 
 import sys
 
@@ -24,7 +24,8 @@ from TAL.HTMLTALParser import HTMLTALParser
 from TAL.TALGenerator import TALGenerator
 from TAL.TALInterpreter import TALInterpreter
 from Expressions import getEngine
-from cStringIO import StringIO
+# Do not use cStringIO here!  It's not unicode aware. :(
+from StringIO import StringIO
 from ExtensionClass import Base
 from ComputedAttribute import ComputedAttribute
 
@@ -208,3 +209,4 @@ class PageTemplateTracebackSupplement:
         if e:
             w = list(w) + list(e)
         self.warnings = w
+

lib/python/Products/PageTemplates/PageTemplateFile.py

@@ -15,7 +15,7 @@
 Zope object encapsulating a Page Template from the filesystem.
 """
 
-__version__='$Revision: 1.20 $'[11:-2]
+__version__='$Revision: 1.21 $'[11:-2]
 
 import os, AccessControl, Acquisition, sys
 from Globals import package_home, DevelopmentMode
@@ -85,7 +85,8 @@ class PageTemplateFile(Script, PageTemplate, Traversable):
             response = self.REQUEST.RESPONSE
             if not response.headers.has_key('content-type'):
                 response.setHeader('content-type', self.content_type)
-        except AttributeError: pass
+        except AttributeError:
+            pass
 
         # Execute the template in a new security context.
         security=getSecurityManager()

lib/python/Products/PageTemplates/PythonExpr.py

@@ -14,7 +14,7 @@
 """Generic Python Expression Handler
 """
 
-__version__='$Revision: 1.8 $'[11:-2]
+__version__='$Revision: 1.9 $'[11:-2]
 
 from TALES import CompilerError
 from sys import exc_info
@@ -78,3 +78,4 @@ class ExprTypeProxy:
     def __call__(self, text):
         return self._handler(self._name, text,
                              self._econtext._engine)(self._econtext)
+

lib/python/Products/PageTemplates/TALES.py

@@ -15,10 +15,12 @@
 An implementation of a generic TALES engine
 """
 
-__version__='$Revision: 1.31 $'[11:-2]
+__version__='$Revision: 1.32 $'[11:-2]
 
 import re, sys, ZTUtils
 from MultiMapping import MultiMapping
+from DocumentTemplate.DT_Util import ustr
+from GlobalTranslationService import getGlobalTranslationService
 
 StringType = type('')
 
@@ -222,11 +224,11 @@ class Context:
     def evaluateBoolean(self, expr):
         return not not self.evaluate(expr)
 
-    def evaluateText(self, expr, None=None):
+    def evaluateText(self, expr):
         text = self.evaluate(expr)
         if text is Default or text is None:
             return text
-        return str(text)
+        return ustr(text)
 
     def evaluateStructure(self, expr):
         return self.evaluate(expr)
@@ -249,6 +251,11 @@ class Context:
     def setPosition(self, position):
         self.position = position
 
+    def translate(self, domain, msgid, mapping=None,
+                  context=None, target_language=None):
+        return getGlobalTranslationService().translate(
+            domain, msgid, mapping=mapping,
+            context=context, target_language=target_language)
 
 
 class TALESTracebackSupplement:
@@ -282,3 +289,4 @@ class SimpleExpr:
         return self._name, self._expr
     def __repr__(self):
         return '<SimpleExpr %s %s>' % (self._name, `self._expr`)
+

lib/python/Products/PageTemplates/ZopePageTemplate.py

@@ -15,7 +15,7 @@
 Zope object encapsulating a Page Template.
 """
 
-__version__='$Revision: 1.43 $'[11:-2]
+__version__='$Revision: 1.44 $'[11:-2]
 
 import os, AccessControl, Acquisition, sys
 from types import StringType
@@ -369,3 +369,4 @@ def initialize(context):
         )
     context.registerHelp()
     context.registerHelpTitle('Zope Help')
+

lib/python/Products/PageTemplates/tests/__init__.py

@@ -34,3 +34,4 @@ class harness2(harness1):
         assert aargs == args, "Harness method arguments"
         assert akwargs == kwargs, "Harness method keyword args"
         return result
+    

lib/python/Products/PageTemplates/tests/input/CheckI18nTranslate.html

+<html>
+<body>
+<head>
+   <p i18n:translate="foobar">baz</p>
+</head>
+</body>
+</html>

lib/python/Products/PageTemplates/tests/input/CheckI18nTranslateHooked.html

+<html>
+<body>
+<head>
+   <p i18n:domain="foo" i18n:translate="bar">baz</p>
+</head>
+</body>
+</html>

lib/python/Products/PageTemplates/tests/input/CheckUnicodeInserts.html

+<html>
+<body>
+<head>
+   <p tal:content="python:u'e acute=\xe9'">e acute here</p>
+</head>
+</body>
+</html>

lib/python/Products/PageTemplates/tests/output/CheckI18nTranslate.html

+<html>
+<body>
+<head>
+   <p>foobar</p>
+</head>
+</body>
+</html>

lib/python/Products/PageTemplates/tests/output/CheckI18nTranslateHooked.html

+<html>
+<body>
+<head>
+   <p>[foo](bar)</p>
+</head>
+</body>
+</html>

lib/python/Products/PageTemplates/tests/output/CheckUnicodeInserts.html

+<html>
+<body>
+<head>
+   <p>e acute=é</p>
+</head>
+</body>
+</html>

lib/python/Products/PageTemplates/tests/testDTMLTests.py

@@ -140,3 +140,4 @@ def test_suite():
 
 if __name__=='__main__':
     main()
+

lib/python/Products/PageTemplates/tests/testHTMLTests.py

@@ -15,8 +15,9 @@ import os, sys, unittest
 
 from Products.PageTemplates.tests import util
 from Products.PageTemplates.PageTemplate import PageTemplate
-import ZODB
-from AccessControl import User, SecurityManager
+from Products.PageTemplates.GlobalTranslationService import \
+     setGlobalTranslationService
+from AccessControl import SecurityManager
 from AccessControl.SecurityManagement import noSecurityManager
 
 from Acquisition import Implicit
@@ -26,6 +27,10 @@ class AqPageTemplate(Implicit, PageTemplate):
 class Folder(util.Base):
     pass
 
+class TestTranslationService:
+    def translate(self, domain, msgid, *args, **kw):
+        return "[%s](%s)" % (domain, msgid)
+
 
 class UnitTestSecurityPolicy:
     """
@@ -69,6 +74,14 @@ class HTMLTests(unittest.TestCase):
         out = apply(t, args, kwargs)
         util.check_html(expect, out)
 
+    def assert_expected_unicode(self, t, fname, *args, **kwargs):
+        t.write(util.read_input(fname))
+        assert not t._v_errors, 'Template errors: %s' % t._v_errors
+        expect = util.read_output(fname)
+        expect = unicode(expect, 'utf8')
+        out = apply(t, args, kwargs)
+        util.check_html(expect, out)
+
     def getProducts(self):
         return [
            {'description': 'This is the tee for those who LOVE Zope. '
@@ -126,8 +139,20 @@ class HTMLTests(unittest.TestCase):
     def checkBatchIteration(self):
         self.assert_expected(self.folder.t, 'CheckBatchIteration.html')
 
+    def checkUnicodeInserts(self):
+        self.assert_expected_unicode(self.folder.t, 'CheckUnicodeInserts.html')
+
+    def checkI18nTranslate(self):
+        self.assert_expected(self.folder.t, 'CheckI18nTranslate.html')
+
+    def checkI18nTranslateHooked(self):
+        old_ts = setGlobalTranslationService(TestTranslationService())
+        self.assert_expected(self.folder.t, 'CheckI18nTranslateHooked.html')
+        setGlobalTranslationService(old_ts)
+
 def test_suite():
     return unittest.makeSuite(HTMLTests, 'check')
 
 if __name__=='__main__':
-    unittest.main(defaultTest='test_suite')
+   main()
+

lib/python/Products/PageTemplates/tests/testTALES.py

@@ -4,6 +4,16 @@ from Products.PageTemplates import TALES
 from Products.PageTemplates.tests import harness1
 import string
 
+class DummyUnicodeExpr:
+    '''Dummy expression type handler returning unicode'''
+    def __init__(self, name, expr, engine):
+        self._name = name
+        self._expr = expr
+    def __call__(self, econtext):
+        return unicode(self._expr, 'latin1')
+    def __repr__(self):
+        return '<SimpleExpr %s %s>' % (self._name, `self._expr`)
+
 class TALESTests(unittest.TestCase):
 
     def testIterator0(self):
@@ -77,6 +87,7 @@ class TALESTests(unittest.TestCase):
     def getContext(self, **kws):
         e = TALES.Engine()
         e.registerType('simple', TALES.SimpleExpr)
+        e.registerType('unicode', DummyUnicodeExpr)
         return apply(e.getContext, (), kws)
 
     def testContext0(self):
@@ -85,6 +96,11 @@ class TALESTests(unittest.TestCase):
         assert se == ('simple', 'x'), (
             'Improperly evaluated expression %s.' % `se`)
 
+    def testContextUnicode(self):
+        '''Test evaluateText on unicode-returning expressions'''
+        se = self.getContext().evaluateText('unicode:\xe9')
+        self.assertEqual(se, u'\xe9')
+
     def testVariables(self):
         '''Test variables'''
         ctxt = self.getContext()
@@ -114,4 +130,4 @@ def test_suite():
     return unittest.makeSuite(TALESTests)
 
 if __name__=='__main__':
-    unittest.main(defaultTest='test_suite')
+    main()

lib/python/StructuredText/ClassicDocumentClass.py

@@ -14,8 +14,8 @@
 import re, ST, STDOM
 from STletters import letters
 
-StringType=type('')
-ListType=type([])
+from types import StringType, UnicodeType, ListType
+StringTypes = (StringType, UnicodeType)
 
 class StructuredTextExample(ST.StructuredTextParagraph):
     """Represents a section of document with literal text, as for examples"""
@@ -235,7 +235,7 @@ class DocumentClass:
         ]
 
     def __call__(self, doc):
-        if type(doc) is type(''):
+        if type(doc) in StringTypes:
             doc=ST.StructuredText(doc)
             doc.setSubparagraphs(self.color_paragraphs(
                doc.getSubparagraphs()))
@@ -245,7 +245,7 @@ class DocumentClass:
         return doc
 
     def parse(self, raw_string, text_type,
-              type=type, st=type(''), lt=type([])):
+              type=type, sts=StringTypes, lt=type([])):
 
         """
         Parse accepts a raw_string, an expr to test the raw_string,
@@ -261,7 +261,7 @@ class DocumentClass:
         tmp = []    # the list to be returned if raw_string is split
         append=tmp.append
 
-        if type(text_type) is st: text_type=getattr(self, text_type)
+        if type(text_type) in sts: text_type=getattr(self, text_type)
 
         while 1:
             t = text_type(raw_string)
@@ -272,7 +272,7 @@ class DocumentClass:
             if start: append(raw_string[0:start])
 
             tt=type(t)
-            if tt is st:
+            if tt in sts:
                 # if we get a string back, add it to text to be parsed
                 raw_string = t+raw_string[end:len(raw_string)]
             else:
@@ -299,12 +299,12 @@ class DocumentClass:
 
         for text_type in types:
 
-            if type(str) is StringType:
+            if type(str) in StringTypes:
                 str = self.parse(str, text_type)
             elif type(str) is ListType:
                 r=[]; a=r.append
                 for s in str:
-                    if type(s) is StringType:
+                    if type(s) in StringTypes:
                         s=self.parse(s, text_type)
                         if type(s) is ListType: r[len(r):]=s
                         else: a(s)
@@ -327,7 +327,7 @@ class DocumentClass:
 
     def color_paragraphs(self, raw_paragraphs,
                            type=type, sequence_types=(type([]), type(())),
-                           st=type('')):
+                           sts=StringTypes):
         result=[]
         for paragraph in raw_paragraphs:
 
@@ -336,7 +336,7 @@ class DocumentClass:
                 continue
 
             for pt in self.paragraph_types:
-                if type(pt) is st:
+                if type(pt) in sts:
                     # grab the corresponding function
                     pt=getattr(self, pt)
                 # evaluate the paragraph

lib/python/StructuredText/DocumentClass.py

@@ -15,8 +15,8 @@ import re, ST, STDOM
 from STletters import letters, digits, literal_punc, under_punc,\
      strongem_punc, phrase_delimiters,dbl_quoted_punc
 
-StringType=type('')
-ListType=type([])
+from types import StringType, UnicodeType, ListType
+StringTypes = (StringType, UnicodeType)
 
 def flatten(obj, append):
     if obj.getNodeType()==STDOM.TEXT_NODE:
@@ -308,7 +308,7 @@ class DocumentClass:
         ]
 
     def __call__(self, doc):
-        if type(doc) is type(''):
+        if type(doc) in StringTypes:
             doc=ST.StructuredText(doc)
             doc.setSubparagraphs(self.color_paragraphs(
                doc.getSubparagraphs()))
@@ -318,7 +318,7 @@ class DocumentClass:
         return doc
 
     def parse(self, raw_string, text_type,
-              type=type, st=type(''), lt=type([])):
+              type=type, sts=StringTypes, lt=type([])):
 
         """
         Parse accepts a raw_string, an expr to test the raw_string,
@@ -334,7 +334,7 @@ class DocumentClass:
         tmp = []    # the list to be returned if raw_string is split
         append=tmp.append
 
-        if type(text_type) is st: text_type=getattr(self, text_type)
+        if type(text_type) in sts: text_type=getattr(self, text_type)
 
         while 1:
             t = text_type(raw_string)
@@ -345,7 +345,7 @@ class DocumentClass:
             if start: append(raw_string[0:start])
 
             tt=type(t)
-            if tt is st:
+            if tt in sts:
                 # if we get a string back, add it to text to be parsed
                 raw_string = t+raw_string[end:len(raw_string)]
             else:
@@ -372,12 +372,12 @@ class DocumentClass:
 
         for text_type in types:
 
-            if type(str) is StringType:
+            if type(str) in StringTypes:
                 str = self.parse(str, text_type)
             elif type(str) is ListType:
                 r=[]; a=r.append
                 for s in str:
-                    if type(s) is StringType:
+                    if type(s) in StringTypes:
                         s=self.parse(s, text_type)
                         if type(s) is ListType: r[len(r):]=s
                         else: a(s)
@@ -400,7 +400,7 @@ class DocumentClass:
 
     def color_paragraphs(self, raw_paragraphs,
                            type=type, sequence_types=(type([]), type(())),
-                           st=type('')):
+                           sts=StringTypes):
         result=[]
         for paragraph in raw_paragraphs:
             if paragraph.getNodeName() != 'StructuredTextParagraph':
@@ -408,7 +408,7 @@ class DocumentClass:
                 continue
 
             for pt in self.paragraph_types:
-                if type(pt) is st:
+                if type(pt) in sts:
                     # grab the corresponding function
                     pt=getattr(self, pt)
                 # evaluate the paragraph

lib/python/StructuredText/STDOM.py

@@ -16,6 +16,9 @@ DOM implementation in StructuredText : Read-Only methods
 All standard Zope objects support DOM to a limited extent.
 """
 
+from types import StringType, UnicodeType
+StringTypes = (StringType, UnicodeType)
+
 # Node type codes
 # ---------------
 
@@ -81,7 +84,7 @@ class ParentNode:
     the child access methods of the DOM.
     """
 
-    def getChildNodes(self, type=type, st=type('')):
+    def getChildNodes(self, type=type, sts=StringTypes):
         """
         Returns a NodeList that contains all children of this node.
         If there are no children, this is a empty NodeList
@@ -89,12 +92,12 @@ class ParentNode:
 
         r=[]
         for n in self.getChildren():
-            if type(n) is st: n=TextNode(n)
+            if type(n) in sts: n=TextNode(n)
             r.append(n.__of__(self))
 
         return  NodeList(r)
 
-    def getFirstChild(self, type=type, st=type('')):
+    def getFirstChild(self, type=type, sts=StringTypes):
         """
         The first child of this node. If there is no such node
         this returns None
@@ -106,12 +109,12 @@ class ParentNode:
 
         n=children[0]
 
-        if type(n) is st:
+        if type(n) in sts:
             n=TextNode(n)
 
         return n.__of__(self)
 
-    def getLastChild(self, type=type, st=type('')):
+    def getLastChild(self, type=type, sts=StringTypes):
         """
         The last child of this node.  If there is no such node
         this returns None.
@@ -119,21 +122,21 @@ class ParentNode:
         children = self.getChildren()
         if not children: return None
         n=chidren[-1]
-        if type(n) is st: n=TextNode(n)
+        if type(n) in sts: n=TextNode(n)
         return n.__of__(self)
 
     """
     create aliases for all above functions in the pythony way.
     """
 
-    def _get_ChildNodes(self, type=type, st=type('')):
-        return self.getChildNodes(type,st)
+    def _get_ChildNodes(self, type=type, sts=StringTypes):
+        return self.getChildNodes(type,sts)
 
-    def _get_FirstChild(self, type=type, st=type('')):
-        return self.getFirstChild(type,st)
+    def _get_FirstChild(self, type=type, sts=StringTypes):
+        return self.getFirstChild(type,sts)
 
-    def _get_LastChild(self, type=type, st=type('')):
-        return self.getLastChild(type,st)
+    def _get_LastChild(self, type=type, sts=StringTypes):
+        return self.getLastChild(type,sts)
 
 class NodeWrapper(ParentNode):
     """
@@ -167,7 +170,7 @@ class NodeWrapper(ParentNode):
 
     def getPreviousSibling(self,
                            type=type,
-                           st=type(''),
+                           sts=StringTypes,
                            getattr=getattr,
                            None=None):
 
@@ -190,13 +193,13 @@ class NodeWrapper(ParentNode):
         try: n=children[index]
         except IndexError: return None
         else:
-            if type(n) is st:
+            if type(n) in sts:
                 n=TextNode(n)
             n._DOMIndex=index
             return n.__of__(self)
 
 
-    def getNextSibling(self, type=type, st=type('')):
+    def getNextSibling(self, type=type, sts=StringTypes):
         """
         The node immediately preceding this node.  If
         there is no such node, this returns None.
@@ -216,7 +219,7 @@ class NodeWrapper(ParentNode):
         except IndexError:
             return None
         else:
-            if type(n) is st:
+            if type(n) in sts:
                 n=TextNode(n)
             n._DOMIndex=index
             return n.__of__(self)
@@ -239,14 +242,14 @@ class NodeWrapper(ParentNode):
 
     def _get_PreviousSibling(self,
                            type=type,
-                           st=type(''),
+                           sts=StringTypes,
                            getattr=getattr,
                            None=None):
 
-        return self.getPreviousSibling(type,st,getattr,None)
+        return self.getPreviousSibling(type,sts,getattr,None)
 
-    def _get_NextSibling(self, type=type, st=type('')):
-        return self.getNextSibling(type,st)
+    def _get_NextSibling(self, type=type, sts=StringTypes):
+        return self.getNextSibling(type,sts)
 
     def _get_OwnerDocument(self):
         return self.getOwnerDocument()
@@ -288,7 +291,7 @@ class Node(ParentNode):
 
     def getPreviousSibling(self,
                            type=type,
-                           st=type(''),
+                           sts=StringTypes,
                            getattr=getattr,
                            None=None):
         """
@@ -296,7 +299,7 @@ class Node(ParentNode):
         there is no such node, this returns None.
         """
 
-    def getNextSibling(self, type=type, st=type('')):
+    def getNextSibling(self, type=type, sts=StringTypes):
         """
         The node immediately preceding this node.  If
         there is no such node, this returns None.
@@ -342,13 +345,13 @@ class Node(ParentNode):
 
     def _get_PreviousSibling(self,
                            type=type,
-                           st=type(''),
+                           sts=StringTypes,
                            getattr=getattr,
                            None=None):
 
-        return self.getPreviousSibling(type,st,getattr,None)
+        return self.getPreviousSibling(type,sts,getattr,None)
 
-    def _get_NextSibling(self, type=type, st=type('')):
+    def _get_NextSibling(self, type=type, sts=StringTypes):
         return self.getNextSibling()
 
     def _get_Attributes(self):
@@ -407,10 +410,10 @@ class Element(Node):
         """A code representing the type of the node."""
         return ELEMENT_NODE
 
-    def getNodeValue(self, type=type, st=type('')):
+    def getNodeValue(self, type=type, sts=StringTypes):
         r=[]
         for c in self.getChildren():
-            if type(c) is not st:
+            if type(c) not in sts:
                 c=c.getNodeValue()
             r.append(c)
         return ''.join(r)
@@ -480,8 +483,8 @@ class Element(Node):
     def _get_NodeType(self):
         return self.getNodeType()
 
-    def _get_NodeValue(self, type=type, st=type('')):
-        return self.getNodeValue(type,st)
+    def _get_NodeValue(self, type=type, sts=StringTypes):
+        return self.getNodeValue(type,sts)
 
     def _get_ParentNode(self):
         return self.getParentNode()
@@ -517,7 +520,7 @@ class NodeList:
     def __init__(self,list=None):
         self._data = list or []
 
-    def __getitem__(self, index, type=type, st=type('')):
+    def __getitem__(self, index, type=type, sts=StringTypes):
         return self._data[index]
 
     def __getslice__(self, i, j):

lib/python/StructuredText/__init__.py

@@ -17,14 +17,14 @@ from StructuredText import html_with_references, HTML, html_quote
 from ST import Basic
 import DocBookClass
 import HTMLWithImages
-from types import StringType
+from types import StringType, UnicodeType
 import DocumentWithImages
 
 ClassicHTML=HTML
 HTMLNG=HTMLClass.HTMLClass()
 
 def HTML(src, level=1):
-    if isinstance(src, StringType):
+    if isinstance(src, StringType) or isinstance(src, UnicodeType):
         return ClassicHTML(src, level)
     return HTMLNG(src, level)
 

lib/python/StructuredText/tests/testStructuredText.py

@@ -18,6 +18,7 @@ from StructuredText import StructuredText
 from StructuredText import HTMLClass
 from StructuredText.StructuredText import HTML
 import sys, os, unittest, cStringIO
+from types import UnicodeType
 from OFS import ndiff
 
 """
@@ -52,6 +53,8 @@ class StructuredTextTests(unittest.TestCase):
             raw_text = readFile(regressions,f)
             assert StructuredText.StructuredText(raw_text),\
                 'StructuredText failed on %s' % f
+            assert StructuredText.StructuredText(unicode(raw_text)),\
+                'StructuredText failed on Unicode %s' % f
 
     def testStructuredTextNG(self):
         """ testing StructuredTextNG """
@@ -60,6 +63,8 @@ class StructuredTextTests(unittest.TestCase):
             raw_text = readFile(regressions,f)
             assert ST.StructuredText(raw_text),\
                 'StructuredText failed on %s' % f
+            assert ST.StructuredText(unicode(raw_text)),\
+                'StructuredText failed on Unicode %s' % f
 
 
     def testDocumentClass(self):
@@ -131,6 +136,7 @@ class BasicTests(unittest.TestCase):
 
     def _test(self,stxtxt , expected):
 
+        if not isinstance(stxtxt, UnicodeType):
             res = HTML(stxtxt,level=1,header=0)
             if res.find(expected)==-1:
                 print "Text:     ",stxtxt
@@ -138,6 +144,18 @@ class BasicTests(unittest.TestCase):
                 print "Expected: ",expected
                 raise AssertionError,"basic test failed for '%s'" % stxtxt
 
+        if isinstance(stxtxt, UnicodeType):
+            ustxtxt = stxtxt
+        else:
+            ustxtxt = unicode(stxtxt)
+        res = HTML(ustxtxt,level=1,header=0)
+        if res.find(expected)==-1:
+            print "Text:     ",stxtxt.encode('latin-1')
+            print "Converted:",res.encode('latin-1')
+            print "Expected: ",expected.encode('latin-1')
+            raise AssertionError, ("basic test failed for Unicode '%s'"
+                                   % stxtxt)
+
 
     def testUnderline(self):
         self._test("xx _this is html_ xx",
@@ -192,6 +210,14 @@ class BasicTests(unittest.TestCase):
         '<code>"literal":http://www.zope.org/.</code>')
 
 
+    def XXXtestUnicodeContent(self):
+        # This fails because ST uses the default locale to get "letters"
+        # whereas it should use \w+ and re.U if the string is Unicode.
+        #self._test(u"h\xe9 **y\xe9** xx",
+        #           u"h\xe9 <strong>y\xe9</strong> xx")
+        pass
+
+
 def test_suite():
     suite = unittest.TestSuite()
     suite.addTest( unittest.makeSuite( StructuredTextTests ) )

lib/python/TAL/DummyEngine.py

@@ -8,7 +8,7 @@
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
+# FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
 """
@@ -18,13 +18,20 @@ Dummy TALES engine so that I can test out the TAL implementation.
 import re
 import sys
 
-import driver
-
 from TALDefs import NAME_RE, TALESError, ErrorInfo
-
-class Default:
+from ITALES import ITALESCompiler, ITALESEngine
+from DocumentTemplate.DT_Util import ustr
+try:
+    from Zope.I18n.ITranslationService import ITranslationService
+    from Zope.I18n.IDomain import IDomain
+except ImportError:
+    # Before 2.7
+    class ITranslationService: pass
+    class IDomain: pass
+
+class _Default:
     pass
-Default = Default()
+Default = _Default()
 
 name_match = re.compile(r"(?s)(%s):(.*)\Z" % NAME_RE).match
 
@@ -36,6 +43,8 @@ class DummyEngine:
     position = None
     source_file = None
 
+    __implements__ = ITALESCompiler, ITALESEngine
+
     def __init__(self, macros=None):
         if macros is None:
             macros = {}
@@ -43,6 +52,7 @@ class DummyEngine:
         dict = {'nothing': None, 'default': Default}
         self.locals = self.globals = dict
         self.stack = [dict]
+        self.translationService = DummyTranslationService()
 
     def getCompilerError(self):
         return CompilerError
@@ -90,13 +100,7 @@ class DummyEngine:
         if type in ("string", "str"):
             return expr
         if type in ("path", "var", "global", "local"):
-            expr = expr.strip()
-            if self.locals.has_key(expr):
-                return self.locals[expr]
-            elif self.globals.has_key(expr):
-                return self.globals[expr]
-            else:
-                raise TALESError("unknown variable: %s" % `expr`)
+            return self.evaluatePathOrVar(expr)
         if type == "not":
             return not self.evaluate(expr)
         if type == "exists":
@@ -116,6 +120,15 @@ class DummyEngine:
             return '%s (%s,%s)' % (self.source_file, lineno, offset)
         raise TALESError("unrecognized expression: " + `expression`)
 
+    def evaluatePathOrVar(self, expr):
+        expr = expr.strip()
+        if self.locals.has_key(expr):
+            return self.locals[expr]
+        elif self.globals.has_key(expr):
+            return self.globals[expr]
+        else:
+            raise TALESError("unknown variable: %s" % `expr`)
+
     def evaluateValue(self, expr):
         return self.evaluate(expr)
 
@@ -125,7 +138,7 @@ class DummyEngine:
     def evaluateText(self, expr):
         text = self.evaluate(expr)
         if text is not None and text is not Default:
-            text = str(text)
+            text = ustr(text)
         return text
 
     def evaluateStructure(self, expr):
@@ -146,6 +159,7 @@ class DummyEngine:
             macro = self.macros[localName]
         else:
             # External macro
+            import driver
             program, macros = driver.compilefile(file)
             macro = macros.get(localName)
             if not macro:
@@ -157,6 +171,7 @@ class DummyEngine:
         file, localName = self.findMacroFile(macroName)
         if not file:
             return file, localName
+        import driver
         doc = driver.parsefile(file)
         return doc, localName
 
@@ -183,6 +198,10 @@ class DummyEngine:
     def getDefault(self):
         return Default
 
+    def translate(self, domain, msgid, mapping):
+        return self.translationService.translate(domain, msgid, mapping)
+    
+
 class Iterator:
 
     def __init__(self, name, seq, engine):
@@ -200,3 +219,31 @@ class Iterator:
         self.nextIndex = i+1
         self.engine.setLocal(self.name, item)
         return 1
+
+class DummyDomain:
+    __implements__ = IDomain
+
+    def translate(self, msgid, mapping=None, context=None,
+                  target_language=None):
+        # This is a fake translation service which simply uppercases non
+        # ${name} placeholder text in the message id.
+        #
+        # First, transform a string with ${name} placeholders into a list of
+        # substrings.  Then upcase everything but the placeholders, then glue
+        # things back together.
+        def repl(m, mapping=mapping):
+            return mapping[m.group(m.lastindex).lower()]
+        cre = re.compile(r'\$(?:([_A-Z]\w*)|\{([_A-Z]\w*)\})')
+        return cre.sub(repl, msgid.upper())
+
+class DummyTranslationService:
+    __implements__ = ITranslationService
+
+    def translate(self, domain, msgid, mapping=None, context=None,
+                  target_language=None):
+        # Ignore domain
+        return self.getDomain(domain).translate(msgid, mapping, context,
+                                                target_language)
+
+    def getDomain(self, domain):
+        return DummyDomain()

lib/python/TAL/HTMLTALParser.py

@@ -8,7 +8,7 @@
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
+# FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
 """
@@ -18,8 +18,9 @@ Parse HTML and compile to TALInterpreter intermediate code.
 import sys
 
 from TALGenerator import TALGenerator
-from TALDefs import ZOPE_METAL_NS, ZOPE_TAL_NS, METALError, TALError
 from HTMLParser import HTMLParser, HTMLParseError
+from TALDefs import \
+     ZOPE_METAL_NS, ZOPE_TAL_NS, ZOPE_I18N_NS, METALError, TALError, I18NError
 
 BOOLEAN_HTML_ATTRS = [
     # List of Boolean attributes in HTML that may be given in
@@ -106,13 +107,20 @@ class HTMLTALParser(HTMLParser):
         self.gen = gen
         self.tagstack = []
         self.nsstack = []
-        self.nsdict = {'tal': ZOPE_TAL_NS, 'metal': ZOPE_METAL_NS}
+        self.nsdict = {'tal': ZOPE_TAL_NS,
+                       'metal': ZOPE_METAL_NS,
+                       'i18n': ZOPE_I18N_NS,
+                       }
 
     def parseFile(self, file):
         f = open(file)
         data = f.read()
         f.close()
+        try:
             self.parseString(data)
+        except TALError, e:
+            e.setFile(file)
+            raise
 
     def parseString(self, data):
         self.feed(data)
@@ -132,9 +140,10 @@ class HTMLTALParser(HTMLParser):
     def handle_starttag(self, tag, attrs):
         self.close_para_tags(tag)
         self.scan_xmlns(attrs)
-        tag, attrlist, taldict, metaldict = self.process_ns(tag, attrs)
+        tag, attrlist, taldict, metaldict, i18ndict \
+             = self.process_ns(tag, attrs)
         self.tagstack.append(tag)
-        self.gen.emitStartElement(tag, attrlist, taldict, metaldict,
+        self.gen.emitStartElement(tag, attrlist, taldict, metaldict, i18ndict,
                                   self.getpos())
         if tag in EMPTY_HTML_TAGS:
             self.implied_endtag(tag, -1)
@@ -142,14 +151,15 @@ class HTMLTALParser(HTMLParser):
     def handle_startendtag(self, tag, attrs):
         self.close_para_tags(tag)
         self.scan_xmlns(attrs)
-        tag, attrlist, taldict, metaldict = self.process_ns(tag, attrs)
+        tag, attrlist, taldict, metaldict, i18ndict \
+             = self.process_ns(tag, attrs)
         if taldict.get("content"):
             self.gen.emitStartElement(tag, attrlist, taldict, metaldict,
-                                      self.getpos())
+                                      i18ndict, self.getpos())
             self.gen.emitEndElement(tag, implied=-1)
         else:
             self.gen.emitStartElement(tag, attrlist, taldict, metaldict,
-                                      self.getpos(), isend=1)
+                                      i18ndict, self.getpos(), isend=1)
         self.pop_xmlns()
 
     def handle_endtag(self, tag):
@@ -252,7 +262,7 @@ class HTMLTALParser(HTMLParser):
             prefix, suffix = name.split(':', 1)
             if prefix == 'xmlns':
                 nsuri = self.nsdict.get(suffix)
-                if nsuri in (ZOPE_TAL_NS, ZOPE_METAL_NS):
+                if nsuri in (ZOPE_TAL_NS, ZOPE_METAL_NS, ZOPE_I18N_NS):
                     return name, name, prefix
             else:
                 nsuri = self.nsdict.get(prefix)
@@ -260,19 +270,19 @@ class HTMLTALParser(HTMLParser):
                     return name, suffix, 'tal'
                 elif nsuri == ZOPE_METAL_NS:
                     return name, suffix,  'metal'
+                elif nsuri == ZOPE_I18N_NS:
+                    return name, suffix, 'i18n'
         return name, name, 0
 
     def process_ns(self, name, attrs):
         attrlist = []
         taldict = {}
         metaldict = {}
+        i18ndict = {}
         name, namebase, namens = self.fixname(name)
         for item in attrs:
             key, value = item
             key, keybase, keyns = self.fixname(key)
-            if ':' in key and not keyns:
-                ns = 0
-            else:
             ns = keyns or namens # default to tag namespace
             if ns and ns != 'unknown':
                 item = (key, value, ns)
@@ -286,7 +296,12 @@ class HTMLTALParser(HTMLParser):
                     raise METALError("duplicate METAL attribute " +
                                      `keybase`, self.getpos())
                 metaldict[keybase] = value
+            elif ns == 'i18n':
+                if i18ndict.has_key(keybase):
+                    raise I18NError("duplicate i18n attribute " +
+                                    `keybase`, self.getpos())
+                i18ndict[keybase] = value
             attrlist.append(item)
         if namens in ('metal', 'tal'):
             taldict['tal tag'] = namens
-        return name, attrlist, taldict, metaldict
+        return name, attrlist, taldict, metaldict, i18ndict

lib/python/TAL/ITALES.py

+"""Interface that a TALES engine provides to the METAL/TAL implementation."""
+
+try:
+    from Interface import Interface
+    from Interface.Attribute import Attribute
+except:
+    # Before 2.7
+    class Interface: pass
+    def Attribute(*args): pass
+
+
+class ITALESCompiler(Interface):
+    """Compile-time interface provided by a TALES implementation.
+
+    The TAL compiler needs an instance of this interface to support
+    compilation of TALES expressions embedded in documents containing
+    TAL and METAL constructs.
+    """
+
+    def getCompilerError():
+        """Return the exception class raised for compilation errors.
+        """
+
+    def compile(expression):
+        """Return a compiled form of 'expression' for later evaluation.
+
+        'expression' is the source text of the expression.
+
+        The return value may be passed to the various evaluate*()
+        methods of the ITALESEngine interface.  No compatibility is
+        required for the values of the compiled expression between
+        different ITALESEngine implementations.
+        """
+
+
+class ITALESEngine(Interface):
+    """Render-time interface provided by a TALES implementation.
+
+    The TAL interpreter uses this interface to TALES to support
+    evaluation of the compiled expressions returned by
+    ITALESCompiler.compile().
+    """
+
+    def getDefault():
+        """Return the value of the 'default' TALES expression.
+
+        Checking a value for a match with 'default' should be done
+        using the 'is' operator in Python.
+        """
+
+    def setPosition((lineno, offset)):
+        """Inform the engine of the current position in the source file.
+
+        This is used to allow the evaluation engine to report
+        execution errors so that site developers can more easily
+        locate the offending expression.
+        """
+
+    def setSourceFile(filename):
+        """Inform the engine of the name of the current source file.
+
+        This is used to allow the evaluation engine to report
+        execution errors so that site developers can more easily
+        locate the offending expression.
+        """
+
+    def beginScope():
+        """Push a new scope onto the stack of open scopes.
+        """
+
+    def endScope():
+        """Pop one scope from the stack of open scopes.
+        """
+
+    def evaluate(compiled_expression):
+        """Evaluate an arbitrary expression.
+
+        No constraints are imposed on the return value.
+        """
+
+    def evaluateBoolean(compiled_expression):
+        """Evaluate an expression that must return a Boolean value.
+        """
+
+    def evaluateMacro(compiled_expression):
+        """Evaluate an expression that must return a macro program.
+        """
+
+    def evaluateStructure(compiled_expression):
+        """Evaluate an expression that must return a structured
+        document fragment.
+
+        The result of evaluating 'compiled_expression' must be a
+        string containing a parsable HTML or XML fragment.  Any TAL
+        markup cnotained in the result string will be interpreted.
+        """
+
+    def evaluateText(compiled_expression):
+        """Evaluate an expression that must return text.
+
+        The returned text should be suitable for direct inclusion in
+        the output: any HTML or XML escaping or quoting is the
+        responsibility of the expression itself.
+        """
+
+    def evaluateValue(compiled_expression):
+        """Evaluate an arbitrary expression.
+
+        No constraints are imposed on the return value.
+        """
+
+    def createErrorInfo(exception, (lineno, offset)):
+        """Returns an ITALESErrorInfo object.
+
+        The returned object is used to provide information about the
+        error condition for the on-error handler.
+        """
+
+    def setGlobal(name, value):
+        """Set a global variable.
+
+        The variable will be named 'name' and have the value 'value'.
+        """
+
+    def setLocal(name, value):
+        """Set a local variable in the current scope.
+
+        The variable will be named 'name' and have the value 'value'.
+        """
+
+    def setRepeat(name, compiled_expression):
+        """
+        """
+
+    def translate(domain, msgid, mapping):
+        """
+        See ITranslationService.translate()
+        """
+
+
+class ITALESErrorInfo(Interface):
+    
+    type = Attribute("type",
+                     "The exception class.")
+
+    value = Attribute("value",
+                      "The exception instance.")
+
+    lineno = Attribute("lineno",
+                       "The line number the error occurred on in the source.")
+
+    offset = Attribute("offset",
+                       "The character offset at which the error occurred.")

lib/python/TAL/TALDefs.py

@@ -8,7 +8,7 @@
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
+# FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
 """
@@ -17,13 +17,16 @@ Common definitions used by TAL and METAL compilation an transformation.
 
 from types import ListType, TupleType
 
-TAL_VERSION = "1.3.2"
+from ITALES import ITALESErrorInfo
+
+TAL_VERSION = "1.4"
 
 XML_NS = "http://www.w3.org/XML/1998/namespace" # URI for XML namespace
 XMLNS_NS = "http://www.w3.org/2000/xmlns/" # URI for XML NS declarations
 
 ZOPE_TAL_NS = "http://xml.zope.org/namespaces/tal"
 ZOPE_METAL_NS = "http://xml.zope.org/namespaces/metal"
+ZOPE_I18N_NS = "http://xml.zope.org/namespaces/i18n"
 
 NAME_RE = "[a-zA-Z_][a-zA-Z0-9_]*"
 
@@ -32,7 +35,6 @@ KNOWN_METAL_ATTRIBUTES = [
     "use-macro",
     "define-slot",
     "fill-slot",
-    "slot",
     ]
 
 KNOWN_TAL_ATTRIBUTES = [
@@ -47,6 +49,16 @@ KNOWN_TAL_ATTRIBUTES = [
     "tal tag",
     ]
 
+KNOWN_I18N_ATTRIBUTES = [
+    "translate",
+    "domain",
+    "target",
+    "source",
+    "attributes",
+    "data",
+    "name",
+    ]
+
 class TALError(Exception):
 
     def __init__(self, msg, position=(None, None)):
@@ -54,6 +66,10 @@ class TALError(Exception):
         self.msg = msg
         self.lineno = position[0]
         self.offset = position[1]
+        self.filename = None
+
+    def setFile(self, filename):
+        self.filename = filename
 
     def __str__(self):
         result = self.msg
@@ -61,6 +77,8 @@ class TALError(Exception):
             result = result + ", at line %d" % self.lineno
         if self.offset is not None:
             result = result + ", column %d" % (self.offset + 1)
+        if self.filename is not None:
+            result = result + ', in file %s' % self.filename
         return result
 
 class METALError(TALError):
@@ -69,9 +87,14 @@ class METALError(TALError):
 class TALESError(TALError):
     pass
 
+class I18NError(TALError):
+    pass
+
 
 class ErrorInfo:
 
+    __implements__ = ITALESErrorInfo
+
     def __init__(self, err, position=(None, None)):
         if isinstance(err, Exception):
             self.type = err.__class__

lib/python/TAL/TALGenerator.py

@@ -8,7 +8,7 @@
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
+# FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
 """
@@ -18,7 +18,16 @@ Code generator for TALInterpreter intermediate code.
 import re
 import cgi
 
-from TALDefs import *
+import TALDefs
+
+from TALDefs import NAME_RE, TAL_VERSION
+from TALDefs import I18NError, METALError, TALError
+from TALDefs import parseSubstitution
+from TranslationContext import TranslationContext, DEFAULT_DOMAIN
+
+I18N_REPLACE = 1
+I18N_CONTENT = 2
+I18N_EXPRESSION = 3
 
 class TALGenerator:
 
@@ -32,8 +41,15 @@ class TALGenerator:
             expressionCompiler = DummyEngine()
         self.expressionCompiler = expressionCompiler
         self.CompilerError = expressionCompiler.getCompilerError()
+        # This holds the emitted opcodes representing the input
         self.program = []
+        # The program stack for when we need to do some sub-evaluation for an
+        # intermediate result.  E.g. in an i18n:name tag for which the
+        # contents describe the ${name} value.
         self.stack = []
+        # Another stack of postponed actions.  Elements on this stack are a
+        # dictionary; key/values contain useful information that
+        # emitEndElement needs to finish its calculations
         self.todoStack = []
         self.macros = {}
         self.slots = {}
@@ -44,6 +60,7 @@ class TALGenerator:
         if source_file is not None:
             self.source_file = source_file
             self.emit("setSourceFile", source_file)
+        self.i18nContext = TranslationContext()
 
     def getCode(self):
         assert not self.stack
@@ -82,6 +99,12 @@ class TALGenerator:
                 # instructions to be joined together.
                 output.append(self.optimizeArgsList(item))
                 continue
+            if opcode == 'noop':
+                # This is a spacer for end tags in the face of i18n:name
+                # attributes.  We can't let the optimizer collect immediately
+                # following end tags into the same rawtextOffset.
+                opcode = None
+                pass
             text = "".join(collect)
             if text:
                 i = text.rfind("\n")
@@ -102,9 +125,28 @@ class TALGenerator:
         else:
             return item[0], tuple(item[1:])
 
-    actionIndex = {"replace":0, "insert":1, "metal":2, "tal":3, "xmlns":4,
-                   0: 0, 1: 1, 2: 2, 3: 3, 4: 4}
+    # These codes are used to indicate what sort of special actions
+    # are needed for each special attribute.  (Simple attributes don't
+    # get action codes.)
+    #
+    # The special actions (which are modal) are handled by
+    # TALInterpreter.attrAction() and .attrAction_tal().
+    #
+    # Each attribute is represented by a tuple:
+    #
+    # (name, value)                 -- a simple name/value pair, with
+    #                                  no special processing
+    #
+    # (name, value, action, *extra) -- attribute with special
+    #                                  processing needs, action is a
+    #                                  code that indicates which
+    #                                  branch to take, and *extra
+    #                                  contains additional,
+    #                                  action-specific information
+    #                                  needed by the processing
+    #
     def optimizeStartTag(self, collect, name, attrlist, end):
+        # return true if the tag can be converted to plain text
         if not attrlist:
             collect.append("<%s%s" % (name, end))
             return 1
@@ -115,18 +157,15 @@ class TALGenerator:
             if len(item) > 2:
                 opt = 0
                 name, value, action = item[:3]
-                action = self.actionIndex[action]
                 attrlist[i] = (name, value, action) + item[3:]
             else:
                 if item[1] is None:
                     s = item[0]
                 else:
-                    s = "%s=%s" % (item[0], quote(item[1]))
+                    s = "%s=%s" % (item[0], TALDefs.quote(item[1]))
                 attrlist[i] = item[0], s
-            if item[1] is None:
-                new.append(" " + item[0])
-            else:
-                new.append(" %s=%s" % (item[0], quote(item[1])))
+                new.append(" " + s)
+        # if no non-optimizable attributes were found, convert to plain text
         if opt:
             new.append(end)
             collect.extend(new)
@@ -222,7 +261,7 @@ class TALGenerator:
         self.emitRawText(cgi.escape(text))
 
     def emitDefines(self, defines):
-        for part in splitParts(defines):
+        for part in TALDefs.splitParts(defines):
             m = re.match(
                 r"(?s)\s*(?:(global|local)\s+)?(%s)\s+(.*)\Z" % NAME_RE, part)
             if not m:
@@ -274,6 +313,49 @@ class TALGenerator:
             assert key == "structure"
             self.emit("insertStructure", cexpr, attrDict, program)
 
+    def emitI18nVariable(self, varname, action, expression):
+        # Used for i18n:name attributes.  arg is extra information describing
+        # how the contents of the variable should get filled in, and it will
+        # either be a 1-tuple or a 2-tuple.  If arg[0] is None, then the
+        # i18n:name value is taken implicitly from the contents of the tag,
+        # e.g. "I live in <span i18n:name="country">the USA</span>".  In this
+        # case, arg[1] is the opcode sub-program describing the contents of
+        # the tag.
+        #
+        # When arg[0] is not None, it contains the tal expression used to
+        # calculate the contents of the variable, e.g.
+        # "I live in <span i18n:name="country"
+        #                  tal:replace="here/countryOfOrigin" />"
+        key = cexpr = None
+        program = self.popProgram()
+        if action == I18N_REPLACE:
+            # This is a tag with an i18n:name and a tal:replace (implicit or
+            # explicit).  Get rid of the first and last elements of the
+            # program, which are the start and end tag opcodes of the tag.
+            program = program[1:-1]
+        elif action == I18N_CONTENT:
+            # This is a tag with an i18n:name and a tal:content
+            # (explicit-only).  Keep the first and last elements of the
+            # program, so we keep the start and end tag output.
+            pass
+        else:
+            assert action == I18N_EXPRESSION
+            key, expr = parseSubstitution(expression)
+            cexpr = self.compileExpression(expr)
+        # XXX Would key be anything but 'text' or None?
+        assert key in ('text', None)
+        self.emit('i18nVariable', varname, program, cexpr)
+
+    def emitTranslation(self, msgid, i18ndata):
+        program = self.popProgram()
+        if i18ndata is None:
+            self.emit('insertTranslation', msgid, program)
+        else:
+            key, expr = parseSubstitution(i18ndata)
+            cexpr = self.compileExpression(expr)
+            assert key == 'text'
+            self.emit('insertTranslation', msgid, program, cexpr)
+
     def emitDefineMacro(self, macroName):
         program = self.popProgram()
         macroName = macroName.strip()
@@ -361,23 +443,30 @@ class TALGenerator:
         return None
 
     def replaceAttrs(self, attrlist, repldict):
+        # Each entry in attrlist starts like (name, value).
+        # Result is (name, value, action, expr, xlat) if there is a
+        # tal:attributes entry for that attribute.  Additional attrs
+        # defined only by tal:attributes are added here.
+        #
+        # (name, value, action, expr, xlat)
         if not repldict:
             return attrlist
         newlist = []
         for item in attrlist:
             key = item[0]
             if repldict.has_key(key):
-                item = item[:2] + ("replace", repldict[key])
+                expr, xlat = repldict[key]
+                item = item[:2] + ("replace", expr, xlat)
                 del repldict[key]
             newlist.append(item)
-        for key, value in repldict.items(): # Add dynamic-only attributes
-            item = (key, None, "insert", value)
-            newlist.append(item)
+        # Add dynamic-only attributes
+        for key, (expr, xlat) in repldict.items():
+            newlist.append((key, None, "insert", expr, xlat))
         return newlist
 
-    def emitStartElement(self, name, attrlist, taldict, metaldict,
+    def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict,
                          position=(None, None), isend=0):
-        if not taldict and not metaldict:
+        if not taldict and not metaldict and not i18ndict:
             # Handle the simple, common case
             self.emitStartTag(name, attrlist, isend)
             self.todoPush({})
@@ -387,18 +476,24 @@ class TALGenerator:
 
         self.position = position
         for key, value in taldict.items():
-            if key not in KNOWN_TAL_ATTRIBUTES:
+            if key not in TALDefs.KNOWN_TAL_ATTRIBUTES:
                 raise TALError("bad TAL attribute: " + `key`, position)
             if not (value or key == 'omit-tag'):
                 raise TALError("missing value for TAL attribute: " +
                                `key`, position)
         for key, value in metaldict.items():
-            if key not in KNOWN_METAL_ATTRIBUTES:
+            if key not in TALDefs.KNOWN_METAL_ATTRIBUTES:
                 raise METALError("bad METAL attribute: " + `key`,
                                  position)
             if not value:
                 raise TALError("missing value for METAL attribute: " +
                                `key`, position)
+        for key, value in i18ndict.items():
+            if key not in TALDefs.KNOWN_I18N_ATTRIBUTES:
+                raise I18NError("bad i18n attribute: " + `key`, position)
+            if not value and key in ("attributes", "data", "id"):
+                raise I18NError("missing value for i18n attribute: " +
+                                `key`, position)
         todo = {}
         defineMacro = metaldict.get("define-macro")
         useMacro = metaldict.get("use-macro")
@@ -413,12 +508,30 @@ class TALGenerator:
         onError = taldict.get("on-error")
         omitTag = taldict.get("omit-tag")
         TALtag = taldict.get("tal tag")
+        i18nattrs = i18ndict.get("attributes")
+        # Preserve empty string if implicit msgids are used.  We'll generate
+        # code with the msgid='' and calculate the right implicit msgid during
+        # interpretation phase.
+        msgid = i18ndict.get("translate")
+        varname = i18ndict.get('name')
+        i18ndata = i18ndict.get('data')
+
+        if i18ndata and not msgid:
+            raise I18NError("i18n:data must be accompanied by i18n:translate",
+                            position)
+
         if len(metaldict) > 1 and (defineMacro or useMacro):
             raise METALError("define-macro and use-macro cannot be used "
                              "together or with define-slot or fill-slot",
                              position)
-        if content and replace:
-            raise TALError("content and replace are mutually exclusive",
+        if replace:
+            if content:
+                raise TALError(
+                    "tal:content and tal:replace are mutually exclusive",
+                    position)
+            if msgid is not None:
+                raise I18NError(
+                    "i18n:translate and tal:replace are mutually exclusive",
                     position)
 
         repeatWhitespace = None
@@ -437,7 +550,7 @@ class TALGenerator:
                 self.inMacroUse = 0
         else:
             if fillSlot:
-                raise METALError, ("fill-slot must be within a use-macro",
+                raise METALError("fill-slot must be within a use-macro",
                                  position)
         if not self.inMacroUse:
             if defineMacro:
@@ -455,13 +568,29 @@ class TALGenerator:
                 self.inMacroUse = 1
             if defineSlot:
                 if not self.inMacroDef:
-                    raise METALError, (
+                    raise METALError(
                         "define-slot must be within a define-macro",
                         position)
                 self.pushProgram()
                 todo["defineSlot"] = defineSlot
 
-        if taldict:
+        if defineSlot or i18ndict:
+            
+            domain = i18ndict.get("domain") or self.i18nContext.domain
+            source = i18ndict.get("source") or self.i18nContext.source
+            target = i18ndict.get("target") or self.i18nContext.target
+            if (  domain != DEFAULT_DOMAIN
+                  or source is not None
+                  or target is not None):
+                self.i18nContext = TranslationContext(self.i18nContext,
+                                                      domain=domain,
+                                                      source=source,
+                                                      target=target)
+                self.emit("beginI18nContext",
+                          {"domain": domain, "source": source,
+                           "target": target})
+                todo["i18ncontext"] = 1
+        if taldict or i18ndict:
             dict = {}
             for item in attrlist:
                 key, value = item[:2]
@@ -487,16 +616,43 @@ class TALGenerator:
         if content:
             todo["content"] = content
         if replace:
+            # tal:replace w/ i18n:name has slightly different semantics.  What
+            # we're actually replacing then is the contents of the ${name}
+            # placeholder.
+            if varname:
+                todo['i18nvar'] = (varname, replace)
+            else:
                 todo["replace"] = replace
             self.pushProgram()
+        # i18n:name w/o tal:replace uses the content as the interpolation
+        # dictionary values
+        elif varname:
+            todo['i18nvar'] = (varname, None)
+            self.pushProgram()
+        if msgid is not None:
+            todo['msgid'] = msgid
+        if i18ndata:
+            todo['i18ndata'] = i18ndata
         optTag = omitTag is not None or TALtag
         if optTag:
             todo["optional tag"] = omitTag, TALtag
             self.pushProgram()
+        if attrsubst or i18nattrs:
             if attrsubst:
-            repldict = parseAttributeReplacements(attrsubst)
+                repldict = TALDefs.parseAttributeReplacements(attrsubst)
+            else:
+                repldict = {}
+            if i18nattrs:
+                i18nattrs = i18nattrs.split()
+            else:
+                i18nattrs = ()
+            # Convert repldict's name-->expr mapping to a
+            # name-->(compiled_expr, translate) mapping
             for key, value in repldict.items():
-                repldict[key] = self.compileExpression(value)
+                repldict[key] = self.compileExpression(value), key in i18nattrs
+            for key in i18nattrs:
+                if not repldict.has_key(key):
+                    repldict[key] = None, 1
         else:
             repldict = {}
         if replace:
@@ -507,6 +663,8 @@ class TALGenerator:
             self.pushProgram()
         if content:
             self.pushProgram()
+        if msgid is not None:
+            self.pushProgram()
         if todo and position != (None, None):
             todo["position"] = position
         self.todoPush(todo)
@@ -535,6 +693,10 @@ class TALGenerator:
         repldict = todo.get("repldict", {})
         scope = todo.get("scope")
         optTag = todo.get("optional tag")
+        msgid = todo.get('msgid')
+        i18ncontext = todo.get("i18ncontext")
+        varname = todo.get('i18nvar')
+        i18ndata = todo.get('i18ndata')
 
         if implied > 0:
             if defineMacro or useMacro or defineSlot or fillSlot:
@@ -546,14 +708,51 @@ class TALGenerator:
             raise exc("%s attributes on <%s> require explicit </%s>" %
                       (what, name, name), position)
 
+        # If there's no tal:content or tal:replace in the tag with the
+        # i18n:name, tal:replace is the default.
+        i18nNameAction = I18N_REPLACE
         if content:
+            if varname:
+                i18nNameAction = I18N_CONTENT
             self.emitSubstitution(content, {})
+        # If we're looking at an implicit msgid, emit the insertTranslation
+        # opcode now, so that the end tag doesn't become part of the implicit
+        # msgid.  If we're looking at an explicit msgid, it's better to emit
+        # the opcode after the i18nVariable opcode so we can better handle
+        # tags with both of them in them (and in the latter case, the contents
+        # would be thrown away for msgid purposes).
+        if msgid is not None and not varname:
+            self.emitTranslation(msgid, i18ndata)
         if optTag:
             self.emitOptTag(name, optTag, isend)
         elif not isend:
+            # If we're processing the end tag for a tag that contained
+            # i18n:name, we need to make sure that optimize() won't collect
+            # immediately following end tags into the same rawtextOffset, so
+            # put a spacer here that the optimizer will recognize.
+            if varname:
+                self.emit('noop')
             self.emitEndTag(name)
+        # If i18n:name appeared in the same tag as tal:replace then we're
+        # going to do the substitution a little bit differently.  The results
+        # of the expression go into the i18n substitution dictionary.
         if replace:
             self.emitSubstitution(replace, repldict)
+        elif varname:
+            if varname[1] is not None:
+                i18nNameAction = I18N_EXPRESSION
+            # o varname[0] is the variable name
+            # o i18nNameAction is either
+            #   - I18N_REPLACE for implicit tal:replace
+            #   - I18N_CONTENT for tal:content
+            #   - I18N_EXPRESSION for explicit tal:replace
+            # o varname[1] will be None for the first two actions and the
+            #   replacement tal expression for the third action.
+            self.emitI18nVariable(varname[0], i18nNameAction, varname[1])
+        # Do not test for "msgid is not None", i.e. we only want to test for
+        # explicit msgids here.  See comment above.
+        if msgid is not None and varname:
+            self.emitTranslation(msgid, i18ndata)
         if repeat:
             self.emitRepeat(repeat)
         if condition:
@@ -562,6 +761,10 @@ class TALGenerator:
             self.emitOnError(name, onError)
         if scope:
             self.emit("endScope")
+        if i18ncontext:
+            self.emit("endI18nContext")
+            assert self.i18nContext.parent is not None
+            self.i18nContext = self.i18nContext.parent
         if defineSlot:
             self.emitDefineSlot(defineSlot)
         if fillSlot:

lib/python/TAL/TALInterpreter.py

@@ -8,7 +8,7 @@
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
+# FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
 """
@@ -17,17 +17,17 @@ Interpreter for a pre-compiled TAL program.
 
 import sys
 import getopt
-
+import re
+from types import ListType
 from cgi import escape
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
+# Do not use cStringIO here!  It's not unicode aware. :(
+from StringIO import StringIO
+from DocumentTemplate.DT_Util import ustr
 
 from TALDefs import quote, TAL_VERSION, TALError, METALError
 from TALDefs import isCurrentVersion, getProgramVersion, getProgramMode
 from TALGenerator import TALGenerator
+from TranslationContext import TranslationContext
 
 BOOLEAN_HTML_ATTRS = [
     # List of Boolean attributes in HTML that should be rendered in
@@ -40,13 +40,12 @@ BOOLEAN_HTML_ATTRS = [
     "defer"
 ]
 
-EMPTY_HTML_TAGS = [
-    # List of HTML tags with an empty content model; these are
-    # rendered in minimized form, e.g. <img />.
-    # From http://www.w3.org/TR/xhtml1/#dtds
-    "base", "meta", "link", "hr", "br", "param", "img", "area",
-    "input", "col", "basefont", "isindex", "frame",
-]
+def normalize(text):
+    # Now we need to normalize the whitespace in implicit message ids and
+    # implicit $name substitution values by stripping leading and trailing
+    # whitespace, and folding all internal whitespace to a single space.
+    return ' '.join(text.split())
+
 
 class AltTALGenerator(TALGenerator):
 
@@ -62,14 +61,16 @@ class AltTALGenerator(TALGenerator):
         if self.enabled:
             apply(TALGenerator.emit, (self,) + args)
 
-    def emitStartElement(self, name, attrlist, taldict, metaldict,
+    def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict,
                          position=(None, None), isend=0):
         metaldict = {}
         taldict = {}
+        i18ndict = {}
         if self.enabled and self.repldict:
             taldict["attributes"] = "x x"
         TALGenerator.emitStartElement(self, name, attrlist,
-                                      taldict, metaldict, position, isend)
+                                      taldict, metaldict, i18ndict,
+                                      position, isend)
 
     def replaceAttrs(self, attrlist, repldict):
         if self.enabled and self.repldict:
@@ -82,10 +83,10 @@ class TALInterpreter:
 
     def __init__(self, program, macros, engine, stream=None,
                  debug=0, wrap=60, metal=1, tal=1, showtal=-1,
-                 strictinsert=1, stackLimit=100):
+                 strictinsert=1, stackLimit=100, i18nInterpolate=1):
         self.program = program
         self.macros = macros
-        self.engine = engine
+        self.engine = engine # Execution engine (aka context)
         self.Default = engine.getDefault()
         self.stream = stream or sys.stdout
         self._stream_write = self.stream.write
@@ -107,12 +108,14 @@ class TALInterpreter:
         self.endsep = "/>"
         self.endlen = len(self.endsep)
         self.macroStack = []
-        self.popMacro = self.macroStack.pop
         self.position = None, None  # (lineno, offset)
         self.col = 0
         self.level = 0
         self.scopeLevel = 0
         self.sourceFile = None
+        self.i18nStack = []
+        self.i18nInterpolate = i18nInterpolate
+        self.i18nContext = TranslationContext()
 
     def StringIO(self):
         # Third-party products wishing to provide a full Unicode-aware
@@ -121,19 +124,20 @@ class TALInterpreter:
 
     def saveState(self):
         return (self.position, self.col, self.stream,
-                self.scopeLevel, self.level)
+                self.scopeLevel, self.level, self.i18nContext)
 
     def restoreState(self, state):
-        (self.position, self.col, self.stream, scopeLevel, level) = state
+        (self.position, self.col, self.stream, scopeLevel, level, i18n) = state
         self._stream_write = self.stream.write
         assert self.level == level
         while self.scopeLevel > scopeLevel:
             self.engine.endScope()
             self.scopeLevel = self.scopeLevel - 1
         self.engine.setPosition(self.position)
+        self.i18nContext = i18n
 
     def restoreOutputState(self, state):
-        (dummy, self.col, self.stream, scopeLevel, level) = state
+        (dummy, self.col, self.stream, scopeLevel, level, i18n) = state
         self._stream_write = self.stream.write
         assert self.level == level
         assert self.scopeLevel == scopeLevel
@@ -142,7 +146,12 @@ class TALInterpreter:
         if len(self.macroStack) >= self.stackLimit:
             raise METALError("macro nesting limit (%d) exceeded "
                              "by %s" % (self.stackLimit, `macroName`))
-        self.macroStack.append([macroName, slots, entering])
+        self.macroStack.append([macroName, slots, entering, self.i18nContext])
+
+    def popMacro(self):
+        stuff = self.macroStack.pop()
+        self.i18nContext = stuff[3]
+        return stuff
 
     def macroContext(self, what):
         macroStack = self.macroStack
@@ -156,9 +165,11 @@ class TALInterpreter:
     def __call__(self):
         assert self.level == 0
         assert self.scopeLevel == 0
+        assert self.i18nContext.parent is None
         self.interpret(self.program)
         assert self.level == 0
         assert self.scopeLevel == 0
+        assert self.i18nContext.parent is None
         if self.col > 0:
             self._stream_write("\n")
             self.col = 0
@@ -174,14 +185,19 @@ class TALInterpreter:
 
     bytecode_handlers = {}
 
-    def interpret(self, program, None=None):
+    def interpret(self, program, tmpstream=None):
         oldlevel = self.level
         self.level = oldlevel + 1
         handlers = self.dispatch
+        if tmpstream:
+            ostream = self.stream
+            owrite = self._stream_write
+            self.stream = tmpstream
+            self._stream_write = tmpstream.write
         try:
             if self.debug:
                 for (opcode, args) in program:
-                    s = "%sdo_%s%s\n" % ("    "*self.level, opcode,
+                    s = "%sdo_%s(%s)\n" % ("    "*self.level, opcode,
                                            repr(args))
                     if len(s) > 80:
                         s = s[:76] + "...\n"
@@ -192,6 +208,9 @@ class TALInterpreter:
                     handlers[opcode](self, args)
         finally:
             self.level = oldlevel
+            if tmpstream:
+                self.stream = ostream
+                self._stream_write = owrite
 
     def do_version(self, version):
         assert version == TAL_VERSION
@@ -229,8 +248,7 @@ class TALInterpreter:
         # for that case.
         _stream_write = self._stream_write
         _stream_write("<" + name)
-        namelen = _len(name)
-        col = self.col + namelen + 1
+        col = self.col + _len(name) + 1
         wrap = self.wrap
         align = col + 1
         if align >= wrap/2:
@@ -262,10 +280,11 @@ class TALInterpreter:
 
     def attrAction(self, item):
         name, value, action = item[:3]
-        if action == 1 or (action > 1 and not self.showtal):
+        if action == 'insert' or (action in ('metal', 'tal', 'xmlns', 'i18n')
+                                  and not self.showtal):
             return 0, name, value
         macs = self.macroStack
-        if action == 2 and self.metal and macs:
+        if action == 'metal' and self.metal and macs:
             if len(macs) > 1 or not macs[-1][2]:
                 # Drop all METAL attributes at a use-depth above one.
                 return 0, name, value
@@ -293,33 +312,43 @@ class TALInterpreter:
 
     def attrAction_tal(self, item):
         name, value, action = item[:3]
-        if action > 1:
+        if action in ('metal', 'tal', 'xmlns', 'i18n'):
             return self.attrAction(item)
         ok = 1
+        expr, msgid = item[3:]
         if self.html and name.lower() in BOOLEAN_HTML_ATTRS:
             evalue = self.engine.evaluateBoolean(item[3])
             if evalue is self.Default:
-                if action == 1: # Cancelled insert
+                if action == 'insert': # Cancelled insert
                     ok = 0
             elif evalue:
                 value = None
             else:
                 ok = 0
         else:
+            if expr is not None:
                 evalue = self.engine.evaluateText(item[3])
                 if evalue is self.Default:
-                if action == 1: # Cancelled insert
+                    if action == 'insert': # Cancelled insert
                         ok = 0
                 else:
                     if evalue is None:
                         ok = 0
                     value = evalue
         if ok:
+            if msgid:
+                value = self.i18n_attribute(value)
             if value is None:
                 value = name
             value = "%s=%s" % (name, quote(value))
         return ok, name, value
 
+    def i18n_attribute(self, s):
+        # s is the value of an attribute before translation
+        # it may have been computed
+        return self.translate(s, {})
+
+
     bytecode_handlers["<attrAction>"] = attrAction
 
     def no_tag(self, start, program):
@@ -354,7 +383,7 @@ class TALInterpreter:
     def dumpMacroStack(self, prefix, suffix, value):
         sys.stderr.write("+---- %s%s = %s\n" % (prefix, suffix, value))
         for i in range(len(self.macroStack)):
-            what, macroName, slots = self.macroStack[i]
+            what, macroName, slots = self.macroStack[i][:3]
             sys.stderr.write("| %2d. %-12s %-12s %s\n" %
                              (i, what, macroName, slots and slots.keys()))
         sys.stderr.write("+--------------------------------------\n")
@@ -412,6 +441,19 @@ class TALInterpreter:
         self.engine.setGlobal(name, self.engine.evaluateValue(expr))
     bytecode_handlers["setGlobal"] = do_setLocal
 
+    def do_beginI18nContext(self, settings):
+        get = settings.get
+        self.i18nContext = TranslationContext(self.i18nContext,
+                                              domain=get("domain"),
+                                              source=get("source"),
+                                              target=get("target"))
+    bytecode_handlers["beginI18nContext"] = do_beginI18nContext
+
+    def do_endI18nContext(self, notused=None):
+        self.i18nContext = self.i18nContext.parent
+        assert self.i18nContext is not None
+    bytecode_handlers["endI18nContext"] = do_endI18nContext
+
     def do_insertText(self, stuff):
         self.interpret(stuff[1])
 
@@ -431,6 +473,72 @@ class TALInterpreter:
             self.col = len(s) - (i + 1)
     bytecode_handlers["insertText"] = do_insertText
 
+    def do_i18nVariable(self, stuff):
+        varname, program, expression = stuff
+        if expression is None:
+            # The value is implicitly the contents of this tag, so we have to
+            # evaluate the mini-program to get the value of the variable.
+            state = self.saveState()
+            try:
+                tmpstream = self.StringIO()
+                self.interpret(program, tmpstream)
+                value = normalize(tmpstream.getvalue())
+            finally:
+                self.restoreState(state)
+        else:
+            # Evaluate the value to be associated with the variable in the
+            # i18n interpolation dictionary.
+            value = self.engine.evaluate(expression)
+        # Either the i18n:name tag is nested inside an i18n:translate in which
+        # case the last item on the stack has the i18n dictionary and string
+        # representation, or the i18n:name and i18n:translate attributes are
+        # in the same tag, in which case the i18nStack will be empty.  In that
+        # case we can just output the ${name} to the stream
+        i18ndict, srepr = self.i18nStack[-1]
+        i18ndict[varname] = value
+        placeholder = '${%s}' % varname
+        srepr.append(placeholder)
+        self._stream_write(placeholder)
+    bytecode_handlers['i18nVariable'] = do_i18nVariable
+
+    def do_insertTranslation(self, stuff):
+        i18ndict = {}
+        srepr = []
+        obj = None
+        self.i18nStack.append((i18ndict, srepr))
+        msgid = stuff[0]
+        # We need to evaluate the content of the tag because that will give us
+        # several useful pieces of information.  First, the contents will
+        # include an implicit message id, if no explicit one was given.
+        # Second, it will evaluate any i18nVariable definitions in the body of
+        # the translation (necessary for $varname substitutions).
+        #
+        # Use a temporary stream to capture the interpretation of the
+        # subnodes, which should /not/ go to the output stream.
+        tmpstream = self.StringIO()
+        self.interpret(stuff[1], tmpstream)
+        # We only care about the evaluated contents if we need an implicit
+        # message id.  All other useful information will be in the i18ndict on
+        # the top of the i18nStack.
+        if msgid == '':
+            msgid = normalize(tmpstream.getvalue())
+        self.i18nStack.pop()
+        # See if there is was an i18n:data for msgid
+        if len(stuff) > 2:
+            obj = self.engine.evaluate(stuff[2])
+        xlated_msgid = self.translate(msgid, i18ndict, obj)
+        # XXX I can't decide whether we want to cgi escape the translated
+        # string or not.  OT1H not doing this could introduce a cross-site
+        # scripting vector by allowing translators to sneak JavaScript into
+        # translations.  OTOH, for implicit interpolation values, we don't
+        # want to escape stuff like ${name} <= "<b>Timmy</b>".
+        #s = escape(xlated_msgid)
+        s = xlated_msgid
+        # If there are i18n variables to interpolate into this string, better
+        # do it now.
+        self._stream_write(s)
+    bytecode_handlers['insertTranslation'] = do_insertTranslation
+
     def do_insertStructure(self, stuff):
         self.interpret(stuff[2])
 
@@ -441,7 +549,7 @@ class TALInterpreter:
         if structure is self.Default:
             self.interpret(block)
             return
-        text = str(structure)
+        text = ustr(structure)
         if not (repldict or self.strictinsert):
             # Take a shortcut, no error checking
             self.stream_write(text)
@@ -482,6 +590,24 @@ class TALInterpreter:
             self.interpret(block)
     bytecode_handlers["loop"] = do_loop
 
+    def translate(self, msgid, i18ndict=None, obj=None):
+        # XXX is this right?
+        if i18ndict is None:
+            i18ndict = {}
+        if obj:
+            i18ndict.update(obj)
+        # XXX need to fill this in with TranslationService calls.  For now,
+        # we'll just do simple interpolation based on a $-strings to %-strings
+        # algorithm in Mailman.
+        if not self.i18nInterpolate:
+            return msgid
+        # XXX Mmmh, it seems that sometimes the msgid is None; is that really
+        # possible?
+        if msgid is None:
+            return None
+        # XXX We need to pass in one of context or target_language
+        return self.engine.translate(self.i18nContext.domain, msgid, i18ndict)
+
     def do_rawtextColumn(self, (s, col)):
         self._stream_write(s)
         self.col = col
@@ -504,6 +630,7 @@ class TALInterpreter:
             if not entering:
                 macs.append(None)
                 self.interpret(macro)
+                assert macs[-1] is None
                 macs.pop()
                 return
         self.interpret(macro)
@@ -526,12 +653,11 @@ class TALInterpreter:
                 raise METALError("macro %s has incompatible mode %s" %
                                  (`macroName`, `mode`), self.position)
         self.pushMacro(macroName, compiledSlots)
-        saved_source = self.sourceFile
-        saved_position = self.position  # Used by Boa Constructor
+        prev_source = self.sourceFile
         self.interpret(macro)
-        if self.sourceFile != saved_source:
-            self.engine.setSourceFile(saved_source)
-            self.sourceFile = saved_source
+        if self.sourceFile != prev_source:
+            self.engine.setSourceFile(prev_source)
+            self.sourceFile = prev_source
         self.popMacro()
     bytecode_handlers["useMacro"] = do_useMacro
 
@@ -547,21 +673,18 @@ class TALInterpreter:
             return
         macs = self.macroStack
         if macs and macs[-1] is not None:
-            saved_source = self.sourceFile
-            saved_position = self.position  # Used by Boa Constructor
             macroName, slots = self.popMacro()[:2]
             slot = slots.get(slotName)
             if slot is not None:
+                prev_source = self.sourceFile
                 self.interpret(slot)
-                if self.sourceFile != saved_source:
-                    self.engine.setSourceFile(saved_source)
-                    self.sourceFile = saved_source
+                if self.sourceFile != prev_source:
+                    self.engine.setSourceFile(prev_source)
+                    self.sourceFile = prev_source
                 self.pushMacro(macroName, slots, entering=0)
                 return
             self.pushMacro(macroName, slots)
-            if len(macs) == 1:
-                self.interpret(block)
-                return
+            # Falling out of the 'if' allows the macro to be interpreted.
         self.interpret(block)
     bytecode_handlers["defineSlot"] = do_defineSlot
 

lib/python/TAL/TALParser.py

@@ -8,7 +8,7 @@
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
+# FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
 """
@@ -16,7 +16,7 @@ Parse XML and compile to TALInterpreter intermediate code.
 """
 
 from XMLParser import XMLParser
-from TALDefs import *
+from TALDefs import XML_NS, ZOPE_I18N_NS, ZOPE_METAL_NS, ZOPE_TAL_NS
 from TALGenerator import TALGenerator
 
 class TALParser(XMLParser):
@@ -58,13 +58,15 @@ class TALParser(XMLParser):
             # attrs is a dict of {name: value}
             attrlist = attrs.items()
             attrlist.sort() # For definiteness
-        name, attrlist, taldict, metaldict = self.process_ns(name, attrlist)
+        name, attrlist, taldict, metaldict, i18ndict \
+              = self.process_ns(name, attrlist)
         attrlist = self.xmlnsattrs() + attrlist
-        self.gen.emitStartElement(name, attrlist, taldict, metaldict)
+        self.gen.emitStartElement(name, attrlist, taldict, metaldict, i18ndict)
 
     def process_ns(self, name, attrlist):
         taldict = {}
         metaldict = {}
+        i18ndict = {}
         fixedattrlist = []
         name, namebase, namens = self.fixname(name)
         for key, value in attrlist:
@@ -77,10 +79,14 @@ class TALParser(XMLParser):
             elif ns == 'tal':
                 taldict[keybase] = value
                 item = item + ("tal",)
+            elif ns == 'i18n':
+                assert 0, "dealing with i18n: " + `(keybase, value)`
+                i18ndict[keybase] = value
+                item = item + ('i18n',)
             fixedattrlist.append(item)
-        if namens in ('metal', 'tal'):
+        if namens in ('metal', 'tal', 'i18n'):
             taldict['tal tag'] = namens
-        return name, fixedattrlist, taldict, metaldict
+        return name, fixedattrlist, taldict, metaldict, i18ndict
 
     def xmlnsattrs(self):
         newlist = []
@@ -89,7 +95,7 @@ class TALParser(XMLParser):
                 key = "xmlns:" + prefix
             else:
                 key = "xmlns"
-            if uri in (ZOPE_METAL_NS, ZOPE_TAL_NS):
+            if uri in (ZOPE_METAL_NS, ZOPE_TAL_NS, ZOPE_I18N_NS):
                 item = (key, uri, "xmlns")
             else:
                 item = (key, uri)
@@ -109,6 +115,8 @@ class TALParser(XMLParser):
                 ns = 'tal'
             elif uri == ZOPE_METAL_NS:
                 ns = 'metal'
+            elif uri == ZOPE_I18N_NS:
+                ns = 'i18n'
             return (prefixed, name, ns)
         return (name, name, None)
 

lib/python/TAL/TranslationContext.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Translation context object for the TALInterpreter's I18N support.
+
+The translation context provides a container for the information
+needed to perform translation of a marked string from a page template.
+
+$Id: TranslationContext.py,v 1.2 2002/09/18 15:12:48 efge Exp $
+"""
+
+DEFAULT_DOMAIN = "default"
+
+class TranslationContext:
+    """Information about the I18N settings of a TAL processor."""
+
+    def __init__(self, parent=None, domain=None, target=None, source=None):
+        if parent:
+            if not domain:
+                domain = parent.domain
+            if not target:
+                target = parent.target
+            if not source:
+                source = parent.source
+        elif domain is None:
+            domain = DEFAULT_DOMAIN
+
+        self.parent = parent
+        self.domain = domain
+        self.target = target
+        self.source = source

lib/python/TAL/driver.py

@@ -9,11 +9,31 @@
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
+# FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
 """
 Driver program to test METAL and TAL implementation.
+
+Usage: driver.py [options] [file]
+Options:
+    -h / --help
+        Print this message and exit.
+    -H / --html
+    -x / --xml
+        Explicitly choose HTML or XML input.  The default is to automatically
+        select based on the file extension.  These options are mutually
+        exclusive.
+    -l
+        Lenient structure insertion.
+    -m
+        Macro expansion only
+    -s
+        Print intermediate opcodes only
+    -t
+        Leave TAL/METAL attributes in output
+    -i
+        Leave I18N substitution strings un-interpolated.
 """
 
 import os
@@ -26,68 +46,131 @@ if __name__ == "__main__":
 
 # Import local classes
 import TALDefs
-import DummyEngine
+from DummyEngine import DummyEngine
+from DummyEngine import DummyTranslationService
 
 FILE = "tests/input/test01.xml"
 
+class TestTranslations(DummyTranslationService):
+    def translate(self, domain, msgid, mapping=None, context=None,
+                  target_language=None):
+        if msgid == 'timefmt':
+            return '%(minutes)s minutes after %(hours)s %(ampm)s' % mapping
+        elif msgid == 'jobnum':
+            return '%(jobnum)s is the JOB NUMBER' % mapping
+        elif msgid == 'verify':
+            s = 'Your contact email address is recorded as %(email)s'
+            return s % mapping
+        elif msgid == 'mailto:${request/submitter}':
+            return 'mailto:bperson@dom.ain'
+        elif msgid == 'origin':
+            return '%(name)s was born in %(country)s' % mapping
+        return DummyTranslationService.translate(self, domain, msgid,
+                                                 mapping, context,
+                                                 target_language)
+
+class TestEngine(DummyEngine):
+    def __init__(self, macros=None):
+        DummyEngine.__init__(self, macros)
+        self.translationService = TestTranslations()
+
+    def evaluatePathOrVar(self, expr):
+        if expr == 'here/currentTime':
+            return {'hours'  : 6,
+                    'minutes': 59,
+                    'ampm'   : 'PM',
+                    }
+        elif expr == 'context/@@object_name':
+            return '7'
+        elif expr == 'request/submitter':
+            return 'aperson@dom.ain'
+        return DummyEngine.evaluatePathOrVar(self, expr)
+
+
+# This is a disgusting hack so that we can use engines that actually know
+# something about certain object paths.  TimeEngine knows about
+# here/currentTime.
+ENGINES = {'test23.html': TestEngine,
+           'test24.html': TestEngine,
+           'test26.html': TestEngine,
+           'test27.html': TestEngine,
+           'test28.html': TestEngine,
+           'test29.html': TestEngine,
+           'test30.html': TestEngine,
+           'test31.html': TestEngine,
+           'test32.html': TestEngine,
+           }
+
+def usage(code, msg=''):
+    # Python 2.1 required
+    print >> sys.stderr, __doc__
+    if msg:
+        print >> sys.stderr, msg
+    sys.exit(code)
+
 def main():
-    versionTest = 1
     macros = 0
     mode = None
     showcode = 0
     showtal = -1
     strictinsert = 1
+    i18nInterpolate = 1
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "hxlmnst")
+        opts, args = getopt.getopt(sys.argv[1:], "hHxlmsti",
+                                   ['help', 'html', 'xml'])
     except getopt.error, msg:
-        sys.stderr.write("\n%s\n" % str(msg))
-        sys.stderr.write(
-            "usage: driver.py [-h|-x] [-l] [-m] [-n] [-s] [-t] [file]\n")
-        sys.stderr.write("-h/-x -- HTML/XML input (default auto)\n")
-        sys.stderr.write("-l -- lenient structure insertion\n")
-        sys.stderr.write("-m -- macro expansion only\n")
-        sys.stderr.write("-n -- turn off the Python 1.5.2 test\n")
-        sys.stderr.write("-s -- print intermediate code\n")
-        sys.stderr.write("-t -- leave tal/metal attributes in output\n")
-        sys.exit(2)
-    for o, a in opts:
-        if o == '-h':
+        usage(2, msg)
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+        if opt in ('-H', '--html'):
+            if mode == 'xml':
+                usage(1, '--html and --xml are mutually exclusive')
             mode = "html"
-        if o == '-l':
+        if opt == '-l':
             strictinsert = 0
-        if o == '-m':
+        if opt == '-m':
             macros = 1
-        if o == '-n':
+        if opt == '-n':
             versionTest = 0
-        if o == '-x':
+        if opt in ('-x', '--xml'):
+            if mode == 'html':
+                usage(1, '--html and --xml are mutually exclusive')
             mode = "xml"
-        if o == '-s':
+        if opt == '-s':
             showcode = 1
-        if o == '-t':
+        if opt == '-t':
             showtal = 1
-    if not versionTest:
-        if sys.version[:5] != "1.5.2":
-            sys.stderr.write(
-                "Use Python 1.5.2 only; use -n to disable this test\n")
-            sys.exit(2)
+        if opt == '-i':
+            i18nInterpolate = 0
     if args:
         file = args[0]
     else:
         file = FILE
     it = compilefile(file, mode)
-    if showcode: showit(it)
-    else: interpretit(it, tal=(not macros), showtal=showtal,
-                      strictinsert=strictinsert)
+    if showcode:
+        showit(it)
+    else:
+        # See if we need a special engine for this test
+        engine = None
+        engineClass = ENGINES.get(os.path.basename(file))
+        if engineClass is not None:
+            engine = engineClass(macros)
+        interpretit(it, engine=engine,
+                    tal=(not macros), showtal=showtal,
+                    strictinsert=strictinsert,
+                    i18nInterpolate=i18nInterpolate)
 
 def interpretit(it, engine=None, stream=None, tal=1, showtal=-1,
-                strictinsert=1):
+                strictinsert=1, i18nInterpolate=1):
     from TALInterpreter import TALInterpreter
     program, macros = it
     assert TALDefs.isCurrentVersion(program)
     if engine is None:
-        engine = DummyEngine.DummyEngine(macros)
+        engine = DummyEngine(macros)
     TALInterpreter(program, macros, engine, stream, wrap=0,
-                   tal=tal, showtal=showtal, strictinsert=strictinsert)()
+                   tal=tal, showtal=showtal, strictinsert=strictinsert,
+                   i18nInterpolate=i18nInterpolate)()
 
 def compilefile(file, mode=None):
     assert mode in ("html", "xml", None)

lib/python/TAL/tests/test_files.py

@@ -17,6 +17,10 @@ class FileTestCase(unittest.TestCase):
         self.__dir = dir
         unittest.TestCase.__init__(self)
 
+    def shortDescription(self):
+        return os.path.join("...", "TAL", "tests", "input",
+                            os.path.basename(self.__file))
+
     def runTest(self):
         basename = os.path.basename(self.__file)
         #sys.stdout.write(basename + " ")

lib/python/TAL/tests/test_htmltalparser.py

-#! /usr/bin/env python1.5
+#! /usr/bin/env python
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
 """Tests for the HTMLTALParser code generator."""
 
 import pprint
@@ -7,7 +20,6 @@ import sys
 from TAL.tests import utils
 import unittest
 
-from string import rfind
 
 from TAL import HTMLTALParser
 from TAL.TALDefs import TAL_VERSION, TALError, METALError
@@ -24,7 +36,7 @@ class TestCaseBase(unittest.TestCase):
         if p1 and p2:
             op1, args1 = p1[-1]
             op2, args2 = p2[0]
-            if op1[:7] == 'rawtext' and op2[:7] == 'rawtext':
+            if op1.startswith('rawtext') and op2.startswith('rawtext'):
                 return (p1[:-1]
                         + [rawtext(args1[0] + args2[0])]
                         + p2[1:])
@@ -60,7 +72,7 @@ class TestCaseBase(unittest.TestCase):
 def rawtext(s):
     """Compile raw text to the appropriate instruction."""
     if "\n" in s:
-        return ("rawtextColumn", (s, len(s) - (rfind(s, "\n") + 1)))
+        return ("rawtextColumn", (s, len(s) - (s.rfind("\n") + 1)))
     else:
         return ("rawtextOffset", (s, len(s)))
 
@@ -100,25 +112,6 @@ class HTMLTALParserTestCases(TestCaseBase):
             self._run_check("<a><b></a></b>", [])
         self.assertRaises(HTMLTALParser.NestingError, check)
 
-    def check_cdata_mode(self):
-        """This routine should NOT detect an error with an end tag </a></b> not
-           matching the start <script> tag.  The contents are within a
-           HTML comment, and should be ignored.
-        """
-        # The above comment is not generally true.  The HTML 4 specification
-        # gives <script> a CDATA content model, which means comments are not
-        # syntactically recognized (those characters contribute to the text
-        # content of the <script> element).  The '</a' in the '</a>' causes
-        # the SGML markup-in-context rules to kick in, and '</a>' should then
-        # be recognized as an improperly nested end tag.  See:
-        # http://www.w3.org/TR/html401/types.html#type-cdata
-        #
-        s = """<html><script>\n<!--\ndocument.write("</a></b>");\n// -->\n</script></html>"""
-        output = [
-            rawtext(s),
-            ]
-        self._run_check(s, output)
-
     def check_code_attr_syntax(self):
         output = [
             rawtext('<a b="v" c="v" d="v" e></a>'),
@@ -174,7 +167,7 @@ class METALGeneratorTestCases(TestCaseBase):
 
     def check_define_macro(self):
         macro = self.initial_program + [
-            ('startTag', ('p', [('metal:define-macro', 'M', 2)])),
+            ('startTag', ('p', [('metal:define-macro', 'M', 'metal')])),
             rawtext('booh</p>'),
             ]
         program = [
@@ -189,17 +182,17 @@ class METALGeneratorTestCases(TestCaseBase):
             ('setPosition', (1, 0)),
             ('useMacro',
              ('M', '$M$', {},
-              [('startTag', ('p', [('metal:use-macro', 'M', 2)])),
+              [('startTag', ('p', [('metal:use-macro', 'M', 'metal')])),
                rawtext('booh</p>')])),
             ])
 
     def check_define_slot(self):
         macro = self.initial_program + [
-            ('startTag', ('p', [('metal:define-macro', 'M', 2)])),
+            ('startTag', ('p', [('metal:define-macro', 'M', 'metal')])),
             rawtext('foo'),
             ('setPosition', (1, 29)),
             ('defineSlot', ('S',
-             [('startTag', ('span', [('metal:define-slot', 'S', 2)])),
+             [('startTag', ('span', [('metal:define-slot', 'S', 'metal')])),
               rawtext('spam</span>')])),
             rawtext('bar</p>'),
             ]
@@ -217,13 +210,13 @@ class METALGeneratorTestCases(TestCaseBase):
             ('useMacro',
              ('M', '$M$',
               {'S': [('startTag', ('span',
-                                   [('metal:fill-slot', 'S', 2)])),
+                                   [('metal:fill-slot', 'S', 'metal')])),
                      rawtext('spam</span>')]},
-             [('startTag', ('p', [('metal:use-macro', 'M', 2)])),
+             [('startTag', ('p', [('metal:use-macro', 'M', 'metal')])),
               rawtext('foo'),
               ('setPosition', (1, 26)),
               ('fillSlot', ('S',
-               [('startTag', ('span', [('metal:fill-slot', 'S', 2)])),
+               [('startTag', ('span', [('metal:fill-slot', 'S', 'metal')])),
                 rawtext('spam</span>')])),
               rawtext('bar</p>')])),
             ])
@@ -239,7 +232,7 @@ class TALGeneratorTestCases(TestCaseBase):
             ('setPosition', (1, 0)),
             ('beginScope', {'tal:define': 'xyzzy string:spam'}),
             ('setLocal', ('xyzzy', '$string:spam$')),
-            ('startTag', ('p', [('tal:define', 'xyzzy string:spam', 3)])),
+            ('startTag', ('p', [('tal:define', 'xyzzy string:spam', 'tal')])),
             ('endScope', ()),
             rawtext('</p>'),
             ])
@@ -250,7 +243,7 @@ class TALGeneratorTestCases(TestCaseBase):
             ('beginScope', {'tal:define': 'local xyzzy string:spam'}),
             ('setLocal', ('xyzzy', '$string:spam$')),
             ('startTag', ('p',
-             [('tal:define', 'local xyzzy string:spam', 3)])),
+             [('tal:define', 'local xyzzy string:spam', 'tal')])),
             ('endScope', ()),
             rawtext('</p>'),
             ])
@@ -261,7 +254,7 @@ class TALGeneratorTestCases(TestCaseBase):
             ('beginScope', {'tal:define': 'global xyzzy string:spam'}),
             ('setGlobal', ('xyzzy', '$string:spam$')),
             ('startTag', ('p',
-             [('tal:define', 'global xyzzy string:spam', 3)])),
+             [('tal:define', 'global xyzzy string:spam', 'tal')])),
             ('endScope', ()),
             rawtext('</p>'),
             ])
@@ -272,7 +265,7 @@ class TALGeneratorTestCases(TestCaseBase):
             ('beginScope', {'tal:define': 'x string:spam; y x'}),
             ('setLocal', ('x', '$string:spam$')),
             ('setLocal', ('y', '$x$')),
-            ('startTag', ('p', [('tal:define', 'x string:spam; y x', 3)])),
+            ('startTag', ('p', [('tal:define', 'x string:spam; y x', 'tal')])),
             ('endScope', ()),
             rawtext('</p>'),
             ])
@@ -283,7 +276,7 @@ class TALGeneratorTestCases(TestCaseBase):
             ('beginScope', {'tal:define': 'x string:;;;;; y x'}),
             ('setLocal', ('x', '$string:;;$')),
             ('setLocal', ('y', '$x$')),
-            ('startTag', ('p', [('tal:define', 'x string:;;;;; y x', 3)])),
+            ('startTag', ('p', [('tal:define', 'x string:;;;;; y x', 'tal')])),
             ('endScope', ()),
             rawtext('</p>'),
             ])
@@ -298,7 +291,7 @@ class TALGeneratorTestCases(TestCaseBase):
             ('setGlobal', ('y', '$x$')),
             ('setLocal', ('z', '$y$')),
             ('startTag', ('p',
-             [('tal:define', 'x string:spam; global y x; local z y', 3)])),
+             [('tal:define', 'x string:spam; global y x; local z y', 'tal')])),
             ('endScope', ()),
             rawtext('</p>'),
             ])
@@ -310,7 +303,7 @@ class TALGeneratorTestCases(TestCaseBase):
             ('setPosition', (1, 3)),
             ('beginScope', {'tal:condition': 'python:1'}),
             ('condition', ('$python:1$',
-             [('startTag', ('span', [('tal:condition', 'python:1', 3)])),
+             [('startTag', ('span', [('tal:condition', 'python:1', 'tal')])),
               rawtext('<b>foo</b></span>')])),
             ('endScope', ()),
             rawtext('</p>'),
@@ -320,7 +313,7 @@ class TALGeneratorTestCases(TestCaseBase):
         self._run_check("<p tal:content='string:foo'>bar</p>", [
              ('setPosition', (1, 0)),
              ('beginScope', {'tal:content': 'string:foo'}),
-             ('startTag', ('p', [('tal:content', 'string:foo', 3)])),
+             ('startTag', ('p', [('tal:content', 'string:foo', 'tal')])),
              ('insertText', ('$string:foo$', [rawtext('bar')])),
              ('endScope', ()),
              rawtext('</p>'),
@@ -330,7 +323,7 @@ class TALGeneratorTestCases(TestCaseBase):
         self._run_check("<p tal:content='text string:foo'>bar</p>", [
              ('setPosition', (1, 0)),
              ('beginScope', {'tal:content': 'text string:foo'}),
-             ('startTag', ('p', [('tal:content', 'text string:foo', 3)])),
+             ('startTag', ('p', [('tal:content', 'text string:foo', 'tal')])),
              ('insertText', ('$string:foo$', [rawtext('bar')])),
              ('endScope', ()),
              rawtext('</p>'),
@@ -341,7 +334,7 @@ class TALGeneratorTestCases(TestCaseBase):
              ('setPosition', (1, 0)),
              ('beginScope', {'tal:content': 'structure string:<br>'}),
              ('startTag', ('p',
-              [('tal:content', 'structure string:<br>', 3)])),
+              [('tal:content', 'structure string:<br>', 'tal')])),
              ('insertStructure',
               ('$string:<br>$', {}, [rawtext('bar')])),
              ('endScope', ()),
@@ -353,7 +346,7 @@ class TALGeneratorTestCases(TestCaseBase):
              ('setPosition', (1, 0)),
              ('beginScope', {'tal:replace': 'string:foo'}),
              ('insertText', ('$string:foo$',
-              [('startTag', ('p', [('tal:replace', 'string:foo', 3)])),
+              [('startTag', ('p', [('tal:replace', 'string:foo', 'tal')])),
                rawtext('bar</p>')])),
              ('endScope', ()),
              ])
@@ -364,7 +357,7 @@ class TALGeneratorTestCases(TestCaseBase):
              ('beginScope', {'tal:replace': 'text string:foo'}),
              ('insertText', ('$string:foo$',
               [('startTag', ('p',
-                             [('tal:replace', 'text string:foo', 3)])),
+                             [('tal:replace', 'text string:foo', 'tal')])),
                rawtext('bar</p>')])),
              ('endScope', ()),
              ])
@@ -375,7 +368,7 @@ class TALGeneratorTestCases(TestCaseBase):
              ('beginScope', {'tal:replace': 'structure string:<br>'}),
              ('insertStructure', ('$string:<br>$', {},
               [('startTag', ('p',
-                [('tal:replace', 'structure string:<br>', 3)])),
+                [('tal:replace', 'structure string:<br>', 'tal')])),
                rawtext('bar</p>')])),
              ('endScope', ()),
              ])
@@ -387,11 +380,11 @@ class TALGeneratorTestCases(TestCaseBase):
              ('beginScope', {'tal:repeat': 'x python:(1,2,3)'}),
              ('loop', ('x', '$python:(1,2,3)$',
               [('startTag', ('p',
-                             [('tal:repeat', 'x python:(1,2,3)', 3)])),
+                             [('tal:repeat', 'x python:(1,2,3)', 'tal')])),
                ('setPosition', (1, 33)),
                ('beginScope', {'tal:replace': 'x'}),
                ('insertText', ('$x$',
-                [('startTag', ('span', [('tal:replace', 'x', 3)])),
+                [('startTag', ('span', [('tal:replace', 'x', 'tal')])),
                  rawtext('dummy</span>')])),
                ('endScope', ()),
                rawtext('</p>')])),
@@ -407,11 +400,11 @@ class TALGeneratorTestCases(TestCaseBase):
              {'tal:attributes': 'href string:http://www.zope.org; x string:y',
               'name': 'bar', 'href': 'foo'}),
             ('startTag', ('a',
-             [('href', 'foo', 0, '$string:http://www.zope.org$'),
+             [('href', 'foo', 'replace', '$string:http://www.zope.org$', 0),
               ('name', 'name="bar"'),
               ('tal:attributes',
-               'href string:http://www.zope.org; x string:y', 3),
-              ('x', None, 1, '$string:y$')])),
+               'href string:http://www.zope.org; x string:y', 'tal'),
+              ('x', None, 'insert', '$string:y$', 0)])),
             ('endScope', ()),
             rawtext('link</a>'),
             ])
@@ -423,11 +416,13 @@ class TALGeneratorTestCases(TestCaseBase):
             ('beginScope',
              {'tal:attributes': 'src string:foo.png',
               'tal:replace': 'structure string:<img>'}),
-            ('insertStructure', ('$string:<img>$',
-             {'src': '$string:foo.png$'},
+            ('insertStructure',
+             ('$string:<img>$',
+              {'src': ('$string:foo.png$', 0)},
               [('startTag', ('p',
-               [('tal:replace', 'structure string:<img>', 3),
-                ('tal:attributes', 'src string:foo.png', 3)])),
+                             [('tal:replace', 'structure string:<img>', 'tal'),
+                              ('tal:attributes', 'src string:foo.png',
+                               'tal')])),
                rawtext('duh</p>')])),
             ('endScope', ()),
             ])
@@ -440,13 +435,13 @@ class TALGeneratorTestCases(TestCaseBase):
              {'tal:content': 'notHere', 'tal:on-error': 'string:error'}),
             ('onError',
              ([('startTag', ('p',
-                [('tal:on-error', 'string:error', 3),
-                 ('tal:content', 'notHere', 3)])),
+                [('tal:on-error', 'string:error', 'tal'),
+                 ('tal:content', 'notHere', 'tal')])),
                ('insertText', ('$notHere$', [rawtext('okay')])),
                rawtext('</p>')],
               [('startTag', ('p',
-                [('tal:on-error', 'string:error', 3),
-                 ('tal:content', 'notHere', 3)])),
+                [('tal:on-error', 'string:error', 'tal'),
+                 ('tal:content', 'notHere', 'tal')])),
                ('insertText', ('$string:error$', [])),
                rawtext('</p>')])),
             ('endScope', ()),
@@ -461,12 +456,12 @@ class TALGeneratorTestCases(TestCaseBase):
             ('onError',
              ([('insertText', ('$notHere$',
                 [('startTag', ('p',
-                  [('tal:on-error', 'string:error', 3),
-                   ('tal:replace', 'notHere', 3)])),
+                  [('tal:on-error', 'string:error', 'tal'),
+                   ('tal:replace', 'notHere', 'tal')])),
                  rawtext('okay</p>')]))],
               [('startTag', ('p',
-                [('tal:on-error', 'string:error', 3),
-                 ('tal:replace', 'notHere', 3)])),
+                [('tal:on-error', 'string:error', 'tal'),
+                 ('tal:replace', 'notHere', 'tal')])),
                ('insertText', ('$string:error$', [])),
                rawtext('</p>')])),
             ('endScope', ()),
@@ -492,6 +487,365 @@ class TALGeneratorTestCases(TestCaseBase):
         self._should_error("<p metal:foobar='x' />", exc)
         self._should_error("<p metal:define-macro='x'>", exc)
 
+    #
+    #  I18N test cases
+    #
+
+    def check_i18n_attributes(self):
+        self._run_check("<img alt='foo' i18n:attributes='alt'>", [
+            ('setPosition', (1, 0)),
+            ('beginScope', {'alt': 'foo', 'i18n:attributes': 'alt'}),
+            ('startTag', ('img',
+             [('alt', 'foo', 'replace', None, 1),
+              ('i18n:attributes', 'alt', 'i18n')])),
+            ('endScope', ()),
+            ])
+
+    def check_i18n_translate(self):
+        # input/test19.html
+        self._run_check('''\
+<span i18n:translate="">Replace this</span>
+<span i18n:translate="msgid">This is a
+translated string</span>
+<span i18n:translate="">And another
+translated string</span>
+''', [
+  ('setPosition', (1, 0)),
+  ('beginScope', {'i18n:translate': ''}),
+  ('startTag', ('span', [('i18n:translate', '', 'i18n')])),
+  ('insertTranslation', ('', [('rawtextOffset', ('Replace this', 12))])),
+  ('rawtextBeginScope',
+   ('</span>\n', 0, (2, 0), 1, {'i18n:translate': 'msgid'})),
+  ('startTag', ('span', [('i18n:translate', 'msgid', 'i18n')])),
+  ('insertTranslation',
+   ('msgid', [('rawtextColumn', ('This is a\ntranslated string', 17))])),
+  ('rawtextBeginScope', ('</span>\n', 0, (4, 0), 1, {'i18n:translate': ''})),
+  ('startTag', ('span', [('i18n:translate', '', 'i18n')])),
+  ('insertTranslation',
+   ('', [('rawtextColumn', ('And another\ntranslated string', 17))])),
+  ('endScope', ()),
+  ('rawtextColumn', ('</span>\n', 0))])
+
+    def check_i18n_translate_with_nested_tal(self):
+        self._run_check('''\
+<span i18n:translate="">replaceable <p tal:replace="str:here">content</p></span>
+''', [
+  ('setPosition', (1, 0)),
+  ('beginScope', {'i18n:translate': ''}),
+  ('startTag', ('span', [('i18n:translate', '', 'i18n')])),
+  ('insertTranslation',
+   ('',
+    [('rawtextOffset', ('replaceable ', 12)),
+     ('setPosition', (1, 36)),
+     ('beginScope', {'tal:replace': 'str:here'}),
+     ('insertText',
+      ('$str:here$',
+       [('startTag', ('p', [('tal:replace', 'str:here', 'tal')])),
+        ('rawtextOffset', ('content</p>', 11))])),
+     ('endScope', ())])),
+  ('endScope', ()),
+  ('rawtextColumn', ('</span>\n', 0))
+  ])
+
+    def check_i18n_name(self):
+        # input/test21.html
+        self._run_check('''\
+<span i18n:translate="">
+  <span tal:replace="str:Lomax" i18n:name="name" /> was born in
+  <span tal:replace="str:Antarctica" i18n:name="country" />.
+</span>
+''', [
+  ('setPosition', (1, 0)),
+  ('beginScope', {'i18n:translate': ''}),
+  ('startTag', ('span', [('i18n:translate', '', 'i18n')])),
+  ('insertTranslation',
+   ('',
+    [('rawtextBeginScope',
+      ('\n  ',
+       2,
+       (2, 2),
+       0,
+       {'i18n:name': 'name', 'tal:replace': 'str:Lomax'})),
+     ('i18nVariable',
+      ('name',
+       [('startEndTag',
+         ('span',
+          [('tal:replace', 'str:Lomax', 'tal'),
+           ('i18n:name', 'name', 'i18n')]))],
+       '$str:Lomax$')),
+     ('rawtextBeginScope',
+      (' was born in\n  ',
+       2,
+       (3, 2),
+       1,
+       {'i18n:name': 'country', 'tal:replace': 'str:Antarctica'})),
+     ('i18nVariable',
+      ('country',
+       [('startEndTag',
+         ('span',
+          [('tal:replace', 'str:Antarctica', 'tal'),
+           ('i18n:name', 'country', 'i18n')]))],
+       '$str:Antarctica$')),
+     ('endScope', ()),
+     ('rawtextColumn', ('.\n', 0))])),
+  ('endScope', ()),
+  ('rawtextColumn', ('</span>\n', 0))
+  ])
+
+    def check_i18n_name_implicit_value(self):
+        # input/test22.html
+        self._run_check('''\
+<span i18n:translate="">
+  <span i18n:name="name"><b>Jim</b></span> was born in
+  <span i18n:name="country">the USA</span>.
+</span>
+''', [
+  ('setPosition', (1, 0)),
+  ('beginScope', {'i18n:translate': ''}),
+  ('startTag', ('span', [('i18n:translate', '', 'i18n')])),
+  ('insertTranslation',
+   ('',
+    [('rawtextBeginScope', ('\n  ', 2, (2, 2), 0, {'i18n:name': 'name'})),
+     ('i18nVariable',
+      ('name',
+       [('rawtextOffset', ('<b>Jim</b>', 10))], None)),
+     ('rawtextBeginScope',
+      (' was born in\n  ', 2, (3, 2), 1, {'i18n:name': 'country'})),
+     ('i18nVariable',
+      ('country',
+       [('rawtextOffset', ('the USA', 7))], None)),
+     ('endScope', ()),
+     ('rawtextColumn', ('.\n', 0))])),
+  ('endScope', ()),
+  ('rawtextColumn', ('</span>\n', 0))
+  ])
+
+    def check_i18n_context_domain(self):
+        self._run_check("<span i18n:domain='mydomain'/>", [
+            ('setPosition', (1, 0)),
+            ('beginI18nContext', {'domain': 'mydomain',
+                                  'source': None, 'target': None}),
+            ('beginScope', {'i18n:domain': 'mydomain'}),
+            ('startEndTag', ('span', [('i18n:domain', 'mydomain', 'i18n')])),
+            ('endScope', ()),
+            ('endI18nContext', ()),
+            ])
+
+    def check_i18n_context_source(self):
+        self._run_check("<span i18n:source='en'/>", [
+            ('setPosition', (1, 0)),
+            ('beginI18nContext', {'source': 'en',
+                                  'domain': 'default', 'target': None}),
+            ('beginScope', {'i18n:source': 'en'}),
+            ('startEndTag', ('span', [('i18n:source', 'en', 'i18n')])),
+            ('endScope', ()),
+            ('endI18nContext', ()),
+            ])
+
+    def check_i18n_context_source_target(self):
+        self._run_check("<span i18n:source='en' i18n:target='ru'/>", [
+            ('setPosition', (1, 0)),
+            ('beginI18nContext', {'source': 'en', 'target': 'ru',
+                                  'domain': 'default'}),
+            ('beginScope', {'i18n:source': 'en', 'i18n:target': 'ru'}),
+            ('startEndTag', ('span', [('i18n:source', 'en', 'i18n'),
+                                      ('i18n:target', 'ru', 'i18n')])),
+            ('endScope', ()),
+            ('endI18nContext', ()),
+            ])
+
+    def check_i18n_context_in_define_slot(self):
+        text = ("<div metal:use-macro='M' i18n:domain='mydomain'>"
+                "<div metal:fill-slot='S'>spam</div>"
+                "</div>")
+        self._run_check(text, [
+            ('setPosition', (1, 0)),
+            ('useMacro',
+             ('M', '$M$',
+              {'S': [('startTag', ('div',
+                                   [('metal:fill-slot', 'S', 'metal')])),
+                     rawtext('spam</div>')]},
+              [('beginI18nContext', {'domain': 'mydomain',
+                                     'source': None, 'target': None}),
+               ('beginScope',
+                {'i18n:domain': 'mydomain', 'metal:use-macro': 'M'}),
+               ('startTag', ('div', [('metal:use-macro', 'M', 'metal'),
+                                     ('i18n:domain', 'mydomain', 'i18n')])),
+               ('setPosition', (1, 48)),
+               ('fillSlot', ('S',
+                             [('startTag',
+                               ('div', [('metal:fill-slot', 'S', 'metal')])),
+                              rawtext('spam</div>')])),
+               ('endScope', ()),
+               rawtext('</div>'),
+               ('endI18nContext', ())])),
+            ])
+
+    def check_i18n_data(self):
+        # input/test23.html
+        self._run_check('''\
+<span i18n:data="here/currentTime"
+      i18n:translate="timefmt">2:32 pm</span>
+''', [
+  ('setPosition', (1, 0)),
+  ('beginScope',
+   {'i18n:translate': 'timefmt', 'i18n:data': 'here/currentTime'}),
+  ('startTag',
+   ('span',
+    [('i18n:data', 'here/currentTime', 'i18n'),
+     ('i18n:translate', 'timefmt', 'i18n')])),
+  ('insertTranslation',
+   ('timefmt', [('rawtextOffset', ('2:32 pm', 7))], '$here/currentTime$')),
+  ('endScope', ()),
+  ('rawtextColumn', ('</span>\n', 0))
+  ])
+
+    def check_i18n_data_with_name(self):
+        # input/test29.html
+        self._run_check('''\
+At the tone the time will be
+<span i18n:data="here/currentTime"
+      i18n:translate="timefmt"
+      i18n:name="time">2:32 pm</span>... beep!
+''', [
+  ('rawtextBeginScope',
+   ('At the tone the time will be\n',
+    0,
+    (2, 0),
+    0,
+    {'i18n:data': 'here/currentTime',
+     'i18n:name': 'time',
+     'i18n:translate': 'timefmt'})),
+  ('insertTranslation',
+   ('timefmt',
+    [('startTag',
+      ('span',
+       [('i18n:data', 'here/currentTime', 'i18n'),
+        ('i18n:translate', 'timefmt', 'i18n'),
+        ('i18n:name', 'time', 'i18n')])),
+     ('i18nVariable', ('time', [], None))],
+    '$here/currentTime$')),
+  ('endScope', ()),
+  ('rawtextColumn', ('... beep!\n', 0))
+  ])
+
+    def check_i18n_explicit_msgid_with_name(self):
+        # input/test26.html
+        self._run_check('''\
+<span i18n:translate="jobnum">
+    Job #<span tal:replace="context/@@object_name"
+               i18n:name="jobnum">NN</span></span>
+''', [
+  ('setPosition', (1, 0)),
+  ('beginScope', {'i18n:translate': 'jobnum'}),
+  ('startTag', ('span', [('i18n:translate', 'jobnum', 'i18n')])),
+  ('insertTranslation',
+   ('jobnum',
+    [('rawtextBeginScope',
+      ('\n    Job #',
+       9,
+       (2, 9),
+       0,
+       {'i18n:name': 'jobnum', 'tal:replace': 'context/@@object_name'})),
+     ('i18nVariable',
+      ('jobnum',
+       [('startTag',
+         ('span',
+          [('tal:replace', 'context/@@object_name', 'tal'),
+           ('i18n:name', 'jobnum', 'i18n')])),
+        ('rawtextOffset', ('NN', 2)),
+        ('rawtextOffset', ('</span>', 7))],
+       '$context/@@object_name$')),
+     ('endScope', ())])),
+  ('endScope', ()),
+  ('rawtextColumn', ('</span>\n', 0))
+  ])
+
+    def check_i18n_name_around_tal_content(self):
+        # input/test28.html
+        self._run_check('''\
+<p i18n:translate="verify">Your contact email address is recorded as
+    <span i18n:name="email">
+    <a href="mailto:user@example.com"
+       tal:content="request/submitter">user@host.com</a></span>
+</p>
+''', [
+  ('setPosition', (1, 0)),
+  ('beginScope', {'i18n:translate': 'verify'}),
+  ('startTag', ('p', [('i18n:translate', 'verify', 'i18n')])),
+  ('insertTranslation',
+   ('verify',
+    [('rawtextBeginScope',
+      ('Your contact email address is recorded as\n    ',
+       4,
+       (2, 4),
+       0,
+       {'i18n:name': 'email'})),
+     ('i18nVariable',
+      ('email',
+       [('rawtextBeginScope',
+         ('\n    ',
+          4,
+          (3, 4),
+          0,
+          {'href': 'mailto:user@example.com',
+           'tal:content': 'request/submitter'})),
+        ('startTag',
+         ('a',
+          [('href', 'href="mailto:user@example.com"'),
+           ('tal:content', 'request/submitter', 'tal')])),
+        ('insertText',
+         ('$request/submitter$',
+          [('rawtextOffset', ('user@host.com', 13))])),
+        ('endScope', ()),
+        ('rawtextOffset', ('</a>', 4))],
+       None)),
+     ('endScope', ()),
+     ('rawtextColumn', ('\n', 0))])),
+  ('endScope', ()),
+  ('rawtextColumn', ('</p>\n', 0))
+  ])
+
+    def check_i18n_name_with_tal_content(self):
+        # input/test27.html
+        self._run_check('''\
+<p i18n:translate="verify">Your contact email address is recorded as
+    <a href="mailto:user@example.com"
+       tal:content="request/submitter"
+       i18n:name="email">user@host.com</a>
+</p>
+''', [
+  ('setPosition', (1, 0)),
+  ('beginScope', {'i18n:translate': 'verify'}),
+  ('startTag', ('p', [('i18n:translate', 'verify', 'i18n')])),
+  ('insertTranslation',
+   ('verify',
+    [('rawtextBeginScope',
+      ('Your contact email address is recorded as\n    ',
+       4,
+       (2, 4),
+       0,
+       {'href': 'mailto:user@example.com',
+        'i18n:name': 'email',
+        'tal:content': 'request/submitter'})),
+     ('i18nVariable',
+      ('email',
+       [('startTag',
+         ('a',
+          [('href', 'href="mailto:user@example.com"'),
+           ('tal:content', 'request/submitter', 'tal'),
+           ('i18n:name', 'email', 'i18n')])),
+        ('insertText',
+         ('$request/submitter$',
+          [('rawtextOffset', ('user@host.com', 13))])),
+        ('rawtextOffset', ('</a>', 4))],
+       None)),
+     ('endScope', ()),
+     ('rawtextColumn', ('\n', 0))])),
+  ('endScope', ()),
+  ('rawtextColumn', ('</p>\n', 0))
+  ])
+
 
 def test_suite():
     suite = unittest.TestSuite()

lib/python/TAL/tests/test_talinterpreter.py

@@ -68,6 +68,24 @@ class OutputPresentationTestCase(TestCaseBase):
         interp()
         self.assertEqual(sio.getvalue(), EXPECTED)
 
+    def check_unicode_content(self):
+        INPUT = """<p tal:content="python:u'dj-vu'">para</p>"""
+        EXPECTED = u"""<p>dj-vu</p>""" "\n"
+        program, macros = self._compile(INPUT)
+        sio = StringIO()
+        interp = TALInterpreter(program, {}, DummyEngine(), sio, wrap=60)
+        interp()
+        self.assertEqual(sio.getvalue(), EXPECTED)
+
+    def check_unicode_structure(self):
+        INPUT = """<p tal:replace="structure python:u'dj-vu'">para</p>"""
+        EXPECTED = u"""dj-vu""" "\n"
+        program, macros = self._compile(INPUT)
+        sio = StringIO()
+        interp = TALInterpreter(program, {}, DummyEngine(), sio, wrap=60)
+        interp()
+        self.assertEqual(sio.getvalue(), EXPECTED)
+
 
 def test_suite():
     suite = unittest.TestSuite()