merge from gotcha-talz3_backport-branch

backport of TAL fixes from z3

- i18n and metal interactions

- fix handling of nested translations with tal:content/replace and i18n:name

some reformatting to ease comparisons between 2.x and 3

lib/python/TAL/TALGenerator.py

@@ -64,6 +64,7 @@ class TALGenerator:
             self.source_file = source_file
             self.emit("setSourceFile", source_file)
         self.i18nContext = TranslationContext()
+        self.i18nLevel = 0
 
     def getCode(self):
         assert not self.stack
@@ -73,7 +74,7 @@ class TALGenerator:
     def optimize(self, program):
         output = []
         collect = []
-        rawseen = cursor = 0
+        cursor = 0
         if self.xml:
             endsep = "/>"
         else:
@@ -118,7 +119,6 @@ class TALGenerator:
                     output.append(("rawtextOffset", (text, len(text))))
             if opcode != None:
                 output.append(self.optimizeArgsList(item))
-            rawseen = cursor+1
             collect = []
         return self.optimizeCommonTriple(output)
 
@@ -180,9 +180,9 @@ class TALGenerator:
         output = program[:2]
         prev2, prev1 = output
         for item in program[2:]:
-            if (  item[0] == "beginScope"
-                  and prev1[0] == "setPosition"
-                  and prev2[0] == "rawtextColumn"):
+            if ( item[0] == "beginScope"
+                 and prev1[0] == "setPosition"
+                 and prev2[0] == "rawtextColumn"):
                 position = output.pop()[1]
                 text, column = output.pop()[1]
                 prev1 = None, None
@@ -319,7 +319,7 @@ class TALGenerator:
             assert key == "structure"
             self.emit("insertStructure", cexpr, attrDict, program)
 
-    def emitI18nVariable(self, varname, action, expression):
+    def emitI18nVariable(self, stuff):
         # Used for i18n:name attributes.  arg is extra information describing
         # how the contents of the variable should get filled in, and it will
         # either be a 1-tuple or a 2-tuple.  If arg[0] is None, then the
@@ -332,6 +332,7 @@ class TALGenerator:
         # calculate the contents of the variable, e.g.
         # "I live in <span i18n:name="country"
         #                  tal:replace="here/countryOfOrigin" />"
+        varname, action, expression = stuff
         m = _name_rx.match(varname)
         if m is None or m.group() != varname:
             raise TALError("illegal i18n:name: %r" % varname, self.position)
@@ -525,6 +526,11 @@ class TALGenerator:
         varname = i18ndict.get('name')
         i18ndata = i18ndict.get('data')
 
+        if varname and not self.i18nLevel:
+            raise I18NError(
+                "i18n:name can only occur inside a translation unit",
+                position)
+
         if i18ndata and not msgid:
             raise I18NError("i18n:data must be accompanied by i18n:translate",
                             position)
@@ -584,7 +590,7 @@ class TALGenerator:
                 todo["defineSlot"] = defineSlot
 
         if defineSlot or i18ndict:
-            
+
             domain = i18ndict.get("domain") or self.i18nContext.domain
             source = i18ndict.get("source") or self.i18nContext.source
             target = i18ndict.get("target") or self.i18nContext.target
@@ -627,22 +633,28 @@ class TALGenerator:
             if repeatWhitespace:
                 self.emitText(repeatWhitespace)
         if content:
-            todo["content"] = content
-        if replace:
+            if varname:
+                todo['i18nvar'] = (varname, I18N_CONTENT, None)
+                todo["content"] = content
+                self.pushProgram()
+            else:
+                todo["content"] = content
+        elif replace:
             # tal:replace w/ i18n:name has slightly different semantics.  What
             # we're actually replacing then is the contents of the ${name}
             # placeholder.
             if varname:
-                todo['i18nvar'] = (varname, replace)
+                todo['i18nvar'] = (varname, I18N_EXPRESSION, replace)
             else:
                 todo["replace"] = replace
             self.pushProgram()
         # i18n:name w/o tal:replace uses the content as the interpolation
         # dictionary values
         elif varname:
-            todo['i18nvar'] = (varname, None)
+            todo['i18nvar'] = (varname, I18N_REPLACE, None)
             self.pushProgram()
         if msgid is not None:
+            self.i18nLevel += 1
             todo['msgid'] = msgid
         if i18ndata:
             todo['i18ndata'] = i18ndata
@@ -682,10 +694,12 @@ class TALGenerator:
         self.emitStartTag(name, self.replaceAttrs(attrlist, repldict), isend)
         if optTag:
             self.pushProgram()
-        if content:
+        if content and not varname:
             self.pushProgram()
         if msgid is not None:
             self.pushProgram()
+        if content and varname:
+            self.pushProgram()
         if todo and position != (None, None):
             todo["position"] = position
         self.todoPush(todo)
@@ -731,10 +745,7 @@ class TALGenerator:
 
         # If there's no tal:content or tal:replace in the tag with the
         # i18n:name, tal:replace is the default.
-        i18nNameAction = I18N_REPLACE
         if content:
-            if varname:
-                i18nNameAction = I18N_CONTENT
             self.emitSubstitution(content, {})
         # If we're looking at an implicit msgid, emit the insertTranslation
         # opcode now, so that the end tag doesn't become part of the implicit
@@ -742,8 +753,14 @@ class TALGenerator:
         # the opcode after the i18nVariable opcode so we can better handle
         # tags with both of them in them (and in the latter case, the contents
         # would be thrown away for msgid purposes).
-        if msgid is not None and not varname:
-            self.emitTranslation(msgid, i18ndata)
+        #
+        # Still, we should emit insertTranslation opcode before i18nVariable
+        # in case tal:content, i18n:translate and i18n:name in the same tag
+        if msgid is not None:
+            if (not varname) or (
+                varname and (varname[1] == I18N_CONTENT)):
+                self.emitTranslation(msgid, i18ndata)
+            self.i18nLevel -= 1
         if optTag:
             self.emitOptTag(name, optTag, isend)
         elif not isend:
@@ -760,20 +777,24 @@ class TALGenerator:
         if replace:
             self.emitSubstitution(replace, repldict)
         elif varname:
-            if varname[1] is not None:
-                i18nNameAction = I18N_EXPRESSION
             # o varname[0] is the variable name
-            # o i18nNameAction is either
+            # o varname[1] is either
             #   - I18N_REPLACE for implicit tal:replace
             #   - I18N_CONTENT for tal:content
             #   - I18N_EXPRESSION for explicit tal:replace
-            # o varname[1] will be None for the first two actions and the
+            # o varname[2] will be None for the first two actions and the
             #   replacement tal expression for the third action.
-            self.emitI18nVariable(varname[0], i18nNameAction, varname[1])
+            assert (varname[1]
+                    in [I18N_REPLACE, I18N_CONTENT, I18N_EXPRESSION])
+            self.emitI18nVariable(varname)
         # Do not test for "msgid is not None", i.e. we only want to test for
         # explicit msgids here.  See comment above.
-        if msgid is not None and varname:
-            self.emitTranslation(msgid, i18ndata)
+        if msgid is not None: 
+            # in case tal:content, i18n:translate and i18n:name in the
+            # same tag insertTranslation opcode has already been
+            # emitted
+            if varname and (varname[1] <> I18N_CONTENT):
+                self.emitTranslation(msgid, i18ndata)
         if repeat:
             self.emitRepeat(repeat)
         if condition:

lib/python/TAL/TALInterpreter.py

@@ -327,7 +327,7 @@ class TALInterpreter:
                 name = prefix + "use-macro"
                 value = macs[-1][0] # Macro name
             elif suffix == "define-slot":
-                name = prefix + "slot"
+                name = prefix + "fill-slot"
             elif suffix == "fill-slot":
                 pass
             else:
@@ -418,9 +418,9 @@ class TALInterpreter:
     def do_rawtextBeginScope_tal(self, (s, col, position, closeprev, dict)):
         self._stream_write(s)
         self.col = col
-        self.position = position
-        self.engine.setPosition(position)
         engine = self.engine
+        self.position = position
+        engine.setPosition(position)
         if closeprev:
             engine.endScope()
             engine.beginScope()

lib/python/TAL/tests/input/test29.html

+<div i18n:translate="">At the tone the time will be
+<span i18n:data="here/currentTime"
+      i18n:translate="timefmt"
+      i18n:name="time">2:32 pm</span>... beep!</div>

lib/python/TAL/tests/input/test_metal1.html

@@ -54,7 +54,7 @@
 
 <xxx metal:use-macro="INNER3">
   <yyy metal:fill-slot="INNERSLOT">
-    <zzz metal:define-macro="INSLOT"><aaa metal:slot="null">INSLOT</aaa></zzz>
+    <zzz metal:define-macro="INSLOT">INSLOT</zzz>
   </yyy>
 </xxx>
 

lib/python/TAL/tests/input/test_metal4.html

+<!-- the outer element *must* be tal:something or metal:something -->
+<metal:block define-macro="page" i18n:domain="zope">
+  <title metal:define-slot="title">Z3 UI</title>
+</metal:block>

lib/python/TAL/tests/input/test_metal5.html

+<!-- the outer element *must* include tal:omit-tag='' -->
+<x tal:omit-tag="" metal:define-macro="page" i18n:domain="zope">
+  <title metal:define-slot="title">Z3 UI</title>
+</x>

lib/python/TAL/tests/input/test_metal6.html

+<metal:block define-macro="page">
+  <html i18:domain="zope">
+     <metal:block define-slot="title">Z3 UI</metal:block>
+  </html>
+</metal:block>

lib/python/TAL/tests/input/test_metal7.html

+<html metal:define-macro="page" i18n:domain="zope">
+    <x metal:define-slot="title" />
+</html>
+<html metal:use-macro="page" />
\ No newline at end of file

lib/python/TAL/tests/output/test29.html

+<div>AT THE TONE THE TIME WILL BE 59 MINUTES AFTER 6 PM... BEEP!</div>

lib/python/TAL/tests/output/test_metal1.html

@@ -22,7 +22,7 @@
 
 <span metal:use-macro="OUTER2">
   AAA
-  <xxx metal:slot="OUTERSLOT">
+  <xxx metal:fill-slot="OUTERSLOT">
   <span>INNER</span>
   </xxx>
   BBB
@@ -48,7 +48,7 @@
 
 <span metal:use-macro="OUTER3">
   AAA
-  <xxx metal:slot="OUTERSLOT">
+  <xxx metal:fill-slot="OUTERSLOT">
   <span>INNER
     <xxx>INNERSLOT</xxx>
   </span>
@@ -63,7 +63,7 @@
 </span>
 
 <span metal:use-macro="INNER3">INNER
-    <xxx metal:slot="INNERSLOT">INNERSLOT</xxx>
+    <xxx metal:fill-slot="INNERSLOT">INNERSLOT</xxx>
   </span>
 
 <span metal:use-macro="INNER3">INNER
@@ -72,8 +72,8 @@
 
 <span metal:use-macro="INNER3">INNER
     <yyy metal:fill-slot="INNERSLOT">
-    <zzz metal:define-macro="INSLOT"><aaa metal:slot="null">INSLOT</aaa></zzz>
+    <zzz metal:define-macro="INSLOT">INSLOT</zzz>
   </yyy>
   </span>
 
-<zzz metal:use-macro="INSLOT"><aaa>INSLOT</aaa></zzz>
+<zzz metal:use-macro="INSLOT">INSLOT</zzz>

lib/python/TAL/tests/output/test_metal4.html

+<!-- the outer element *must* be tal:something or metal:something -->
+<metal:block define-macro="page" i18n:domain="zope">
+  <title metal:define-slot="title">Z3 UI</title>
+</metal:block>

lib/python/TAL/tests/output/test_metal5.html

+<!-- the outer element *must* include tal:omit-tag='' -->
+<x tal:omit-tag="" metal:define-macro="page" i18n:domain="zope">
+  <title metal:define-slot="title">Z3 UI</title>
+</x>

lib/python/TAL/tests/output/test_metal6.html

+<metal:block define-macro="page">
+  <html i18:domain="zope">
+     <metal:block define-slot="title">Z3 UI</metal:block>
+  </html>
+</metal:block>

lib/python/TAL/tests/output/test_metal7.html

+<html metal:define-macro="page" i18n:domain="zope">
+    <x metal:define-slot="title" />
+</html>
+<html metal:use-macro="page" i18n:domain="zope">
+    <x metal:fill-slot="title" />
+</html>

lib/python/TAL/tests/test_htmltalparser.py

@@ -614,6 +614,38 @@ translated string</span>
   ('rawtextColumn', ('</span>\n', 0))
   ])
 
+    def test_i18n_name_with_content(self):
+        self._run_check('<div i18n:translate="">This is text for '
+            '<span i18n:translate="" tal:content="bar" i18n:name="bar_name"/>.'
+            '</div>', [
+('setPosition', (1, 0)),
+('beginScope', {'i18n:translate': ''}),
+('startTag', ('div', [('i18n:translate', '', 'i18n')])),
+('insertTranslation',
+ ('',
+  [('rawtextOffset', ('This is text for ', 17)),
+   ('setPosition', (1, 40)),
+   ('beginScope',
+    {'tal:content': 'bar', 'i18n:name': 'bar_name', 'i18n:translate': ''}),
+   ('i18nVariable',
+       ('bar_name',
+        [('startTag',
+           ('span',
+            [('i18n:translate', '', 'i18n'),
+             ('tal:content', 'bar', 'tal'),
+             ('i18n:name', 'bar_name', 'i18n')])),
+         ('insertTranslation',
+           ('',
+             [('insertText', ('$bar$', []))])),
+         ('rawtextOffset', ('</span>', 7))],
+      None)),
+   ('endScope', ()),
+   ('rawtextOffset', ('.', 1))])),
+('endScope', ()),
+('rawtextOffset', ('</div>', 6)) 
+  ])
+
+
     def check_i18n_name_implicit_value(self):
         # input/test22.html
         self._run_check('''\
@@ -725,31 +757,38 @@ translated string</span>
     def check_i18n_data_with_name(self):
         # input/test29.html
         self._run_check('''\
-At the tone the time will be
+<div i18n:translate="">At the tone the time will be
 <span i18n:data="here/currentTime"
       i18n:translate="timefmt"
-      i18n:name="time">2:32 pm</span>... beep!
-''', [
-  ('rawtextBeginScope',
-   ('At the tone the time will be\n',
-    0,
-    (2, 0),
-    0,
-    {'i18n:data': 'here/currentTime',
-     'i18n:name': 'time',
-     'i18n:translate': 'timefmt'})),
-  ('insertTranslation',
-   ('timefmt',
-    [('startTag',
-      ('span',
-       [('i18n:data', 'here/currentTime', 'i18n'),
-        ('i18n:translate', 'timefmt', 'i18n'),
-        ('i18n:name', 'time', 'i18n')])),
-     ('i18nVariable', ('time', [], None))],
-    '$here/currentTime$')),
-  ('endScope', ()),
-  ('rawtextColumn', ('... beep!\n', 0))
-  ])
+      i18n:name="time">2:32 pm</span>... beep!</div>
+''',
+[('setPosition', (1, 0)),
+ ('beginScope', {'i18n:translate': ''}),
+ ('startTag', ('div', [('i18n:translate', '', 'i18n')])),
+ ('insertTranslation',
+  ('',
+   [('rawtextBeginScope',
+     ('At the tone the time will be\n',
+      0,
+      (2, 0),
+      0,
+      {'i18n:data': 'here/currentTime',
+       'i18n:name': 'time',
+       'i18n:translate': 'timefmt'})),
+    ('insertTranslation',
+     ('timefmt',
+      [('startTag',
+        ('span',
+         [('i18n:data', 'here/currentTime', 'i18n'),
+          ('i18n:translate', 'timefmt', 'i18n'),
+          ('i18n:name', 'time', 'i18n')])),
+       ('i18nVariable', ('time', [], None))],
+      '$here/currentTime$')),
+    ('endScope', ()),
+    ('rawtextOffset', ('... beep!', 9))])),
+ ('endScope', ()),
+ ('rawtextColumn', ('</div>\n', 0))]
+)
 
     def check_i18n_explicit_msgid_with_name(self):
         # input/test26.html

lib/python/TAL/tests/test_talinterpreter.py

@@ -20,11 +20,11 @@ import unittest
 
 from StringIO import StringIO
 
-from TAL.TALDefs import METALError
+from TAL.TALDefs import METALError, I18NError
 from TAL.HTMLTALParser import HTMLTALParser
 from TAL.TALInterpreter import TALInterpreter
+from TAL.DummyEngine import DummyEngine, DummyTranslationService
 from TAL.TALInterpreter import interpolate
-from TAL.DummyEngine import DummyEngine
 
 
 class TestCaseBase(unittest.TestCase):
@@ -60,6 +60,130 @@ class MacroErrorsTestCase(TestCaseBase):
         self.macro[0] = ("version", "duh")
 
 
+class I18NCornerTestCase(TestCaseBase):
+
+    def setUp(self):
+        self.engine = DummyEngine()
+        self.engine.setLocal('bar', 'BaRvAlUe')
+
+    def _check(self, program, expected):
+        result = StringIO()
+        self.interpreter = TALInterpreter(program, {}, self.engine,
+                                          stream=result)
+        self.interpreter()
+        self.assertEqual(expected, result.getvalue())
+
+    def test_content_with_messageid_and_i18nname_and_i18ntranslate(self):
+        # Let's tell the user this is incredibly silly!
+        self.assertRaises(
+            I18NError, self._compile,
+            '<span i18n:translate="" tal:content="bar" i18n:name="bar_name"/>')
+
+    def test_content_with_plaintext_and_i18nname_and_i18ntranslate(self):
+        # Let's tell the user this is incredibly silly!
+        self.assertRaises(
+            I18NError, self._compile,
+            '<span i18n:translate="" i18n:name="color_name">green</span>')
+
+    def test_translate_static_text_as_dynamic(self):
+        program, macros = self._compile(
+            '<div i18n:translate="">This is text for '
+            '<span i18n:translate="" tal:content="bar" i18n:name="bar_name"/>.'
+            '</div>')
+        self._check(program,
+                    '<div>THIS IS TEXT FOR <span>BARVALUE</span>.</div>\n')
+
+    def test_translate_static_text_as_dynamic_from_bytecode(self):
+        program =  [('version', '1.4'),
+ ('mode', 'html'),
+('setPosition', (1, 0)),
+('beginScope', {'i18n:translate': ''}),
+('startTag', ('div', [('i18n:translate', '', 'i18n')])),
+('insertTranslation',
+ ('',
+  [('rawtextOffset', ('This is text for ', 17)),
+   ('setPosition', (1, 40)),
+   ('beginScope',
+    {'tal:content': 'bar', 'i18n:name': 'bar_name', 'i18n:translate': ''}),
+   ('i18nVariable',
+       ('bar_name',
+        [('startTag',
+           ('span',
+            [('i18n:translate', '', 'i18n'),
+             ('tal:content', 'bar', 'tal'),
+             ('i18n:name', 'bar_name', 'i18n')])),
+         ('insertTranslation',
+           ('',
+             [('insertText', ('$bar$', []))])),
+         ('rawtextOffset', ('</span>', 7))],
+      None)),
+   ('endScope', ()),
+   ('rawtextOffset', ('.', 1))])),
+('endScope', ()),
+('rawtextOffset', ('</div>', 6)) 
+]
+        self._check(program,
+                    '<div>THIS IS TEXT FOR <span>BARVALUE</span>.</div>\n')
+
+    def test_for_correct_msgids(self):
+
+        class CollectingTranslationService(DummyTranslationService):
+            data = []
+
+            def translate(self, domain, msgid, mapping=None,
+                          context=None, target_language=None, default=None):
+                self.data.append(msgid)
+                return DummyTranslationService.translate(
+                    self,
+                    domain, msgid, mapping, context, target_language, default)
+
+        xlatsvc = CollectingTranslationService()
+        self.engine.translationService = xlatsvc
+        result = StringIO()
+        program, macros = self._compile(
+            '<div i18n:translate="">This is text for '
+            '<span i18n:translate="" tal:content="bar" '
+            'i18n:name="bar_name"/>.</div>')
+        self.interpreter = TALInterpreter(program, {}, self.engine,
+                                          stream=result)
+        self.interpreter()
+        self.assert_('BaRvAlUe' in xlatsvc.data)
+        self.assert_('This is text for ${bar_name}.' in
+                     xlatsvc.data)
+        self.assertEqual(
+            '<div>THIS IS TEXT FOR <span>BARVALUE</span>.</div>\n',
+            result.getvalue())
+
+
+class I18NErrorsTestCase(TestCaseBase):
+
+    def _check(self, src, msg):
+        try:
+            self._compile(src)
+        except I18NError:
+            pass
+        else:
+            self.fail(msg)
+
+    def test_id_with_replace(self):
+        self._check('<p i18n:id="foo" tal:replace="string:splat"></p>',
+                    "expected i18n:id with tal:replace to be denied")
+
+    def test_missing_values(self):
+        self._check('<p i18n:attributes=""></p>',
+                    "missing i18n:attributes value not caught")
+        self._check('<p i18n:data=""></p>',
+                    "missing i18n:data value not caught")
+        self._check('<p i18n:id=""></p>',
+                    "missing i18n:id value not caught")
+
+    def test_id_with_attributes(self):
+        self._check('''<input name="Delete"
+                       tal:attributes="name string:delete_button"
+                       i18n:attributes="name message-id">''',
+            "expected attribute being both part of tal:attributes" +
+            " and having a msgid in i18n:attributes to be denied")
+
 class OutputPresentationTestCase(TestCaseBase):
 
     def check_attribute_wrapping(self):
@@ -159,6 +283,7 @@ def test_suite():
     suite.addTest(unittest.makeSuite(MacroErrorsTestCase, "check_"))
     suite.addTest(unittest.makeSuite(OutputPresentationTestCase, "check_"))
     suite.addTest(unittest.makeSuite(InterpolateTestCase, "check_"))
+    suite.addTest(unittest.makeSuite(I18NCornerTestCase))
     return suite