fixup! web_renderjs_ui: use lxml to extract data-i18n messages

a928abd2 (web_renderjs_ui: use lxml to extract data-i18n messages, 2020-12-14) was an old version of the patch that was pushed accidentally. This applies the missing bits.

fixup! web_renderjs_ui: use lxml to extract data-i18n messages
a928abd2 (web_renderjs_ui: use lxml to extract data-i18n messages, 2020-12-14) was an old version of the patch that was pushed accidentally. This applies the missing bits.
0039aa3f · Jérome Perrin · 1137838d · 0039aa3f · 0039aa3f
Commit 0039aa3f authored Dec 14, 2020 by Jérome Perrin
2 changed files
--- a/bt5/erp5_web_renderjs_ui/SkinTemplateItem/portal_skins/erp5_web_renderjs/WebSite_getTranslationDataTextContent.py
+++ b/bt5/erp5_web_renderjs_ui/SkinTemplateItem/portal_skins/erp5_web_renderjs/WebSite_getTranslationDataTextContent.py
@@ -12,22 +12,20 @@ Base_translateString = context.Base_translateString
 #   <!-- data-i18n="The message" -->
 translatable_message_set = set([])

-# Web pages can be in web page module ...
 web_page_reference_list = context.Base_getTranslationSourceFileList(only_html=1)
-not_found_in_web_page_reference_set = set([])
 for web_page_reference in web_page_reference_list:
+  # Web pages can be in web page module ...
  web_page = context.getDocumentValue(web_page_reference)
-  if web_page is None:
-    not_found_in_web_page_reference_set.add(web_page_reference)
+  if web_page is not None:
+    web_page_text_content = web_page.getTextContent()
  else:
-    for message in portal.ERP5Site_extractTranslationMessageListFromHTML(web_page.getTextContent()):
-      translatable_message_set.add(message)
-# ... or in skin folders
-for web_page_reference in not_found_in_web_page_reference_set:
-  if not '/' in web_page_reference:
+    # ... or in skin folders
    web_page = context.restrictedTraverse(web_page_reference, None)
    if web_page is not None and hasattr(web_page, 'manage_FTPget'):
-      for message in portal.ERP5Site_extractTranslationMessageListFromHTML(web_page.manage_FTPget()):
+      web_page_text_content = web_page.manage_FTPget()
+
+  if web_page_text_content:
+    for message in portal.ERP5Site_extractTranslationMessageListFromHTML(web_page_text_content):
      translatable_message_set.add(message)

 tmp = {}
@@ -36,7 +34,12 @@ for language in context.getAvailableLanguageSet():
  for word in translatable_message_set:
    tmp[language][word] = unicode(Base_translateString(word, lang = language), 'utf-8')

-return u"""/**
+
+# We pass unicode to this json.dump(ensure_ascii=False), so that it produce
+# UTF-8 string and not escaped characters. At the end we return an UTF-8
+# encoded string and not an unicode instance, because text_content property
+# is usually UTF-8 encoded str (not unicode).
+return (u"""/**
 * This translation data is generated automatically and updated with upgrader in post-upgarde.
 * Do not edit manually, but use "Update Translation Data" action on web site to update from
 * Localizer and from data-i18n tags on web pages.
@@ -55,4 +58,4 @@ return u"""/**
            sort_keys=True,
            indent=2,
            ensure_ascii=False,
-            separators=(',', ': ')).splitlines()))
+            separators=(',', ': ')).splitlines()))).encode('utf-8')
--- a/bt5/erp5_web_renderjs_ui_test/TestTemplateItem/portal_components/test.erp5.testRJSUpgrader.py
+++ b/bt5/erp5_web_renderjs_ui_test/TestTemplateItem/portal_components/test.erp5.testRJSUpgrader.py
@@ -200,7 +200,7 @@ class TestRenderUpdateTranslationData(RenderJSUpgradeTestCase):
    self.web_script_translation_data_js.publish()
    self.web_page_translation_gadget = self.portal.web_page_module.newContent(
        portal_type='Web Page',
-        reference='{}.html'.format(self.id()),
+        reference='{}_translation.html'.format(self.id()),
        content_type='text/html',
        text_content=textwrap.dedent('''
            <!DOCTYPE html>
@@ -313,7 +313,7 @@ class TestRenderUpdateTranslationData(RenderJSUpgradeTestCase):
    self.assertIn('"Message with [square brackets]":', translation_data_text_content)
    self.assertIn('"Message in <a href=\'link\'>HTML</a>":', translation_data_text_content)
    self.assertIn('"Message in script attributes":', translation_data_text_content)
-    self.assertIn(u'"メッサージュ":', translation_data_text_content)
+    self.assertIn('"メッサージュ":', translation_data_text_content)

  def test_WebSite_getTranslationDataTextContent_extract_from_page_template(self):
    self.portal.portal_skins.custom.manage_addProduct['PageTemplates'].manage_addPageTemplate(