Fix charset of web scripts

/reviewed-on !740

bt5/erp5_officejs/SkinTemplateItem/portal_skins/erp5_web_officejs_ui/WebPage_viewAsWeb.py

@@ -24,13 +24,13 @@ else:
 
 # set headers depending on type of script
 if (portal_type == "Web Script"):
-  response.setHeader('Content-Type', 'application/javascript')
+  response.setHeader('Content-Type', 'application/javascript; charset=utf-8')
 
 elif (portal_type == "Web Style"):
-  response.setHeader('Content-Type', 'text/css')
+  response.setHeader('Content-Type', 'text/css; charset=utf-8')
 
 elif (portal_type == "Web Manifest"):
-  response.setHeader('Content-Type', 'text/cache-manifest')
+  response.setHeader('Content-Type', 'text/cache-manifest; charset=utf-8')
 
 elif (portal_type == "Web Page"):
   if (mapping_dict is not None):

bt5/erp5_web_manifest/SkinTemplateItem/portal_skins/manifest_renderjs_ui/WebPage_viewAsWeb.py

@@ -14,13 +14,13 @@ web_content = web_page.getTextContent()
 
 # set headers depending on type of script
 if (portal_type == "Web Script"):
-  response.setHeader('Content-Type', 'application/javascript')
+  response.setHeader('Content-Type', 'application/javascript; charset=utf-8')
 
 elif (portal_type == "Web Style"):
-  response.setHeader('Content-Type', 'text/css')
+  response.setHeader('Content-Type', 'text/css; charset=utf-8')
 
 elif (portal_type == "Web Manifest"):
-  response.setHeader('Content-Type', 'text/cache-manifest')
+  response.setHeader('Content-Type', 'text/cache-manifest; charset=utf-8')
 
 else:
   if (mapping_dict is not None):
@@ -32,6 +32,6 @@ else:
   # Only fetch code (html, js, css, image) and data from this ERP5, to prevent any data leak as the web site do not control the gadget's code
   response.setHeader("Content-Security-Policy", "default-src 'none'; img-src 'self' data:; media-src 'self'; connect-src 'self'; script-src 'self' 'unsafe-eval'; font-src netdna.bootstrapcdn.com fonts.googleapis.com fonts.gstatic.com; style-src 'self' netdna.bootstrapcdn.com fonts.googleapis.com fonts.gstatic.com 'unsafe-inline' data:; frame-src 'self' https://jsbin.com data:;")
 
-  response.setHeader('Content-Type', 'text/html')
+  response.setHeader('Content-Type', 'text/html; charset=utf-8')
 
 return web_content

bt5/erp5_web_renderjs_ui/SkinTemplateItem/portal_skins/erp5_web_renderjs/WebScript_init.py

-context.setContentType('text/javascript')
+context.setContentType('application/javascript')

bt5/erp5_web_renderjs_ui/SkinTemplateItem/portal_skins/erp5_web_renderjs_ui/WebPage_viewAsWeb.py

@@ -22,13 +22,13 @@ web_content = web_page.getTextContent()
 
 # set headers depending on type of script
 if (portal_type == "Web Script"):
-  response.setHeader('Content-Type', 'application/javascript')
+  response.setHeader('Content-Type', 'application/javascript; charset=utf-8')
 
 elif (portal_type == "Web Style"):
-  response.setHeader('Content-Type', 'text/css')
+  response.setHeader('Content-Type', 'text/css; charset=utf-8')
 
 elif (portal_type == "Web Manifest"):
-  response.setHeader('Content-Type', 'text/cache-manifest')
+  response.setHeader('Content-Type', 'text/cache-manifest; charset=utf-8')
 
 else:
   if (mapping_dict is not None):

bt5/erp5_web_renderjs_ui_test/TestTemplateItem/portal_components/test.erp5.testRJSPortalType.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+
+class TestRenderJSPortalType(ERP5TypeTestCase):
+  """Test Web Script & Web Style portal types added by Render JS.
+  """
+
+  def afterSetUp(self):
+    self.web_site = self.portal.web_site_module.newContent(
+      portal_type='Web Site',
+      skin_selection_name='RJS',
+    )
+    self.web_site.publish()
+
+  def test_web_style(self):
+    web_style = self.portal.web_page_module.newContent(
+      portal_type='Web Style',
+      reference='test_web_style.css'
+    )
+    web_style.setTextContent('/* cl\xc3\xa0sse */ .classe { background: red }')
+    web_style.publish()
+    self.tic()
+    self.assertEqual('text/css', web_style.getContentType())
+
+    # test HTTP response
+    response = self.publish(
+      '%s/%s' % (self.web_site.getPath(), web_style.getReference())
+    )
+    self.assertEqual(
+      '/* cl\xc3\xa0sse */ .classe { background: red }',
+      response.getBody()
+    )
+    self.assertEqual(
+      'text/css; charset=utf-8',
+      response.getHeader('content-type')
+    )
+
+  def test_web_script(self):
+    web_script = self.portal.web_page_module.newContent(
+      portal_type='Web Script',
+      reference='test_web_script.js'
+    )
+    web_script.setTextContent('alert("h\xc3\xa9h\xc3\xa9")')
+    web_script.publish()
+    self.tic()
+    self.assertEqual('application/javascript', web_script.getContentType())
+
+    # test HTTP response
+    response = self.publish(
+      '%s/%s' % (self.web_site.getPath(), web_script.getReference())
+    )
+    self.assertEqual(
+      'alert("h\xc3\xa9h\xc3\xa9")',
+      response.getBody()
+    )
+    self.assertEqual(
+      'application/javascript; charset=utf-8',
+      response.getHeader('content-type')
+    )
+

bt5/erp5_web_renderjs_ui_test/TestTemplateItem/portal_components/test.erp5.testRJSPortalType.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Test Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>testRJSPortalType</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>test.erp5.testRJSPortalType</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Test Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_renderjs_ui_test/bt/template_test_id_list

@@ -22,4 +22,5 @@ test.erp5.testFunctionalRJSMatrixbox
 test.erp5.testFunctionalRJSEditorGadget
 test.erp5.testFunctionalRJSRecoverPassword
 test.erp5.testFunctionalRJSInterfaceValidator
-test.erp5.testFunctionalRJSDms
\ No newline at end of file
+test.erp5.testFunctionalRJSDms
+test.erp5.testRJSPortalType
\ No newline at end of file

product/ERP5Type/patches/default_zpublisher_encoding.py

@@ -27,6 +27,10 @@
 ##############################################################################
 
 import ZPublisher.HTTPRequest
+import ZPublisher.HTTPResponse
+import ZPublisher.Converters
 
 # Force (do not depend on) the default-zpublisher-encoding setting of zope.conf
 ZPublisher.HTTPRequest.default_encoding = 'utf-8'
+ZPublisher.HTTPResponse.default_encoding = 'utf-8'
+ZPublisher.Converters.default_encoding = 'utf-8'