Merge of fix for #1490 to 2.8 branch: new zope.conf option for controlling the...

Merge of fix for #1490 to 2.8 branch: new zope.conf option for controlling the encoding of data sent to ZPublisher without any specified encoding.

doc/CHANGES.txt

@@ -33,6 +33,10 @@ Zope Changes
 
     Bugs Fixed
 
+      - Collector #1490: Added a new zope.conf option to control the
+        character set used to encode unicode data that reaches
+        ZPublisher without any specified encoding.
+
       - disabled ".. include" directive for all the ZReST product and the 
         reStructuredText package
 

lib/python/ZPublisher/Converters.py

@@ -17,10 +17,13 @@ from types import ListType, TupleType, UnicodeType
 from DateTime import DateTime
 from cgi import escape
 
+# This may get overwritten during configuration
+default_encoding = 'iso-8859-15'
+
 def field2string(v):
     if hasattr(v,'read'): return v.read()
-    elif isinstance(v,UnicodeType) :
-        return v.encode('iso-8859-15')
+    elif isinstance(v,UnicodeType):
+        return v.encode(default_encoding)
     else:
         return str(v)
 

lib/python/ZPublisher/HTTPRequest.py

@@ -25,6 +25,9 @@ from TaintedString import TaintedString
 from maybe_lock import allocate_lock
 xmlrpc=None # Placeholder for module that we'll import if we have to.
 
+# This may get overwritten during configuration
+default_encoding = 'iso-8859-15'
+
 isCGI_NAME = {
         'SERVER_SOFTWARE' : 1,
         'SERVER_NAME' : 1,
@@ -522,7 +525,7 @@ class HTTPRequest(BaseRequest):
                                 if hasattr(converter,'convert_unicode'):
                                     item = converter.convert_unicode(item)
                                 else:
-                                    item = converter(item.encode('iso-8859-15'))
+                                    item = converter(item.encode(default_encoding))
                             else:
                                 item=converter(item)
 

lib/python/ZPublisher/HTTPResponse.py

@@ -26,6 +26,8 @@ from cgi import escape
 
 nl2sp = maketrans('\n',' ')
 
+# This may get overwritten during configuration
+default_encoding = 'iso-8859-15'
 
 # Enable APPEND_TRACEBACKS to make Zope append tracebacks like it used to,
 # but a better solution is to make standard_error_message display error_tb.
@@ -444,7 +446,7 @@ class HTTPResponse(BaseResponse):
                 encoding = match.group(1)
                 return body.encode(encoding)
         # Use the default character encoding
-        return body.encode('iso-8859-15','replace')
+        return body.encode(default_encoding,'replace')
 
     def setBase(self,base):
         """Set the base URL for the returned document.

lib/python/Zope2/Startup/datatypes.py

@@ -211,3 +211,13 @@ class ZopeDatabase(ZODBDatabase):
                 return (real_root, real_path, container_class)
         raise LookupError('Nothing known about mount path %s' % mount_path)
     
+def default_zpublisher_encoding(value):
+    # This is a bit clunky but necessary :-(
+    # These modules are imported during the configuration process
+    # so a module-level call to getConfiguration in any of them
+    # results in getting config data structure without the necessary
+    # value in it.
+    from ZPublisher import Converters, HTTPRequest, HTTPResponse
+    Converters.default_encoding = value
+    HTTPRequest.default_encoding = value
+    HTTPResponse.default_encoding = value

lib/python/Zope2/Startup/zopeschema.xml

@@ -813,5 +813,13 @@
 
   <section type="zoperunner" name="*" attribute="runner"/>
 
+  <key name="default-zpublisher-encoding" datatype=".default_zpublisher_encoding">
+     <description>
+       This key controls what character set is used to encode unicode
+       data that reaches ZPublisher without any other specified encoding.
+     </description>
+     <metadefault>iso-8859-15</metadefault>
+  </key>
+
 
 </schema>

skel/etc/zope.conf.in

@@ -848,6 +848,17 @@ instancehome $INSTANCE
 #
 #    large-file-threshold 1Mb
 
+# Directive: default_zpublisher_encoding
+#
+# Description:
+#     This controls what character set is used to encode unicode
+#     data that reaches ZPublisher without any other specified encoding.
+#
+# Default: iso-8859-15
+#
+# Example:
+#
+#    default_zpublisher_encoding utf-8
 
 # Directives: servers
 #