- Added a ZConfig directive 'large-file-threshold' to control
        the request content-size threshold at which a temporary file
        gets created. Use the same value for deciding between reading
        the whole request in memory or just a chunk inside
        webdav.NullResource.PUT().

      - Stitch newly-created object into it's container *before*
        calling it's PUT() method. This fixes an issue with
        OFS.File/OFS.Image that would result into reading the whole
        file in memory and wrapping it into a *single* Pdata object.

      - Import ZServer.CONNECTION_LIMIT variable *inside* the method
        that uses it. Before this, the variable was imported at the
        module level, thus binding it too early which would cause the
        ZConfig handler to have no real effect.

doc/CHANGES.txt

@@ -24,6 +24,19 @@ Zope Changes
 
   after Zope 2.8a1
 
+    Features added
+
+      - Added a ZConfig directive 'large-file-threshold' to control
+        the request content-size threshold at which a temporary file
+        gets created. Use the same value for deciding between reading
+        the whole request in memory or just a chunk inside
+        webdav.NullResource.PUT().
+
+      - Stitch newly-created object into it's container *before*
+        calling it's PUT() method. This fixes an issue with
+        OFS.File/OFS.Image that would result into reading the whole
+        file in memory and wrapping it into a *single* Pdata object.
+
     Bugs fixed
 
       - docutils: updated to V 0.3.5. The Zope core now contains a full copy of
@@ -69,10 +82,10 @@ Zope Changes
         a more verbose error message is issued, the same way it's done
         on attribute/item traversal.
 
-
-      - Collector #1523: replace the text field for importing .zexp/.xml 
+      - Collector #1523: replace the text field for importing .zexp/.xml
         files with a selection list
 
+
   Zope 2.8a1
 
 

lib/python/ZServer/FCGIServer.py

@@ -32,7 +32,7 @@ from medusa import logger
 from medusa.counter import counter
 from medusa.http_server import compute_timezone_for_log
 
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
 
 from PubCore import handle
 from PubCore.ZEvent import Wakeup
@@ -642,6 +642,7 @@ class FCGIServer(asyncore.dispatcher):
 
 
     def readable(self):
+        from ZServer import CONNECTION_LIMIT
         return len(asyncore.socket_map) < CONNECTION_LIMIT
 
 

lib/python/ZServer/FTPServer.py

@@ -72,7 +72,7 @@ from medusa import filesys
 from FTPResponse import make_response
 from FTPRequest import FTPRequest
 
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
 
 from cStringIO import StringIO
 import os
@@ -653,6 +653,7 @@ class FTPServer(ftp_server):
         self.ftp_channel_class (self, conn, addr, self.module)
 
     def readable(self):
+        from ZServer import CONNECTION_LIMIT
         return len(asyncore.socket_map) < CONNECTION_LIMIT
 
     def listen(self, num):

lib/python/ZServer/HTTPServer.py

@@ -52,7 +52,7 @@ from medusa.test import  max_sockets
 from medusa.default_handler import unquote
 from asyncore import compact_traceback, dispatcher
 
-from ZServer import CONNECTION_LIMIT, ZOPE_VERSION, ZSERVER_VERSION
+from ZServer import ZOPE_VERSION, ZSERVER_VERSION
 from ZServer import requestCloseOnExec
 from zLOG import LOG, register_subsystem, BLATHER, INFO, WARNING, ERROR
 import DebugLogger
@@ -74,9 +74,10 @@ header2env={'content-length'    : 'CONTENT_LENGTH',
 
 class zhttp_collector:
     def __init__(self, handler, request, size):
+        from ZServer import LARGE_FILE_THRESHOLD
         self.handler = handler
         self.request = request
-        if size > 524288:
+        if size > LARGE_FILE_THRESHOLD:
             # write large upload data to a file
             from tempfile import TemporaryFile
             self.data = TemporaryFile('w+b')
@@ -407,8 +408,9 @@ class zhttp_server(http_server):
         requestCloseOnExec(self.socket)
 
     def readable(self):
+        from ZServer import CONNECTION_LIMIT
         return self.accepting and \
-                len(asyncore.socket_map) < CONNECTION_LIMIT
+               len(asyncore.socket_map) < CONNECTION_LIMIT
 
     def listen(self, num):
         # override asyncore limits for nt's listen queue size

lib/python/ZServer/PCGIServer.py

@@ -35,7 +35,7 @@ from medusa.http_server import compute_timezone_for_log
 from asyncore import compact_traceback
 
 import ZServer
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
 
 from PubCore import handle
 from PubCore.ZEvent import Wakeup
@@ -327,6 +327,7 @@ class PCGIServer(asyncore.dispatcher):
         self.channel_class(self, conn, addr)
 
     def readable(self):
+        from ZServer import CONNECTION_LIMIT
         return len(asyncore.socket_map) < CONNECTION_LIMIT
 
     def writable (self):

lib/python/ZServer/__init__.py

@@ -21,13 +21,18 @@ import utils
 exit_code = 0
 
 # the ZServer version number
-ZSERVER_VERSION='1.1'
+ZSERVER_VERSION = '1.1'
 
 # the maximum number of incoming connections to ZServer
-CONNECTION_LIMIT=1000 # may be reset by max_listen_sockets handler in Zope
+CONNECTION_LIMIT = 1000 # may be reset by max_listen_sockets handler in Zope
+
+# request bigger than this size get saved into a
+# temporary file instead of being read completely into memory
+LARGE_FILE_THRESHOLD = 1 << 19 # may be reset by large_file_threshold
+                               # handler in Zope
 
 # the Zope version string
-ZOPE_VERSION=utils.getZopeVersion()
+ZOPE_VERSION = utils.getZopeVersion()
 
 # backwards compatibility aliases
 from utils import requestCloseOnExec

lib/python/Zope/Startup/handlers.py

@@ -91,6 +91,10 @@ def rest_language_code(value):
     value and _setenv('REST_LANGUAGE_CODE' , value)
     return value
 
+def large_file_threshold(value):
+    import ZServer
+    ZServer.LARGE_FILE_THRESHOLD = value
+
 # server handlers
 
 def root_handler(config):

lib/python/Zope/Startup/zopeschema.xml

@@ -734,8 +734,9 @@
     </description>
   </section>
 
-  <!-- max-listen-sockets should really go into the ZServer package, but
-       I can't quite figure out how to put it there -->
+  <!-- max-listen-sockets and large-file-threshold should really go
+       into the ZServer package, but I can't quite figure out how to
+       put it there -->
 
   <key name="max-listen-sockets" datatype="integer"
        default="1000">
@@ -745,6 +746,14 @@
      </description>
   </key>
 
+  <key name="large-file-threshold" datatype="byte-size"
+       handler="large_file_threshold" default="512KB">
+     <description>
+       Requests bigger than this size get saved into a temporary file
+       instead of being read completely into memory.
+     </description>
+  </key>
+
   <multisection type="ZServer.server" name="*" attribute="servers"/>
   <key name="port-base" datatype="integer" default="0">
     <description>

lib/python/webdav/NullResource.py

@@ -81,13 +81,16 @@ class NullResource(Persistent, Acquisition.Implicit, Resource):
             ob=File(name, '', body, content_type=typ)
         return ob
 
-    PUT__roles__=('Anonymous',)
+    PUT__roles__ = ('Anonymous',)
     def PUT(self, REQUEST, RESPONSE):
-        """Create a new non-collection resource."""
+        """Create a new non-collection resource.
+        """
+        from ZServer import LARGE_FILE_THRESHOLD
+
         self.dav__init(REQUEST, RESPONSE)
 
-        name=self.__name__
-        parent=self.__parent__
+        name = self.__name__
+        parent = self.__parent__
 
         ifhdr = REQUEST.get_header('If', '')
         if WriteLockInterface.isImplementedBy(parent) and parent.wl_isLocked():
@@ -101,17 +104,40 @@ class NullResource(Persistent, Acquisition.Implicit, Resource):
             # There was an If header, but the parent is not locked
             raise PreconditionFailed
 
-        body=REQUEST.get('BODY', '')
+        # SDS: Only use BODY if the file size is smaller than
+        # LARGE_FILE_THRESHOLD, otherwise read LARGE_FILE_THRESHOLD
+        # bytes from the file which should be enough to trigger
+        # content_type detection, and possibly enough for CMF's
+        # content_type_registry too.
+        #
+        # Note that body here is really just used for detecting the
+        # content type and figuring out the correct factory. The correct
+        # file content will be uploaded on ob.PUT(REQUEST, RESPONSE) after
+        # the object has been created.
+        #
+        # A problem I could see is content_type_registry predicates
+        # that do depend on the whole file being passed here as an
+        # argument. There's none by default that does this though. If
+        # they really do want to look at the file, they should use
+        # REQUEST['BODYFILE'] directly and try as much as possible not
+        # to read the whole file into memory.
+
+        if int(REQUEST.get('CONTENT_LENGTH') or 0) > LARGE_FILE_THRESHOLD:
+            file = REQUEST['BODYFILE']
+            body = file.read(LARGE_FILE_THRESHOLD)
+            file.seek(0)
+        else:
+            body = REQUEST.get('BODY', '')
+
         typ=REQUEST.get_header('content-type', None)
         if typ is None:
             typ, enc=OFS.content_types.guess_content_type(name, body)
 
         factory = getattr(parent, 'PUT_factory', self._default_PUT_factory )
         ob = factory(name, typ, body)
-        ob = (ob is None and
-              self._default_PUT_factory(name, typ, body) or
-              ob
-              )
+        if ob is None:
+            ob = self._default_PUT_factory(name, typ, body)
+
         # We call _verifyObjectPaste with verify_src=0, to see if the
         # user can create this type of object (and we don't need to
         # check the clipboard.
@@ -122,9 +148,11 @@ class NullResource(Persistent, Acquisition.Implicit, Resource):
         except:
             raise Forbidden, sys.exc_info()[1]
 
-        # Delegate actual PUT handling to the new object.
-        ob.PUT(REQUEST, RESPONSE)
+        # Delegate actual PUT handling to the new object,
+        # SDS: But just *after* it has been stored.
         self.__parent__._setObject(name, ob)
+        ob = self.__parent__._getOb(name)
+        ob.PUT(REQUEST, RESPONSE)
 
         RESPONSE.setStatus(201)
         RESPONSE.setBody('')

skel/etc/zope.conf.in

@@ -5,9 +5,20 @@
 # This is the Zope configuration file.  The Zope configuration file
 # shows what the default configuration directives are, and show
 # examples for each directive.  To declare a directive, make sure that
-# you add it to a line that does not begin with '#'.  Note that comments 
+# you add it to a line that does not begin with '#'.  Note that comments
 # are only allowed at the beginning of a line: you may not add comments
 # after directive text on the same line.
+#
+# Note for Developers
+# ===================
+#
+# This file is *not* auto-generated. If you create a new directive you
+# very likely want to include an example of how to use the new
+# directive in this file.
+#
+# You shouldn't modify 'zope.conf.in' to change
+# configuration. Instead, you should make a copy into 'zope.conf' and
+# modify that to avoid checking in changes to this file by mistake.
 
 # ZConfig "defines" used for later textual substitution
 
@@ -151,7 +162,7 @@ instancehome $INSTANCE
 #     this directive set to "on" or you will receive an error.  If using ZEO,
 #     at least one ZEO client must be run with this directive set to "on"
 #     once, the others can have it turned off.
-#     NOTE: If your main storage is mounted read-only, 
+#     NOTE: If your main storage is mounted read-only,
 #     you must set this directive to "off".
 #
 # Default: on
@@ -781,6 +792,18 @@ instancehome $INSTANCE
 #    max-listen-sockets 500
 
 
+# Directive: large-file-threshold
+#
+# Description:
+#     Requests bigger than this size get saved into a temporary file
+#     instead of being read completely into memory.
+#
+# Default: 512K
+#
+# Example:
+#
+#    large-file-threshold 1Mb
+
 
 # Directives: servers
 #