Experimental WSGI + Twisted support.

Merge from svn+ssh://regebro@svn.zope.org/repos/main/Zope/branches/regebro-wsgi_support2

doc/CHANGES.txt

@@ -48,6 +48,27 @@ Zope Changes
       - Using FastCGI is offically deprecated.
 
     Features added
+    
+      - Experimental WSGI and Twisted support for http.
+        Zope now has a WSGI interface for integration with other
+        web-servers than ZServer. Most notably Twisted is supported.
+        The WSGI application is ZPublisher.WSGIPublisher.publish_module
+        
+        You can make ZServer use the twisted interface with the 
+        "use-wsgi on" keyword in the http-server section in zope.conf.
+        
+        You can run Twisted by installing Twisted (2.1 recommended) and
+        replacing the http-server section with a server section in 
+        zope.conf. It is not possible to run a Twisted server together with
+        a ZServer at the same time.
+        
+          <server>
+            address 8080
+            type Zope2-HTTP
+          </server>
+        
+        WSGI: http://www.python.org/dev/peps/pep-0333/
+        Twisted: http://twistedmatrix.com/
 
       - The traversal has been refactored to take heed of Zope3s 
         IPublishTraverse adapter interfaces. The ZCML directives 

lib/python/Lifetime/__init__.py

@@ -31,6 +31,11 @@ def shutdown(exit_code,fast = 0):
         import ZServer
         ZServer.exit_code = exit_code
         _shutdown_phase = 1
+        try:
+            from twisted.internet import reactor
+            reactor.callLater(0.1, reactor.stop)
+        except ImportError:
+            pass
     if fast:
         # Someone wants us to shutdown fast. This is hooked into SIGTERM - so
         # possibly the system is going down and we can expect a SIGKILL within

lib/python/Products/PythonScripts/tests/testPythonScript.py

@@ -223,7 +223,8 @@ class TestPythonScriptErrors(PythonScriptTestBase):
 
     def testBadImports(self):
         self.assertPSRaises(ImportError, body="from string import *")
-        self.assertPSRaises(ImportError, body="import mmap")
+        self.assertPSRaises(ImportError, body="from datetime import datetime")
+        #self.assertPSRaises(ImportError, body="import mmap")
 
     def testAttributeAssignment(self):
         # It's illegal to assign to attributes of anything that

lib/python/ZPublisher/Publish.py

@@ -122,7 +122,6 @@ def publish(request, module_name, after_list, debug=0,
 
         return response
     except:
-
         # DM: provide nicer error message for FTP
         sm = None
         if response is not None:

lib/python/ZServer/HTTPResponse.py

@@ -311,6 +311,16 @@ class ChannelPipe:
             self._close=1
         self._request.reply_code=response.status
 
+    def start_response(self, status, headers, exc_info=None):
+        # Used for WSGI
+        self._request.reply_code = int(status.split(' ')[0])
+        status = 'HTTP/%s %s\r\n' % (self._request.version, status)
+        self.write(status)
+        headers = '\r\n'.join([': '.join(x) for x in headers])
+        self.write(headers)
+        self.write('\r\n\r\n')
+        return self.write
+        
 
 is_proxying_match = re.compile(r'[^ ]* [^ \\]*:').match
 proxying_connection_re = re.compile ('Proxy-Connection: (.*)', re.IGNORECASE)

lib/python/ZServer/HTTPServer.py

@@ -279,6 +279,48 @@ class zhttp_handler:
             </ul>""" %(self.module_name, self.hits)
             )
 
+from HTTPResponse import ChannelPipe
+
+class zwsgi_handler(zhttp_handler):
+    
+    def continue_request(self, sin, request):
+        "continue handling request now that we have the stdin"
+
+        s=get_header(CONTENT_LENGTH, request.header)
+        if s:
+            s=int(s)
+        else:
+            s=0
+        DebugLogger.log('I', id(request), s)
+
+        env=self.get_environment(request)
+        version = request.version
+        if version=='1.0' and is_proxying_match(request.request):
+            # a request that was made as if this zope was an http 1.0 proxy.
+            # that means we have to use some slightly different http
+            # headers to manage persistent connections.
+            connection_re = proxying_connection_re
+        else:
+            # a normal http request
+            connection_re = CONNECTION
+        
+        env['http_connection'] = get_header(connection_re,
+                                            request.header).lower()
+        env['server_version']=request.channel.server.SERVER_IDENT
+
+        env['wsgi.output'] = ChannelPipe(request)
+        env['wsgi.input'] = sin
+        env['wsgi.errors']       = sys.stderr
+        env['wsgi.version']      = (1,0)
+        env['wsgi.multithread']  = True
+        env['wsgi.multiprocess'] = True
+        env['wsgi.run_once']     = True
+        env['wsgi.url_scheme']   = env['SERVER_PROTOCOL'].split('/')[0]
+
+        request.channel.current_request=None
+        request.channel.queue.append(('Zope2WSGI', env, 
+                                      env['wsgi.output'].start_response))
+        request.channel.work()
 
 
 class zhttp_channel(http_channel):

lib/python/ZServer/PubCore/ZServerPublisher.py

@@ -14,13 +14,25 @@
 class ZServerPublisher:
     def __init__(self, accept):
         from ZPublisher import publish_module
+        from ZPublisher.WSGIPublisher import publish_module as publish_wsgi
         while 1:
-            try:
-                name, request, response=accept()
-                publish_module(
-                    name,
-                    request=request,
-                    response=response)
-            finally:
-                response._finish()
-                request=response=None
+            name, a, b=accept()
+            if name == "Zope2":
+                try:
+                    publish_module(
+                        name,
+                        request=a,
+                        response=b)
+                finally:
+                    b._finish()
+                    a=b=None
+
+            elif name == "Zope2WSGI":
+                try:
+                    res = publish_wsgi(a, b)
+                    for r in res:
+                        a['wsgi.output'].write(r)
+                finally:
+                    # TODO: Support keeping connections open.
+                    a['wsgi.output']._close = 1
+                    a['wsgi.output'].close()

lib/python/ZServer/component.xml

@@ -19,6 +19,7 @@
          receive WebDAV source responses to GET requests.
        </description>
      </key>
+     <key name="use-wsgi" datatype="boolean" default="off" />
   </sectiontype>
 
   <sectiontype name="webdav-source-server"
@@ -26,6 +27,7 @@
                implements="ZServer.server">
      <key name="address" datatype="inet-binding-address"/>
      <key name="force-connection-close" datatype="boolean" default="off"/>
+     <key name="use-wsgi" datatype="boolean" default="off" />
   </sectiontype>
 
   <sectiontype name="persistent-cgi"

lib/python/ZServer/datatypes.py

@@ -71,6 +71,7 @@ class HTTPServerFactory(ServerFactory):
         # webdav-source-server sections won't have webdav_source_clients:
         webdav_clients = getattr(section, "webdav_source_clients", None)
         self.webdav_source_clients = webdav_clients
+        self.use_wsgi = section.use_wsgi
 
     def create(self):
         from ZServer.AccessLogger import access_logger
@@ -86,7 +87,10 @@ class HTTPServerFactory(ServerFactory):
 
     def createHandler(self):
         from ZServer import HTTPServer
-        return HTTPServer.zhttp_handler(self.module, '', self.cgienv)
+        if self.use_wsgi:
+            return HTTPServer.zwsgi_handler(self.module, '', self.cgienv)
+        else:
+            return HTTPServer.zhttp_handler(self.module, '', self.cgienv)
 
 
 class WebDAVSourceServerFactory(HTTPServerFactory):

lib/python/Zope2/Startup/__init__.py

@@ -20,12 +20,16 @@ import sys
 import socket
 from re import compile
 from socket import gethostbyaddr
+try:
+    import twisted.internet.reactor
+    _use_twisted = True
+except ImportError:
+    _use_twisted = True
+    
 
 import ZConfig
-
 from ZConfig.components.logger import loghandler
 
-
 logger = logging.getLogger("Zope")
 started = False
 
@@ -96,7 +100,10 @@ class ZopeStarter:
         self.makePidFile()
         self.setupInterpreter()
         self.startZope()
-        self.registerSignals()
+        from App.config import getConfiguration
+        config = getConfiguration()
+        if not config.twisted_servers:
+            self.registerSignals()
         # emit a "ready" message in order to prevent the kinds of emails
         # to the Zope maillist in which people claim that Zope has "frozen"
         # after it has emitted ZServer messages.
@@ -106,10 +113,24 @@ class ZopeStarter:
     def run(self):
         # the mainloop.
         try:
+            from App.config import getConfiguration
+            config = getConfiguration()
             import ZServer
-            import Lifetime
-            Lifetime.loop()
-            sys.exit(ZServer.exit_code)
+            if config.twisted_servers and config.servers:
+                raise ZConfig.ConfigurationError(
+                    "You can't run both ZServer servers and twisted servers.")
+            if config.twisted_servers:
+                if not _use_twisted:
+                    raise ZConfig.ConfigurationError(
+                        "You do not have twisted installed.")
+                twisted.internet.reactor.run()
+                # Storing the exit code in the ZServer even for twisted, 
+                # but hey, it works...
+                sys.exit(ZServer.exit_code)
+            else:
+                import Lifetime
+                Lifetime.loop()
+                sys.exit(ZServer.exit_code)
         finally:
             self.shutdown()
 

lib/python/Zope2/Startup/datatypes.py

@@ -339,3 +339,11 @@ def zopeClassFactory(jar, module, name,
 # Zope class factory."  This no longer works with the implementation of
 # mounted databases, so we just use the zopeClassFactory as the default
 
+try:
+    from zope.app.twisted.server import ServerFactory
+    class TwistedServerFactory(ServerFactory):
+        pass
+except ImportError:
+    class TwistedServerFactory:
+        def __init__(self, section):
+            raise ImportError("You do not have twisted installed.")

lib/python/Zope2/Startup/handlers.py

 import os
 import sys
+import time
+import logging
 from re import compile
 from socket import gethostbyaddr
 
+try:
+    import twisted.internet
+    from twisted.application.service import MultiService
+    import zope.app.appsetup.interfaces
+    import zope.app.twisted.main
+
+    import twisted.web2.wsgi
+    import twisted.web2.server
+    import twisted.web2.log
+    
+    try:
+        from twisted.web2.http import HTTPFactory
+    except ImportError:
+        from twisted.web2.channel.http import HTTPFactory
+    
+    from zope.component import provideUtility
+    from zope.app.twisted.server import ServerType, SSLServerType
+    from zope.app.twisted.interfaces import IServerType
+    from ZPublisher.WSGIPublisher import publish_module
+    
+    _use_twisted = True
+except ImportError:
+    _use_twisted = False
+
 # top-level key handlers
 
 
@@ -133,7 +159,7 @@ def catalog_getObject_raises(value):
         "'catalog-getObject-raises' option will be removed in Zope 2.10:\n",
         DeprecationWarning)
 
-        from Products.ZCatalog import CatalogBrains 
+        from Products.ZCatalog import CatalogBrains
         CatalogBrains.GETOBJECT_RAISES = bool(value)
 
     return value
@@ -143,7 +169,8 @@ def catalog_getObject_raises(value):
 def root_handler(config):
     """ Mutate the configuration with defaults and perform
     fixups of values that require knowledge about configuration
-    values outside of their context. """
+    values outside of their context.
+    """
 
     # Set environment variables
     for k,v in config.environment.items():
@@ -165,7 +192,7 @@ def root_handler(config):
     instanceprod = os.path.join(config.instancehome, 'Products')
     if instanceprod not in config.products:
         config.products.append(instanceprod)
-    
+
     import Products
     L = []
     for d in config.products + Products.__path__:
@@ -190,6 +217,23 @@ def root_handler(config):
                         config.cgi_environment,
                         config.port_base)
 
+    if not config.twisted_servers:
+        config.twisted_servers = []
+    else:
+        # Set number of threads (reuse zserver_threads variable)
+        twisted.internet.reactor.suggestThreadPoolSize(config.zserver_threads)
+
+        # Create a root service
+        rootService = MultiService()
+
+        for server in config.twisted_servers:
+            service = server.create(None)
+            service.setServiceParent(rootService)
+
+        rootService.startService()
+        twisted.internet.reactor.addSystemEventTrigger(
+            'before', 'shutdown', rootService.stopService)
+
     # set up trusted proxies
     if config.trusted_proxies:
         import ZPublisher.HTTPRequest
@@ -217,3 +261,15 @@ def _name2Ips(host, isIp_=compile(r'(\d+\.){3}').match):
     if isIp_(host): return [host]
     return gethostbyaddr(host)[2]
 
+
+# Twisted support:
+
+def createHTTPFactory(ignored):
+    resource = twisted.web2.wsgi.WSGIResource(publish_module)
+    resource = twisted.web2.log.LogWrapperResource(resource)
+
+    return HTTPFactory(twisted.web2.server.Site(resource))
+
+if _use_twisted:
+    http = ServerType(createHTTPFactory, 8080)
+    provideUtility(http, IServerType, 'Zope2-HTTP')

lib/python/Zope2/Startup/zopeschema.xml

@@ -11,6 +11,12 @@
   <import package="tempstorage"/>
   <import package="Zope2.Startup" file="warnfilter.xml"/>
 
+  <sectiontype name="server" datatype="Zope2.Startup.datatypes.TwistedServerFactory">
+    <key name="type" required="yes" />
+    <key name="address" datatype="inet-address" />
+    <key name="backlog" datatype="integer" default="50" />
+  </sectiontype>
+
   <sectiontype name="logger" datatype=".LoggerFactory">
     <description>
       This "logger" type only applies to access and request ("trace")
@@ -805,7 +811,9 @@
      <metadefault>on</metadefault>
   </key>
 
+  <multisection type="server" name="*" attribute="twisted_servers" />
   <multisection type="ZServer.server" name="*" attribute="servers"/>
+
   <key name="port-base" datatype="integer" default="0">
     <description>
       Base port number that gets added to the specific port numbers

skel/etc/zope.conf.in

@@ -904,6 +904,8 @@ instancehome $INSTANCE
   # valid keys are "address" and "force-connection-close"
   address 8080
   # force-connection-close on
+  # You can also use the WSGI interface between ZServer and ZPublisher:
+  # use-wsgi on
 </http-server>
 
 # Examples:
@@ -947,6 +949,13 @@ instancehome $INSTANCE
 #    user admin
 #    password 123
 #  </clock-server>
+#
+#  <server>
+#    # This uses Twisted as the web-server. You must install Twisted 
+#    # separately. You can't run Twisted and ZServer at same time.
+#    address 8080
+#    type Zope2-HTTP
+#  </server>
 
 
 # Database (zodb_db) section