Merge chrism-clockserver-merge branch into HEAD.

doc/CHANGES.txt

@@ -26,6 +26,58 @@ Zope Changes
 
     Features added
 
+      - Added a "clock server" servertype which allows users to
+        configure methods that should be called periodically as if
+        they were being called by a remote user agent on one of Zope's
+        HTTP ports.  This is meant to replace wget+cron for some class
+        of periodic callables.
+
+        To use, create a "clock-server" directive section anywhere
+        in your zope.conf file, like so:
+
+         <clock-server>
+            method /do_stuff
+            period 60
+            user admin
+            password 123
+            host localhost
+         </clock-server>
+
+        Any number of clock-server sections may be defined within a
+        single zope.conf.  Note that you must specify a
+        username/password combination with the appropriate level of
+        access to call the method you've defined.  You can omit the
+        username and password if the method is anonymously callable.
+        Obviously the password is stored in the clear in the config
+        file, so you need to protect the config file with filesystem
+        security if the Zope account is privileged and those who have
+        filesystem access should not see the password.
+
+        Descriptions of the values within the clock-server section
+        follow::
+
+          method -- the traversal path (from the Zope root) to an
+          executable Zope method (Python Script, external method,
+          product method, etc).  The method must take no arguments or
+          must obtain its arguments from a query string.
+
+          period -- the number of seconds between each clock "tick" (and
+          thus each call to the above "method").  The lowest number
+          providable here is typically 30 (this is the asyncore mainloop
+          "timeout" value).
+
+          user -- a zope username.
+
+          password -- the password for the zope username provided above.
+
+          host -- the hostname passed in via the "Host:" header in the
+          faux request.  Could be useful if you have virtual host rules
+          set up inside Zope itself.
+
+        To make sure the clock is working, examine your Z2.log file.  It
+        should show requests incoming via a "Zope Clock Server"
+        useragent.
+
       - Added a 'conflict-error-log-level' directive to zope.conf, to set
         the level at which conflict errors (which are normally retried
         automatically) are logged. The default is 'info'.

lib/python/ZServer/ClockServer.py

+##############################################################################
+#
+# Copyright (c) 2005 Chris McDonough. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+""" Zope clock server.  Generate a faux HTTP request on a regular basis
+by coopting the asyncore API. """
+
+import os
+import socket
+import time
+import StringIO
+import base64
+import asyncore
+
+from ZServer.medusa.http_server import http_request
+from ZServer.medusa.default_handler import unquote
+from ZServer.PubCore import handle
+from ZServer.HTTPResponse import make_response
+from ZPublisher.HTTPRequest import HTTPRequest
+
+def timeslice(period, when=0):
+    return when - (when % period)
+
+class LogHelper:
+    def __init__(self, logger):
+        self.logger = logger
+
+    def log(self, ip, msg, **kw):
+        self.logger.log(ip + ' ' + msg)
+
+class DummyChannel:
+    # we need this minimal do-almost-nothing channel class to appease medusa
+    addr = ['127.0.0.1']
+    closed = 1
+
+    def __init__(self, server):
+        self.server = server
+        
+    def push_with_producer(self):
+        pass
+
+    def close_when_done(self):
+        pass
+    
+class ClockServer(asyncore.dispatcher):
+    # prototype request environment
+    _ENV = dict(REQUEST_METHOD = 'GET',
+                SERVER_PORT = 'Clock',
+                SERVER_NAME = 'Zope Clock Server',
+                SERVER_SOFTWARE = 'Zope',
+                SERVER_PROTOCOL = 'HTTP/1.0',
+                SCRIPT_NAME = '',
+                GATEWAY_INTERFACE='CGI/1.1',
+                REMOTE_ADDR = '0')
+
+    # required by ZServer
+    SERVER_IDENT = 'Zope Clock' 
+
+    def __init__ (self, method, period=60, user=None, password=None,
+                  host=None, logger=None):
+        self.period = period
+        self.method = method
+
+        self.last_slice = timeslice(period)
+
+        h = self.headers = []
+        h.append('User-Agent: Zope Clock Server Client')
+        h.append('Accept: text/html,text/plain')
+        if not host:
+            host = socket.gethostname()
+        h.append('Host: %s' % host)
+        auth = False
+        if user and password:
+            encoded = base64.encodestring('%s:%s' % (user, password))
+            encoded = encoded.replace('\012', '')
+            h.append('Authorization: Basic %s' % encoded)
+            auth = True
+
+        asyncore.dispatcher.__init__(self)
+        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.logger = LogHelper(logger)
+        self.log_info('Clock server for "%s" started (user: %s, period: %s)'
+                      % (method, auth and user or 'Anonymous', self.period))
+
+    def get_requests_and_response(self):
+        out = StringIO.StringIO()
+        s_req = '%s %s HTTP/%s' % ('GET', self.method, '1.0')
+        req = http_request(DummyChannel(self), s_req, 'GET', self.method,
+                           '1.0', self.headers)
+        env = self.get_env(req)
+        resp = make_response(req, env)
+        zreq = HTTPRequest(out, env, resp)
+        return req, zreq, resp
+
+    def get_env(self, req):
+        env = self._ENV.copy()
+        (path, params, query, fragment) = req.split_uri()
+        if params:
+            path = path + params # undo medusa bug
+        while path and path[0] == '/':
+            path = path[1:]
+        if '%' in path:
+            path = unquote(path)
+        if query:
+            # ZPublisher doesn't want the leading '?'
+            query = query[1:]
+        env['PATH_INFO']= '/' + path
+        env['PATH_TRANSLATED']= os.path.normpath(
+            os.path.join(os.getcwd(), env['PATH_INFO']))
+        if query:
+            env['QUERY_STRING'] = query
+        env['channel.creation_time']=time.time()
+        for header in req.header:
+            key,value=header.split(":",1)
+            key=key.upper()
+            value=value.strip()
+            key='HTTP_%s' % ("_".join(key.split( "-")))
+            if value:
+                env[key]=value
+        return env
+
+    def readable(self):
+        # generate a request at most once every self.period seconds
+        slice = timeslice(self.period)
+        if slice != self.last_slice:
+            # no need for threadsafety here, as we're only ever in one thread
+            self.last_slice = slice
+            req, zreq, resp = self.get_requests_and_response()
+            handle('Zope', zreq, resp)
+        return 0
+
+    def handle_read(self):
+        pass
+
+    def handle_write (self):
+        self.log_info ('unexpected write event', 'warning')
+
+    def writable(self):
+        return 0
+
+    def handle_error (self):      # don't close the socket on error
+        (file,fun,line), t, v, tbinfo = asyncore.compact_traceback()
+        self.log_info('Problem in Clock (%s:%s %s)' % (t, v, tbinfo),
+                      'error')
+
+
+

lib/python/ZServer/component.xml

@@ -58,4 +58,42 @@
      <key name="address" datatype="inet-binding-address"/>
   </sectiontype>
 
+  <sectiontype name="clock-server"
+               datatype=".ClockServerFactory"
+               implements="ZServer.server">
+     <key name="method" datatype="string">
+       <description>
+       The traversal path (from the Zope root) to an
+       executable Zope method (Python Script, external method, product
+       method, etc).  The method must take no arguments.  Ex: "/site/methodname"
+       </description>
+     </key>
+     <key name="period" datatype="integer" default="60">
+     <description>
+       The number of seconds between each clock "tick" (and
+       thus each call to the above "method").  The lowest number
+       providable here is typically 30 (this is the asyncore mainloop
+       "timeout" value).  The default is 60.  Ex: "30"
+     </description>
+     </key>
+     <key name="user" datatype="string">
+     <description>
+       A zope username. Ex: "admin"
+     </description>
+     </key>
+     <key name="password" datatype="string">
+     <description>
+      The password for the zope username provided above.  Careful: this
+      is obviously not encrypted in the config file. Ex: "123"
+     </description>
+     </key>
+     <key name="host" datatype="string">
+     <description>
+       The hostname passed in via the "Host:" header in the
+       faux request.  Could be useful if you have virtual host rules
+       set up inside Zope itself. Ex: "www.example.com"
+     </description>
+     </key>
+  </sectiontype>
+
 </component>

lib/python/ZServer/datatypes.py

@@ -198,3 +198,20 @@ class ICPServerFactory(ServerFactory):
     def create(self):
         from ZServer.ICPServer import ICPServer
         return ICPServer(self.ip, self.port)
+
+class ClockServerFactory(ServerFactory):
+    def __init__(self, section):
+        ServerFactory.__init__(self)
+        self.method = section.method
+        self.period = section.period
+        self.user = section.user
+        self.password = section.password
+        self.hostheader = section.host
+        self.host = None # appease configuration machinery
+
+    def create(self):
+        from ZServer.ClockServer import ClockServer
+        from ZServer.AccessLogger import access_logger
+        return ClockServer(self.method, self.period, self.user,
+                           self.password, self.hostheader, access_logger)
+    

lib/python/ZServer/tests/test_clockserver.py

+import unittest
+from StringIO import StringIO
+
+from ZServer import ClockServer
+
+class LogHelperTests(unittest.TestCase):
+    def _getTargetClass(self):
+        return ClockServer.LogHelper
+
+    def _makeOne(self, *arg, **kw):
+        return self._getTargetClass()(*arg **kw)
+
+    def test_helper(self):
+        from StringIO import StringIO
+        logger = StringIO()
+        logger.log = logger.write
+        helper = self._makeOne(logger)
+        self.assertEqual(helper.logger, logger)
+        logger.log('ip', 'msg', foo=1, bar=2)
+        logger.seek(0)
+        self.assertEqual(logger.read(), 'ip msg')
+
+class ClockServerTests(unittest.TestCase):
+    def _getTargetClass(self):
+        return ClockServer.ClockServer
+
+    def _makeOne(self, *arg, **kw):
+        return self._getTargetClass()(*arg **kw)
+
+    def test_ctor(self):
+        logger = StringIO()
+        logger.log = logger.write
+        server = self._makeOne(method='a', period=60, user='charlie',
+                               password='brown', host='localhost',
+                               logger=logger)
+        auth = 'charlie:brown'.encode('base64')
+        self.assertEqual(server.headers,
+                         ['User-Agent: Zope Clock Server Client',
+                          'Accept: text/html,text/plain',
+                          'Host: localhost',
+                          'Authorization: Basic %s' % auth])
+        logger.seek(0)
+        self.assertEqual(
+            logger.read(),
+            'Clock server for "a" started (user: charlie, period: 60)')
+                          
+    def test_get_requests_and_response(self):
+        logger = StringIO()
+        logger.log = logger.write
+        server = self._makeOne(method='a', period=60, user='charlie',
+                               password='brown', host='localhost',
+                               logger=logger)
+        req, zreq, resp = server.get_requests_and_response()
+
+        from ZServer.medusa.http_server import http_request
+        from ZServer.HTTPResponse import HTTPResponse
+        from ZPublisher.HTTPRequest import HTTPRequest
+        self.failUnless(issubclass(req, http_request))
+        self.failUnless(issubclass(resp, HTTPResponse))
+        self.failUnlesss(issubclass(zreq, HTTPRequest))
+
+    def test_get_env(self):
+        logger = StringIO()
+        logger.log = logger.write
+        server = self._makeOne(method='a', period=60, user='charlie',
+                               password='brown', host='localhost',
+                               logger=logger)
+        class dummy_request:
+            def split_uri(self):
+                return '/a%20', '/b', '?foo=bar', ''
+
+            header = ['BAR']
+        env = server.get_env(dummy_request())
+        _ENV = dict(REQUEST_METHOD = 'GET',
+                    SERVER_PORT = 'Clock',
+                    SERVER_NAME = 'Zope Clock Server',
+                    SERVER_SOFTWARE = 'Zope',
+                    SERVER_PROTOCOL = 'HTTP/1.0',
+                    SCRIPT_NAME = '',
+                    GATEWAY_INTERFACE='CGI/1.1',
+                    REMOTE_ADDR = '0')
+        for k, v in _ENV.items():
+            self.assertEqual(env[k], v)
+        self.assertEqual(env['PATH_INFO'], '')
+        self.assertEqual(env['PATH_TRANSLATED'], '')
+        self.assertEqual(env['QUERY_STRING'], 'foo=bar')
+        self.assert_(env['channel.creation_time'])
+
+    def test_handle_write(self):
+        logger = StringIO()
+        logger.log = logger.write
+        server = self._makeOne(method='a', period=60, user='charlie',
+                               password='brown', host='localhost',
+                               logger=logger)
+        server.handle_write()
+        logger.seek(0)
+        self.assertEqual(logger.read(), 'unexpected write event')
+
+    def test_handle_error(self):
+        logger = StringIO()
+        logger.log = logger.write
+        server = self._makeOne(method='a', period=60, user='charlie',
+                               password='brown', host='localhost',
+                               logger=logger)
+        server.handle_error()
+        logger.seek(0)
+        self.assertEqual(logger.read, 'foo')
+
+def test_suite():
+    suite = unittest.makeSuite(ClockServerTests)
+    suite.addTest(unittest.makeSuite(LogHelperTests))
+    return suite
+
+if __name__ == "__main__":
+    unittest.main(defaultTest="test_suite")

lib/python/ZServer/tests/test_config.py

@@ -61,8 +61,8 @@ class BaseTest(unittest.TestCase):
         self.assertEqual(factory.module, "module")
         self.assertEqual(factory.cgienv.items(), [("key", "value")])
         if port is None:
-            self.assert_(factory.host is None)
-            self.assert_(factory.port is None)
+            self.assert_(factory.host is None, factory.host)
+            self.assert_(factory.port is None, factory.port)
         else:
             self.assertEqual(factory.host, expected_factory_host)
             self.assertEqual(factory.port, 9300 + port)
@@ -226,6 +226,25 @@ class ZServerConfigurationTestCase(BaseTest, WarningInterceptor):
         self.check_prepare(factory)
         factory.create().close()
 
+    def test_clockserver_factory(self):
+        factory = self.load_factory("""\
+            <clock-server>
+              method /foo/bar
+              period 30
+              user chrism
+              password 123
+              host www.example.com
+            </clock-server>
+            """)
+        self.assert_(isinstance(factory,
+                                ZServer.datatypes.ClockServerFactory))
+        self.assertEqual(factory.method, '/foo/bar')
+        self.assertEqual(factory.period, 30)
+        self.assertEqual(factory.user, 'chrism')
+        self.assertEqual(factory.password, '123')
+        self.assertEqual(factory.hostheader, 'www.example.com')
+        factory.create().close()
+
 
 class MonitorServerConfigurationTestCase(BaseTest):
 

skel/etc/zope.conf.in

@@ -882,10 +882,10 @@ instancehome $INSTANCE
 #
 # Description:
 #     A set of sections which allow the specification of Zope's various
-#     ZServer servers.  7 different server types may be defined:
+#     ZServer servers.  8 different server types may be defined:
 #     http-server, ftp-server, webdav-source-server, persistent-cgi,
-#     fast-cgi, monitor-server, and icp-server.  If no servers are
-#     defined, the default servers are used.
+#     fast-cgi, monitor-server, icp-server, and clock-server.  If no servers
+#     are defined, the default servers are used.
 #
 #     Ports may be specified using the 'address' directive either in simple
 #     form (80) or in complex form including hostname 127.0.0.1:80.  If the
@@ -939,6 +939,14 @@ instancehome $INSTANCE
 #    # valid key is "address"
 #    address 888
 #  </icp-server>
+#
+#  <clock-server>
+#    # starts a clock which calls /foo/bar every 30 seconds
+#    method /foo/bar
+#    period 30
+#    user admin
+#    password 123
+#  </clock-server>
 
 
 # Database (zodb_db) section