Rewrote zeopack:

The zeopack script has gotten a numner of improvements:

- Simplified command-line interface. (The old interface is still
    supported, except that support for ZEO version 1 servers has been
    dropped.)

- Multiple storages can be packed in sequence.

- This simplifies pack scheduling on servers serving multiple
      databases.

- All storages are packed to the same time.

- You can now specify a time of day to pack to.

- The script will now time out if it can't connect to s storage in
    60 seconds.

- It now has a test. :)

src/CHANGES.txt

@@ -8,6 +8,30 @@
 New Features
 ------------
 
+3.9.0a13 (2009-??-??)
+=====================
+
+New Features
+------------
+
+- The zeopack script has gotten a numner of improvements:
+
+  - Simplified command-line interface. (The old interface is still
+    supported, except that support for ZEO version 1 servers has been
+    dropped.)
+
+  - Multiple storages can be packed in sequence.
+
+    - This simplifies pack scheduling on servers serving multiple
+      databases.
+
+    - All storages are packed to the same time.
+
+  - You can now specify a time of day to pack to.
+
+  - The script will now time out if it can't connect to s storage in
+    60 seconds.
+
 - The connection now estimates the object size based on its pickle size
   and informs the cache about size changes.
 
@@ -19,15 +43,6 @@ New Features
   There are corresponding methods to read and set the new configuration
   parameters.
 
-  XXX There are known issues with this implementation that need to be
-  sorted out before it is "released".
-
-- Moved a blob related logging message to debug level, for the case of
-  everything working as intended.
-
-- Changed the url in the package metadata to point to PyPi. Also removed
-  references to download.zope.org.
-
 3.9.0a12 (2009-02-26)
 =====================
 

src/ZEO/scripts/tests.py

+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""XXX short summary goes here.
+
+$Id$
+"""
+import unittest
+from zope.testing import doctest
+
+def test_suite():
+    return unittest.TestSuite((
+        doctest.DocFileSuite('zeopack.test'),
+        ))
+

src/ZEO/scripts/zeopack.py

 #!/usr/bin/env python2.3
 
-"""Connect to a ZEO server and ask it to pack.
-
-Usage: zeopack.py [options]
-
-Options:
-
-    -p port -- port to connect to
-
-    -h host -- host to connect to (default is current host)
-
-    -U path -- Unix-domain socket to connect to
+import logging
+import optparse
+import socket
+import sys
+import time
+import traceback
+import ZEO.ClientStorage
 
-    -S name -- storage name (default is '1')
+usage = """Usage: %prog [options] [servers]
 
-    -d days -- pack objects more than days old
+Pack one or more storages hosted by ZEO servers.
 
-    -1 -- Connect to a ZEO 1 server
+The positional arguments specify 0 or more tcp servers to pack, where
+each is of the form:
 
-    -W -- wait for server to come up.  Normally the script tries to
-       connect for 10 seconds, then exits with an error.  The -W
-       option is only supported with ZEO 1.
+    host:port[:name]
 
-You must specify either -p and -h or -U.
 """
 
-import getopt
-import socket
-import sys
-import time
-
-from ZEO.ClientStorage import ClientStorage
-
 WAIT = 10 # wait no more than 10 seconds for client to connect
 
-def connect(storage):
-    # The connect-on-startup logic that ZEO provides isn't too useful
-    # for this script.  We'd like to client to attempt to startup, but
-    # fail if it can't get through to the server after a reasonable
-    # amount of time.  There's no external support for this, so we'll
-    # expose the ZEO 1.0 internals.  (consenting adults only)
-    t0 = time.time()
-    while t0 + WAIT > time.time():
-        storage._call.connect()
-        if storage._connected:
-            return
-    raise RuntimeError("Unable to connect to ZEO server")
-
-def pack1(addr, storage, days, wait):
-    cs = ClientStorage(addr, storage=storage,
-                       wait_for_server_on_startup=wait)
-    if wait:
-        # _startup() is an artifact of the way ZEO 1.0 works.  The
-        # ClientStorage doesn't get fully initialized until registerDB()
-        # is called.  The only thing we care about, though, is that
-        # registerDB() calls _startup().
-        cs._startup()
-    else:
-        connect(cs)
-    cs.invalidator = None
-    cs.pack(wait=1, days=days)
-    cs.close()
-
-def pack2(addr, storage, days):
-    cs = ClientStorage(addr, storage=storage, wait=1, read_only=1)
-    cs.pack(wait=1, days=days)
-    cs.close()
-
-def usage(exit=1):
-    print __doc__
-    print " ".join(sys.argv)
-    sys.exit(exit)
-
-def main():
-    host = None
-    port = None
-    unix = None
-    storage = '1'
-    days = 0
-    wait = 0
-    zeoversion = 2
+def _main(args=None, prog=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    parser = optparse.OptionParser(usage, prog=prog)
+
+    parser.add_option(
+        "-d", "--days", dest="days", type='int', default=0,
+        help=("Pack objects that are older than this number of days")
+        )
+
+    parser.add_option(
+        "-t", "--time", dest="time",
+        help=("Time of day to pack to of the form: HH[:MM[:SS]]. "
+              "Defaults to current time.")
+        )
+
+    parser.add_option(
+        "-u", "--unix", dest="unix_sockets", action="append",
+        help=("A unix-domain-socket server to connect to, of the form: "
+              "path[:name]")
+        )
+
+    parser.remove_option('-h')
+    parser.add_option(
+        "-h", dest="host",
+        help=("Deprecated: "
+              "Used with the -p and -S options, specified the host to "
+              "connect to.")
+        )
+
+    parser.add_option(
+        "-p", type="int", dest="port",
+        help=("Deprecated: "
+              "Used with the -h and -S options, specifies "
+              "the port to connect to.")
+        )
+
+    parser.add_option(
+        "-S", dest="name", default='1',
+        help=("Deprecated: Used with the -h and -p, options, or with the "
+              "-U option specified the storage name to use. Defaults to 1.")
+        )
+
+    parser.add_option(
+        "-U", dest="unix",
+        help=("Deprecated: Used with the -S option, "
+              "Unix-domain socket to connect to.")
+        )
+
+    if not args:
+        parser.print_help()
+        return
+
+    def error(message):
+        sys.stderr.write("Error:\n%s\n" % message)
+        sys.exit(1)
+
+    options, args = parser.parse_args(args)
+
+    packt = time.time()
+    if options.time:
+        time_ = map(int, options.time.split(':'))
+        if len(time_) == 1:
+            time_ += (0, 0)
+        elif len(time_) == 2:
+            time_ += (0,)
+        elif len(time_) > 3:
+            error("Invalid time value: %r" % options.time)
+
+        packt = time.localtime(packt)
+        packt = time.mktime(packt[:3]+tuple(time_)+packt[6:])
+
+    packt -= options.days * 86400
+
+    servers = []
+
+    if options.host:
+        if not options.port:
+            error("If host (-h) is specified then a port (-p) must be "
+                  "specified as well.")
+        servers.append(((options.host, options.port), options.name))
+    elif options.port:
+        error("If port (-p) is specified then a host (-h) must be "
+              "specified as well.")
+
+    if options.unix:
+        servers.append((options.unix, options.name))
+
+    for server in args:
+        data = server.split(':')
+        if len(data) in (2, 3):
+            host = data[0]
+            try:
+                port = int(data[1])
+            except ValueError:
+                error("Invalid port in server specification: %r" % server)
+            addr = host, port
+            if len(data) == 2:
+                name = '1'
+            else:
+                name = data[2]
+        else:
+            error("Invalid server specification: %r" % server)
+
+        servers.append((addr, name))
+
+    for server in options.unix_sockets or ():
+        data = server.split(':')
+        if len(data) == 1:
+            addr = data[0]
+            name = '1'
+        elif len(data) == 2:
+            addr = data[0]
+            name = data[1]
+        else:
+            error("Invalid server specification: %r" % server)
+
+        servers.append((addr, name))
+
+    if not servers:
+        error("No servers specified.")
+
+    for addr, name in servers:
+        try:
+            cs = ZEO.ClientStorage.ClientStorage(
+                addr, storage=name, wait=False, read_only=1)
+            for i in range(60):
+                if cs.is_connected():
+                    break
+                time.sleep(1)
+            else:
+                sys.stderr.write("Couldn't connect to: %r\n"
+                                 % ((addr, name), ))
+                cs.close()
+                continue
+            cs.pack(packt, wait=True)
+            cs.close()
+        except:
+            traceback.print_exception(*(sys.exc_info()+(99, sys.stderr)))
+            error("Error packing storage %s in %r" % (name, addr))
+
+def main(*args):
+    root_logger = logging.getLogger()
+    old_level = root_logger.getEffectiveLevel()
+    logging.getLogger().setLevel(logging.WARNING)
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(logging.Formatter(
+        "%(name)s %(levelname)s %(message)s"))
+    logging.getLogger().addHandler(handler)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:d:W1')
-        for o, a in opts:
-            if o == '-p':
-                port = int(a)
-            elif o == '-h':
-                host = a
-            elif o == '-U':
-                unix = a
-            elif o == '-S':
-                storage = a
-            elif o == '-d':
-                days = int(a)
-            elif o == '-W':
-                wait = 1
-            elif o == '-1':
-                zeoversion = 1
-    except Exception, err:
-        print err
-        usage()
-
-    if unix is not None:
-        addr = unix
-    else:
-        if host is None:
-            host = socket.gethostname()
-        if port is None:
-            usage()
-        addr = host, port
-
-    if zeoversion == 1:
-        pack1(addr, storage, days, wait)
-    else:
-        pack2(addr, storage, days)
+        _main(*args)
+    finally:
+        logging.getLogger().setLevel(old_level)
+        logging.getLogger().removeHandler(handler)
 
 if __name__ == "__main__":
-    try:
-        main()
-    except Exception, err:
-        print err
-        sys.exit(1)
+    main()
+

src/ZEO/scripts/zeopack.test

+zeopack
+=======
+
+The zeopack script can be used to pack one or more storages.  It uses
+ClientStorage to do this.  To test it's behavior, we'll replace the
+normal ClientStorage with a fake one that echos information we'll want
+for our test:
+
+    >>> class ClientStorage:
+    ...     connect_wait = 0
+    ...     def __init__(self, *args, **kw):
+    ...         if args[0] == 'bad':
+    ...             import logging
+    ...             logging.getLogger('test.ClientStorage').error(
+    ...                 "I hate this address, %r", args[0])
+    ...             raise ValueError("Bad address")
+    ...         print "ClientStorage(%s %s)" % (
+    ...             repr(args)[1:-1],
+    ...             ', '.join("%s=%r" % i for i in sorted(kw.items())),
+    ...             )
+    ...     def pack(self, *args, **kw):
+    ...         print "pack(%s %s)" % (
+    ...             repr(args)[1:-1],
+    ...             ', '.join("%s=%r" % i for i in sorted(kw.items())),
+    ...             )
+    ...     def is_connected(self):
+    ...         self.connect_wait -= 1
+    ...         print 'is_connected', self.connect_wait < 0
+    ...         return self.connect_wait < 0
+    ...     def close(self):
+    ...         print "close()"
+
+    >>> import ZEO
+    >>> ClientStorage_orig = ZEO.ClientStorage.ClientStorage
+    >>> ZEO.ClientStorage.ClientStorage = ClientStorage
+
+Now, we're ready to try the script:
+
+    >>> from ZEO.scripts.zeopack import main
+
+If we call it with no arguments, we get help:
+
+>>> main([], 'zeopack')
+Usage: zeopack [options] [servers]
+<BLANKLINE>
+Pack one or more storages hosted by ZEO servers.
+<BLANKLINE>
+The positional arguments specify 0 or more tcp servers to pack, where
+each is of the form:
+<BLANKLINE>
+    host:port[:name]
+<BLANKLINE>
+<BLANKLINE>
+<BLANKLINE>
+Options:
+  -d DAYS, --days=DAYS  Pack objects that are older than this number of days
+  -t TIME, --time=TIME  Time of day to pack to of the form: HH[:MM[:SS]].
+                        Defaults to current time.
+  -u UNIX_SOCKETS, --unix=UNIX_SOCKETS
+                        A unix-domain-socket server to connect to, of the
+                        form: path[:name]
+  -h HOST               Deprecated: Used with the -p and -S options, specified
+                        the host to connect to.
+  -p PORT               Deprecated: Used with the -h and -S options, specifies
+                        the port to connect to.
+  -S NAME               Deprecated: Used with the -h and -p, options, or with
+                        the -U option specified the storage name to use.
+                        Defaults to 1.
+  -U UNIX               Deprecated: Used with the -S option, Unix-domain
+                        socket to connect to.
+
+Since packing involved time, we'd better have our way with it:
+
+    >>> import time
+    >>> time_orig = time.time
+    >>> time.time = lambda : 1237906517.0
+    >>> import os
+    >>> oldtz = os.environ.get('TZ')
+    >>> os.environ['TZ'] = 'PST+07'
+    >>> sleep_orig = time.sleep
+    >>> def sleep(t):
+    ...     print 'sleep(%r)' % t
+    >>> time.sleep = sleep
+
+Normally, we pass one or more TCP server specifications:
+
+    >>> main(["host1:8100", "host1:8100:2"])
+    ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
+    is_connected True
+    pack(1237906517.0, wait=True)
+    close()
+    ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+    is_connected True
+    pack(1237906517.0, wait=True)
+    close()
+
+We can also pass unix-domain-sockey servers using the -u option:
+
+    >>> main(["-ufoo", "-ubar:spam", "host1:8100", "host1:8100:2"])
+    ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
+    is_connected True
+    pack(1237906517.0, wait=True)
+    close()
+    ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+    is_connected True
+    pack(1237906517.0, wait=True)
+    close()
+    ClientStorage('foo', read_only=1, storage='1', wait=False)
+    is_connected True
+    pack(1237906517.0, wait=True)
+    close()
+    ClientStorage('bar', read_only=1, storage='spam', wait=False)
+    is_connected True
+    pack(1237906517.0, wait=True)
+    close()
+
+The -d option causes a pack time the given number of days earlier to
+be used:
+
+    >>> main(["-ufoo", "-ubar:spam", "-d3", "host1:8100", "host1:8100:2"])
+    ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
+    is_connected True
+    pack(1237647317.0, wait=True)
+    close()
+    ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+    is_connected True
+    pack(1237647317.0, wait=True)
+    close()
+    ClientStorage('foo', read_only=1, storage='1', wait=False)
+    is_connected True
+    pack(1237647317.0, wait=True)
+    close()
+    ClientStorage('bar', read_only=1, storage='spam', wait=False)
+    is_connected True
+    pack(1237647317.0, wait=True)
+    close()
+
+The -t option allows us to control the time of day:
+
+    >>> main(["-ufoo", "-d3", "-t1:30", "host1:8100:2"])
+    ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+    is_connected True
+    pack(1237624200.0, wait=True)
+    close()
+    ClientStorage('foo', read_only=1, storage='1', wait=False)
+    is_connected True
+    pack(1237624200.0, wait=True)
+    close()
+
+Connection timeout
+------------------
+
+The zeopack script tells ClientStorage not to wait for connections
+before returning from the constructor, but will time out after 60
+seconds of waiting for a connect.
+
+    >>> ClientStorage.connect_wait = 3
+    >>> main(["-d3", "-t1:30", "host1:8100:2"])
+    ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+    is_connected False
+    sleep(1)
+    is_connected False
+    sleep(1)
+    is_connected False
+    sleep(1)
+    is_connected True
+    pack(1237624200.0, wait=True)
+    close()
+
+    >>> def call_main(args):
+    ...     import sys
+    ...     old_stderr = sys.stderr
+    ...     sys.stderr = sys.stdout
+    ...     try:
+    ...         try:
+    ...             main(args)
+    ...         except SystemExit, v:
+    ...             print "Exited", v
+    ...     finally:
+    ...         sys.stderr = old_stderr
+
+    >>> ClientStorage.connect_wait = 999
+    >>> call_main(["-d3", "-t1:30", "host1:8100", "host1:8100:2"])
+    ... # doctest: +ELLIPSIS
+    ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
+    is_connected False
+    sleep(1)
+    ...
+    is_connected False
+    sleep(1)
+    Couldn't connect to: (('host1', 8100), '1')
+    close()
+    ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+    is_connected False
+    sleep(1)
+    ...
+    is_connected False
+    sleep(1)
+    Couldn't connect to: (('host1', 8100), '2')
+    close()
+
+    >>> ClientStorage.connect_wait = 0
+
+
+Legacy support
+--------------
+
+    >>> main(["-d3", "-h", "host1", "-p", "8100", "-S", "2"])
+    ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+    is_connected True
+    pack(1237647317.0, wait=True)
+    close()
+
+    >>> main(["-d3", "-U", "foo/bar", "-S", "2"])
+    ClientStorage('foo/bar', read_only=1, storage='2', wait=False)
+    is_connected True
+    pack(1237647317.0, wait=True)
+    close()
+
+Error handling
+--------------
+
+    >>> call_main(["-d3"])
+    Error:
+    No servers specified.
+    Exited 1
+
+    >>> call_main(["-d3", "a"])
+    Error:
+    Invalid server specification: 'a'
+    Exited 1
+
+    >>> call_main(["-d3", "a:b:c:d"])
+    Error:
+    Invalid server specification: 'a:b:c:d'
+    Exited 1
+
+    >>> call_main(["-d3", "a:b:2"])
+    Error:
+    Invalid port in server specification: 'a:b:2'
+    Exited 1
+
+    >>> call_main(["-d3", "-u", "a:b:2"])
+    Error:
+    Invalid server specification: 'a:b:2'
+    Exited 1
+
+    >>> call_main(["-d3", "-u", "bad"]) # doctest: +ELLIPSIS
+    test.ClientStorage ERROR I hate this address, 'bad'
+    Traceback (most recent call last):
+    ...
+    ValueError: Bad address
+    Error:
+    Error packing storage 1 in 'bad'
+    Exited 1
+
+
+
+Note that in the previous example, the first line was output through logging.
+
+.. tear down
+
+    >>> ZEO.ClientStorage.ClientStorage = ClientStorage_orig
+    >>> time.time = time_orig
+    >>> time.sleep = sleep_orig
+    >>> if oldtz is None:
+    ...     del os.environ['TZ']
+    ... else:
+    ...     os.environ['TZ'] = oldtz