merged console-scripts branch

setup.py

@@ -48,30 +48,6 @@ setup(name='Zope2',
       packages=find_packages('src'),
       package_dir={'': 'src'},
 
-      data_files=[
-        ('skel', ['skel/README.txt']),
-        ('skel/bin', ['skel/bin/runzope.bat.in',
-                      'skel/bin/runzope.in',
-                      'skel/bin/zopectl.bat.in',
-                      'skel/bin/zopectl.in',
-                      'skel/bin/zopeservice.py.in',
-                     ]),
-        ('skel/etc', ['skel/etc/site.zcml',
-                      'skel/etc/zope.conf.in',
-                     ]),
-        ('skel/etc/package-includes', []),
-        ('skel/Extensions', ['skel/Extensions/README.txt']),
-        ('skel/import', ['skel/import/README.txt',
-                         'skel/import/ZopeTutorialExamples.zexp',
-                        ]),
-        ('skel/lib/python', ['skel/lib/python/README.txt.in' ]),
-        ('skel/log', ['skel/log/README.txt']),
-        ('skel/Products', ['skel/Products/__init__.py',
-                           'skel/Products/README.txt',
-                          ]),
-        ('skel/var', ['skel/var/README.txt']),
-      ],
-
       ext_modules=[
 
         # AccessControl
@@ -265,4 +241,10 @@ setup(name='Zope2',
 
       include_package_data=True,
       zip_safe=False,
+      entry_points={
+         'console_scripts' : [
+            'mkzopeinstance=Zope2.utilities.mkzopeinstance:main',
+            'zpasswd=Zope2.utilities.zpasswd:main',
+        ]
+      },
       )

src/Zope2/utilities/README.txt

+This directory contains utility scripts and modules that augment Zope.
+
+To get detailed usage information, run any of these scripts without arguments:
+
+  load_site.py -- Load a Zope site from files and directories
+
+    This script illustrates used of the Zope RPC mechanism,
+    ZPublisher.Client.  It provides some examples of pitfalls
+    and their work-arounds.
+
+  check_catalog.py -- Perform some consistency tests on a ZCatalog instance
+
+  mkzopeinstance.py -- create a Zope instance home
+
+  copyzopeskel.py -- copy a Zope instance home skeleton directory to target
+
+  mkzeoinstance.py -- create a ZEO instance home
+
+  requestprofiler.py -- parse and analyze the Zope "detailed" log file
+
+  zpasswd.py -- generate "access" or "inituser" files for use with Zope
+
+  compilezpy.py -- compile all .py files to .pyc files in the current
+     directory and below
+
+  decompilezpy.py -- remove all .py[co] files in the current directory
+     and below
+

src/Zope2/utilities/__init__.py

+# placeholder

src/Zope2/utilities/check_catalog.py

+#!/usr/bin/env python2.4
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Script to check consistency of a ZCatalog
+
+$Id$
+"""
+import Zope2
+import os,sys,re,getopt
+from types import IntType
+from BTrees.IIBTree import IISet,difference,intersection
+
+def checkCatalog(path,indexes):
+    """ perform some consistency checks on a ZCatalog instance"""
+
+    root = Zope2.app()
+
+    try:
+        catalog = root.unrestrictedTraverse(path)
+    except AttributeError:
+        print 'Error: catalog object not found'
+        sys.exit(1)
+
+    # get Catalog instance
+    _cat = catalog._catalog
+
+    # check Catalog internal BTrees
+    l_data  = list(_cat.data.keys())
+    l_data.sort()
+    l_uids  = list(_cat.uids.values())
+    l_uids.sort()
+    l_paths = list(_cat.data.keys())
+    l_paths.sort()
+
+    print "Checking catalog internal BTrees"
+    print "\tINFO: Mapping data:  %d entries" % len(l_data)
+    print "\tINFO: Mapping uids:  %d entries" % len(l_uids)
+    print "\tINFO: Mapping paths: %d entries" % len(l_paths)
+
+    if l_data == l_uids:
+        print "\tOK:  Mapping data equals Mapping uids"
+    else:
+        print "\tERR: Mapping data does not equal Mapping uids"
+
+    if l_data == l_paths:
+        print "\tOK:  Mapping data equals Maaping paths"
+    else:
+        print "\tERR: Mapping data does not equal Maaping paths"
+
+
+    # check BTrees of indexes
+
+    for id,idx in _cat.indexes.items():
+
+        if indexes and not idx.meta_type in indexes: continue
+
+        print "Checking index '%s' (type: %s)" % (id, idx.meta_type)
+
+        if idx.meta_type in ['FieldIndex','KeywordIndex']:
+
+            # check forward entries
+            RIDS = IISet()
+            for key, rids in idx._index.items():
+                if isinstance(rids,IntType):
+                    RIDS.insert(  rids  )
+                else:
+                    map(RIDS.insert , rids.keys())
+
+            diff = difference(RIDS, IISet(_cat.data.keys()))
+            if len(diff)!=0:
+                print '\tERR: Problem with forward entries'
+                print '\tERR: too much forward entries:', diff
+            else:
+                print '\tOK:  Forward entries (%d entries)'  % (len(RIDS))
+
+
+        elif idx.meta_type in ['PathIndex']:
+
+            RIDS = IISet()
+
+            for rids in map(None,idx._index.values()):
+                map(RIDS.insert , rids.values()[0])
+
+            diff = difference(RIDS, IISet(_cat.data.keys()))
+            if len(diff)!=0:
+                print '\tERR: Problem with forward entries'
+                print '\tERR: too much forward entries:', diff
+            else:
+                print '\tOK:  Forward entries (%d entries)'  % (len(RIDS))
+
+
+        if idx.meta_type in ['FieldIndex','KeywordIndex','PathIndex']:
+
+            # check backward entries
+            RIDS = IISet(idx._unindex.keys())
+            diff = difference(RIDS, IISet(_cat.data.keys()))
+            if len(diff)!=0:
+                print '\tERR: Problem with backward entries'
+                print '\tERR: too much backward entries:', diff
+            else:
+                print '\tOK:  Backward entries (%d entries)'  % (len(RIDS))
+
+
+
+
+
+
+
+def usage():
+    print "Usage: %s [--FieldIndex|KeywordIndex|PathIndex] /path/to/ZCatalog" % \
+                os.path.basename(sys.argv[0])
+    print
+    print "This scripts checks the consistency of the internal"
+    print "BTrees of a ZCatalog and its indexes."
+    sys.exit(1)
+
+
+def main():
+
+    opts,args = getopt.getopt(sys.argv[1:],'h',\
+            ['help','FieldIndex','KeywordIndex','PathIndex'])
+
+    indexes = []
+
+    for o,v in opts:
+
+        if o in ['-h','--help']: usage()
+        if o in ['--FieldIndex','--KeywordIndex','--PathIndex']:
+            indexes.append(o[2:])
+
+    checkCatalog(args,indexes)
+
+
+if __name__=='__main__':
+    main()

src/Zope2/utilities/compilezpy.py

+#!/usr/bin/env python2.4
+
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import compileall, os, sys
+
+class Shutup:
+    def write(*args): pass # :)
+
+class NoteErr:
+    wrote = 0
+    def write(self, *args):
+        self.wrote = 1
+        apply(stderr.write, args)
+
+def compile_non_test(dir):
+    """Byte-compile all modules except those in test directories."""
+    success = compileall.compile_dir(dir, maxlevels=0)
+    try:
+        names = os.listdir(dir)
+    except os.error:
+        print "Can't list", dir
+        names = []
+    names.sort()
+    for name in names:
+        fullname = os.path.join(dir, name)
+        if (name != os.curdir and name != os.pardir and
+            os.path.isdir(fullname) and not os.path.islink(fullname) and
+            name != 'test' and name != 'tests' and name != 'skins'):
+            success = success and compile_non_test(fullname)
+    return success
+
+print
+print '-'*78
+print 'Compiling python modules'
+stdout = sys.stdout
+stderr = sys.stderr
+try:
+    try:
+        success = 0
+        sys.stdout = Shutup()
+        sys.stderr = NoteErr()
+        success = compile_non_test(os.getcwd())
+    finally:
+        success = success and not sys.stderr.wrote
+        sys.stdout = stdout
+        sys.stderr = stderr
+except:
+    success = 0
+    import traceback
+    traceback.print_exc()
+
+if not success:
+    print
+    print '!' * 78
+    print 'There were errors during Python module compilation.'
+    print '!' * 78
+    print
+    sys.exit(1)

src/Zope2/utilities/copyzopeskel.py

+#!/usr/bin/env python2.4
+
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""%(program)s:  Create a target directory from a skeleton directory.
+
+usage:  %(program)s [options]
+
+Options:
+-h/--help -- print this help text
+-s/--sourcedir -- the skeleton source directory
+-t/--targetdir -- the directory to which the skeleton files will be copied
+-u/--uid       -- the username/uid of the user who will own the target files
+-g/--gid       -- the groupname/gid of the group who will own the target files
+-r/--replace   -- specify replacement value for .in file
+
+This script may be used to install a custom Zope instance home
+skeleton directory.  It is most useful when used to install a skeleton
+which does not follow the standard 'one-directory-as-instance-home'
+paradigm used by the stock Zope source install.
+
+The value of --targetdir should be the directory where you would like to copy
+the skeleton hierarchy.  For many packagers, this will be "/" or "/usr"
+or "/usr/local".
+
+The value of --sourcedir should be a directory which contains a custom skeleton
+hierarchy.   For many packagers, the skeleton source directory may contain
+directories like "usr" and "bin" and these directories will contain files
+and other directories, comprising an installation hierarchy suitable for your
+platform.
+
+The skeleton source hierarchy may contain any kind of file.  Files
+in the skeleton hierarchy that end with a ".in" extension will go through
+textual substitution before they are placed in the target directory.  When
+they are placed in the target directory, the ".in" extension is removed.
+
+Specify textual replacement values by passing one or more --replace= options
+to the script.  The value of each replace option needs to be
+in the following format: --replace=key:value.  'key' is the value that
+will be replaced (minus the "<<" and ">>" values expected by the
+replacement).  'value' is the value that should be used for replacement.
+
+Files which do not have an ".in" extension are copied without substitution.
+All file mode bits from files/dirs in the skeleton source directory are copied
+along with the file/directory itself.  If the --uid and/or --gid flags are
+used, all directories and files created by this script will be provided with
+this owner and/or group.  Otherwise, the uid and group owner of the files will
+be the executing user's.  Existing directory structures and files are left
+unchanged.  If a file already exists in the target directory, it is left
+unchanged and the source file is not copied.  If a directory already exists in
+the target directory, its ownership information and mode bit settings are left
+unchanged.
+"""
+
+import os
+import shutil
+import sys
+import getopt
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:],
+            "hs:t:u:g:r:",
+            ["help", "sourcedir=", "targetdir=", "uid=", "gid=",
+             "replace="]
+            )
+    except getopt.GetoptError, msg:
+        usage(sys.stderr, msg)
+        sys.exit(2)
+
+    script = os.path.abspath(sys.argv[0])
+    sourcedir = None
+    targetdir = None
+    uid = None
+    gid = None
+    replacements = {}
+
+    for opt, arg in opts:
+        if opt in ("-h", "--help"):
+            usage(sys.stdout)
+            sys.exit()
+        if opt in ("-s", "--sourcedir"):
+            sourcedir = os.path.abspath(os.path.expanduser(arg))
+            if not sourcedir:
+                usage(sys.stderr, "sourcedir must not be empty")
+                sys.exit(2)
+        if opt in ("-t", "--targetdir"):
+            targetdir = os.path.abspath(os.path.expanduser(arg))
+            if not targetdir:
+                usage(sys.stderr, "targetdir must not be empty")
+                sys.exit(2)
+        if opt in ("-u", "--uid"):
+            if not arg:
+                usage(sys.stderr, "uid must not be empty")
+                sys.exit(2)
+            try:
+                if os.getuid() != 0:
+                    usage(sys.stderr, "You must be root to specify a uid")
+                    sys.exit(2)
+                try:
+                    uid = int(arg)
+                except:
+                    try:
+                        import pwd
+                        uid = pwd.getpwnam(arg)[2]
+                        if not gid:
+                            gid = pwd.getpwnam(arg)[3]
+                    except KeyError:
+                        usage(sys.stderr,
+                              "The user indicated by uid does not exist on "
+                              "your system")
+                        sys.exit(2)
+            except (ImportError, AttributeError):
+                usage(sys.stderr,
+                       "Your system does not support the gid or uid options")
+                sys.exit(2)
+        if opt in ("-g", "--gid"):
+            if not arg:
+                usage(sys.stderr, "gid must not be empty")
+                sys.exit(2)
+            try:
+                if os.getuid() != 0:
+                    usage(sys.stderr, "You must be root to specify a gid")
+                    sys.exit(2)
+                try:
+                    gid = int(arg)
+                except:
+                    try:
+                        import pwd
+                        gid = pwd.getpwnam(arg)[3]
+                    except KeyError:
+                        usage(sys.stderr,
+                              "The user indicated by gid does not exist on "
+                              "your system")
+                        sys.exit(2)
+            except (ImportError, AttributeError):
+                usage(sys.stderr,
+                       "Your system does not support the gid or uid options")
+                sys.exit(2)
+
+        if opt in ("-r", "--replace"):
+            if not arg:
+                continue
+            k, v = arg.split(':', 1)
+            replacements[k] = v
+
+    if not sourcedir:
+        usage(sys.stderr, "Must specify sourcedir")
+        sys.exit(2)
+
+    if not targetdir:
+        usage(sys.stderr, "Must specify targetdir")
+        sys.exit(2)
+
+    copyskel(sourcedir, targetdir, uid, gid, **replacements)
+
+def copyskel(sourcedir, targetdir, uid, gid, **replacements):
+    """ This is an independent function because we'd like to
+    import and call it from mkzopeinstance """
+    # Create the top of the instance:
+    if not os.path.exists(targetdir):
+        os.makedirs(targetdir)
+
+    # This is fairly ugly.  The chdir() makes path manipulation in the
+    # walk() callback a little easier (less magical), so we'll live
+    # with it.
+    pwd = os.getcwd()
+    os.chdir(sourcedir)
+    try:
+        try:
+            os.path.walk(os.curdir, copydir,
+                         (targetdir, replacements, uid, gid))
+        finally:
+            os.chdir(pwd)
+    except (IOError, OSError), msg:
+        print >>sys.stderr, msg
+        sys.exit(1)
+
+CVS_DIRS = [os.path.normcase("CVS"), os.path.normcase(".svn")]
+
+def copydir((targetdir, replacements, uid, gid), sourcedir, names):
+    # Don't recurse into CVS directories:
+    for name in names[:]:
+        if os.path.normcase(name) in CVS_DIRS:
+            names.remove(name)
+        elif os.path.isfile(os.path.join(sourcedir, name)):
+            # Copy the file:
+            sn, ext = os.path.splitext(name)
+            if os.path.normcase(ext) == ".in":
+                dst = os.path.join(targetdir, sourcedir, sn)
+                if os.path.exists(dst):
+                    continue
+                copyin(os.path.join(sourcedir, name), dst, replacements, uid,
+                       gid)
+                if uid is not None:
+                    os.chown(dst, uid, gid)
+            else:
+                src = os.path.join(sourcedir, name)
+                dst = os.path.join(targetdir, src)
+                if os.path.exists(dst):
+                    continue
+                shutil.copyfile(src, dst)
+                shutil.copymode(src, dst)
+                if uid is not None:
+                    os.chown(dst, uid, gid)
+        else:
+            # Directory:
+            dn = os.path.join(targetdir, sourcedir, name)
+            if not os.path.exists(dn):
+                os.mkdir(dn)
+                shutil.copymode(os.path.join(sourcedir, name), dn)
+                if uid is not None:
+                    os.chown(dn, uid, gid)
+
+def copyin(src, dst, replacements, uid, gid):
+    ifp = open(src)
+    text = ifp.read()
+    ifp.close()
+    for k in replacements:
+        text = text.replace("<<%s>>" % k, replacements[k])
+    ofp = open(dst, "w")
+    ofp.write(text)
+    ofp.close()
+    shutil.copymode(src, dst)
+    if uid is not None:
+        os.chown(dst, uid, gid)
+
+def usage(stream, msg=None):
+    if msg:
+        print >>stream, msg
+        print >>stream
+    program = os.path.basename(sys.argv[0])
+    print >>stream, __doc__ % {"program": program}
+
+if __name__ == '__main__':
+    main()

src/Zope2/utilities/decompilezpy.py

+#!/usr/bin/env python2.4
+
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import os
+import sys
+
+def main(dirname):
+    os.path.walk(dirname, rmpycs, None)
+
+def rmpycs(arg, dirname, names):
+    for name in names:
+        path = os.path.join(dirname, name)
+        if ( name.endswith('.pyc') or name.endswith('.pyo') and
+             os.path.isfile(path) ):
+            os.unlink(path)
+
+if __name__ == '__main__':
+    main(sys.argv[1])

src/Zope2/utilities/fixheaders.py

+
+# Script to fix the header files to ZPL 2.1
+
+import os
+
+for dirpath, dirnames, filenames in os.walk('.'):
+
+    for fname in filenames:
+        base,ext = os.path.splitext(fname)
+        if not ext in ('.py', '.c', '.h'): continue
+        fullname = os.path.join(dirpath, fname)
+        if '.svn' in fullname: continue
+        data = open(fullname).read()
+
+        changed = False
+        if 'Version 2.1 (ZPL)' in data:
+            data = data.replace('Version 2.1 (ZPL)', 'Version 2.1 (ZPL)')
+            changed = True
+
+        if '(c) 2002 Zope Corporation' in data:
+            data = data.replace('(c) 2002 Zope Corporation', '(c) 2002 Zope Corporation')
+            changed = True
+
+        print fullname, changed
+        if changed:
+            open(fullname, 'w').write(data)

src/Zope2/utilities/instance.py

+import optparse
+import os
+import os.path
+import sys
+import re
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+# I would use string.Template, but it's just too hard to change it to
+# respect only ${brace} syntax instead of both that and $name syntax.
+TOKEN_RE = re.compile(r'\$\{([\.\w/-]+)\}')
+def rewrite(repltext, **kw):
+    def replace(match):
+        return kw[match.group(1)]
+    return TOKEN_RE.sub(replace, repltext)
+
+def mkinstance(conf):
+    sandbox = conf['sandbox']
+        
+    for dir in ('bin', 'etc', 'var', 'Products', 'import', 'log'):
+        path = os.path.join(sandbox, dir)
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+    try:
+        # this is (stupidly) required by the zdrun key in the <runner>
+        # section of zeo.conf
+        import zdaemon
+        conf['zdaemon_pkgdir'] = zdaemon.__path__[0]
+    except ImportError:
+        conf['zdaemon_pkgdir'] = '{unknown}'
+
+    zope_conf = conf['zope_conf']
+    site_zcml = conf['site_zcml']
+        
+    for source, target in (
+        ('zope2.ini', 'zope2.ini'),
+        (site_zcml, 'site.zcml'),
+        (zope_conf, 'zope.conf'),
+        ('apache2.conf', 'apache2.conf'),
+        ('zeo.conf', 'zeo.conf'),
+        ):
+        template = open(os.path.join(here, 'etc', source), 'r').read()
+        result = rewrite(template, **conf)
+        targetfile = os.path.join(sandbox, 'etc', target)
+        if not os.path.exists(targetfile):
+            open(targetfile, 'w').write(result)
+
+    template = open(os.path.join(here, 'etc', 'zope2.wsgi'), 'r').read()
+    result = rewrite(template, **conf)
+    targetfile = os.path.join(sandbox, 'bin', 'zope2.wsgi')
+    if not os.path.exists(targetfile):
+        open(targetfile, 'w').write(result)
+        os.chmod(targetfile, 0755)
+
+def main(argv=sys.argv):
+    """ Console script target """
+    parser = optparse.OptionParser(
+        usage='%prog [OPTIONS]'
+        )
+    parser.add_option('-s', '--sandbox', action='store', dest='sandbox',
+                      default='.', help='Create the instance in this directory')
+    parser.add_option('-p', '--zope-port', action='store', dest='zope_port',
+                      default='8080', help='Zope HTTP port')
+    parser.add_option('-j', '--zeo-port', action='store', dest='zeo_port',
+                      default='8100', help='ZEO server port number')
+    parser.add_option('-z', '--use-zeo', action='store_true', dest='use_zeo',
+                      default=False, help='Use ZEO to house main storage')
+    options, args = parser.parse_args(argv)
+    try:
+        # Zope 2.10+ (Five 1.5.3?+)
+        from Products.Five.fivedirectives import IRegisterPackageDirective
+        options.site_zcml = 'zope-2.10+-site.zcml'
+    except ImportError:
+        # Zope 2.9 (or Five before 1.5.3?)
+        options.site_zcml = 'zope-2.9-site.zcml'
+    if options.use_zeo:
+        options.zope_conf = 'zope-zeoclient.conf'
+    else:
+        options.zope_conf = 'zope-nonzeoclient.conf'
+    options.python = sys.executable
+    conf = options.__dict__
+    sandbox = conf['sandbox']
+    conf['sandbox'] = os.path.abspath(os.path.normpath(os.path.expanduser(
+        sandbox)))
+    mkinstance(conf)

src/Zope2/utilities/load_site.py

+#!/usr/bin/env python2.4
+
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Load a Zope site from a collection of files or directories
+"""
+
+usage=""" [options] url file .....
+
+    where options are:
+
+      -D
+
+         For HTML documents, replace the start of the content, up to
+         and including the opening body tag with a DTML var tag that
+         inserts the standard header. Also replace the closing body
+         and html tag with a DTML var tag that inserts the standard
+         footer.
+
+      -I
+
+         For each index.html, add an index_html that redirects.
+
+      -p path
+
+         Path to ZPublisher.  If not provided, load_site will
+         make an attempt to figure it out.
+
+      -u user:password
+
+         Credentials
+
+      -v
+
+         Run in verbose mode.
+
+      -9
+
+         Use *old* zope method names.
+"""
+
+import sys, getopt, os, string
+ServerError=''
+verbose=0
+old=0
+doctor=0
+index_html=0
+
+def main():
+    user, password = 'superuser', '123'
+    opts, args = getopt.getopt(sys.argv[1:], 'p:u:DIv9')
+    global verbose
+    global old
+    global doctor
+    global index_html
+    havepath=None
+    for o, v in opts:
+        if o=='-p':
+            d, f = os.path.split(v)
+            if f=='ZPublisher': sys.path.insert(0,d)
+            else: sys.path.insert(0,v)
+            havepath=1
+        elif o=='-u':
+            v = string.split(v,':')
+            user, password = v[0], string.join(v[1:],':')
+        elif o=='-D': doctor=1
+        elif o=='-I': index_html=1
+        elif o=='-v': verbose=1
+        elif o=='-9': old=1
+
+    if not args:
+        print sys.argv[0]+usage
+        sys.exit(1)
+
+    if not havepath:
+        here=os.path.split(sys.argv[0])[0]
+        if os.path.exists(os.path.join(here,'ZPublisher')):
+            sys.path.insert(0,here)
+        else:
+            here=os.path.split(here)[0]
+            here=os.path.join(here,'lib','python')
+            if os.path.exists(os.path.join(here,'ZPublisher')):
+                sys.path.insert(0,here)
+
+
+    url=args[0]
+    files=args[1:]
+
+    import ZPublisher.Client
+    global ServerError
+    ServerError=ZPublisher.Client.ServerError
+    object=ZPublisher.Client.Object(url, username=user, password=password)
+
+    for f in files: upload_file(object, f)
+
+def call(f, *args, **kw):
+    # Call a function ignoring redirect bci errors.
+    try: apply(f,args, kw)
+    except ServerError, v:
+        if str(v)[:1] != '3':
+            raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
+
+def upload_file(object, f):
+    if os.path.isdir(f): return upload_dir(object, f)
+    dir, name = os.path.split(f)
+    root, ext = os.path.splitext(name)
+    if ext in ('file', 'dir'): ext=''
+    else:
+        ext=string.lower(ext)
+        if ext and ext[0] in '.': ext=ext[1:]
+    if ext and globals().has_key('upload_'+ext):
+        if verbose: print 'upload_'+ext, f
+        return globals()['upload_'+ext](object, f)
+
+    if verbose: print 'upload_file', f, ext
+    call(object.manage_addFile, id=name, file=open(f,'rb'))
+
+def upload_dir(object, f):
+    if verbose: print 'upload_dir', f
+    dir, name = os.path.split(f)
+    call(object.manage_addFolder, id=name)
+    object=object.__class__(object.url+'/'+name,
+                            username=object.username,
+                            password=object.password)
+    for n in os.listdir(f):
+        upload_file(object, os.path.join(f,n))
+
+# ----- phd -----
+# Modified by Oleg Broytmann <phd2@earthling.net>
+
+from sgmllib import SGMLParser
+
+def join_attrs(attrs):
+    attr_list = []
+    for attrname, value in attrs:
+        attr_list.append('%s="%s"' % (attrname, string.strip(value)))
+
+    if attr_list:
+        s = " " + string.join(attr_list, " ")
+    else:
+        s = ""
+
+    return s
+
+
+class HeadParser(SGMLParser):
+    def __init__(self):
+        SGMLParser.__init__(self)
+
+        self.seen_starthead = 0
+        self.seen_endhead   = 0
+        self.seen_startbody = 0
+
+        self.head = ""
+        self.title = ""
+        self.accumulator = ""
+
+
+    def handle_data(self, data):
+        if data:
+            self.accumulator = self.accumulator + data
+
+    def handle_charref(self, ref):
+        self.handle_data("&#%s;" % ref)
+
+    def handle_entityref(self, ref):
+        self.handle_data("&%s;" % ref)
+
+    def handle_comment(self, data):
+        if data:
+            self.accumulator = self.accumulator + "<!--%s-->" % data
+
+
+    def start_head(self, attrs):
+        if not self.seen_starthead:
+            self.seen_starthead = 1
+            self.head = ""
+            self.title = ""
+            self.accumulator = ""
+
+    def end_head(self):
+        if not self.seen_endhead:
+            self.seen_endhead = 1
+            self.head = self.head + self.accumulator
+            self.accumulator = ""
+
+
+    def start_title(self, attrs):
+        self.head = self.head + self.accumulator
+        self.accumulator = ""
+
+    def end_title(self):
+        self.title = self.accumulator
+        self.accumulator = ""
+
+
+    def start_body(self, attrs):
+        if not self.seen_startbody:
+            self.seen_startbody = 1
+            self.accumulator = ""
+
+    def end_body(self): pass # Do not put </BODY> and </HTML>
+    def end_html(self): pass # into output stream
+
+
+    # Pass other tags unmodified
+    def unknown_starttag(self, tag, attrs):
+        self.accumulator = self.accumulator + "<%s%s>" % (string.upper(tag), join_attrs(attrs))
+
+    def unknown_endtag(self, tag):
+        self.accumulator = self.accumulator + "</%s>" % string.upper(tag)
+
+
+
+def parse_html(infile):
+    parser = HeadParser()
+
+    while 1:
+        line = infile.readline()
+        if not line: break
+        parser.feed(line)
+
+    parser.close()
+    infile.close()
+
+    return (string.strip(parser.title), string.strip(parser.head),
+            string.strip(parser.accumulator))
+
+
+def upload_html(object, f):
+    dir, name = os.path.split(f)
+    f=open(f)
+
+    if doctor:
+        title, head, body = parse_html(f)
+        if old:
+            body = ("<!--#var standard_html_header-->\n\n" +
+                    body + "\n\n<!--#var standard_html_footer-->")
+        else:
+            body = ("<dtml-var standard_html_header>\n\n" +
+                    body + "\n\n<dtml-var standard_html_footer>")
+
+    else:
+        if old: f=f.read()
+        title, head, body = '', '', f
+
+    if old:
+        call(object.manage_addDocument, id=name, file=body)
+        if index_html and name in ('index.html', 'index.htm'):
+            call(object.manage_addDocument, id='index_html',
+                 file=('<!--#raise Redirect-->'
+                       '<!--#var URL1-->/%s'
+                       '<!--#/raise-->' % name
+                       ))
+    else:
+        call(object.manage_addDTMLDocument, id=name, title=title, file=body)
+        if index_html and name in ('index.html', 'index.htm'):
+            call(object.manage_addDTMLMethod, id='index_html',
+                 file=('<dtml-raise Redirect>'
+                       '<dtml-var URL1>/%s'
+                       '</dtml-raise>' % name
+                       ))
+
+    # Now add META and other tags as property
+    if head:
+        object=object.__class__(object.url+'/'+name,
+                              username=object.username,
+                              password=object.password)
+        call(object.manage_addProperty,
+             id="loadsite-head", type="text", value=head)
+
+# ----- /phd -----
+
+upload_htm=upload_html
+
+def upload_dtml(object, f):
+    dir, name = os.path.split(f)
+    f=open(f)
+
+    if old:
+        f=f.read()
+        call(object.manage_addDocument, id=name, file=f)
+    else:
+        call(object.manage_addDTMLMethod, id=name, file=f)
+
+
+def upload_gif(object, f):
+    dir, name = os.path.split(f)
+    call(object.manage_addImage, id=name, file=open(f,'rb'))
+
+upload_jpg=upload_gif
+upload_png=upload_gif
+
+if __name__=='__main__': main()

src/Zope2/utilities/mkzeoinstance.py

+#!/usr/bin/env python2.4
+
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import os
+import sys
+
+mydir = os.path.dirname(os.path.abspath(sys.argv[0]))
+zopehome = os.path.dirname(mydir)
+softwarehome = os.path.join(zopehome, "lib", "python")
+
+if softwarehome not in sys.path:
+    sys.path.insert(0, softwarehome)
+
+from ZEO.mkzeoinst import ZEOInstanceBuilder
+
+if __name__ == "__main__":
+    ZEOInstanceBuilder().run()

src/Zope2/utilities/mkzopeinstance.py

+#!/usr/bin/env python2.4
+
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+"""%(program)s:  Create a Zope instance home.
+
+usage:  %(program)s [options]
+
+Options:
+-h/--help -- print this help text
+-d/--dir  -- the dir in which the instance home should be created
+-u/--user NAME:PASSWORD -- set the user name and password of the initial user
+-s/--skelsrc -- the dir from which skeleton files should be copied
+
+When run without arguments, this script will ask for the information necessary
+to create a Zope instance home.
+"""
+
+import getopt
+import os
+import shutil
+import sys
+import copyzopeskel
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:],
+            "hu:d:s:",
+            ["help", "user=", "dir=", "skelsrc="]
+            )
+    except getopt.GetoptError, msg:
+        usage(sys.stderr, msg)
+        sys.exit(2)
+
+    script = os.path.abspath(sys.argv[0])
+    user = None
+    password = None
+    skeltarget = None
+    skelsrc = None
+
+    for opt, arg in opts:
+        if opt in ("-d", "--dir"):
+            skeltarget = os.path.abspath(os.path.expanduser(arg))
+            if not skeltarget:
+                usage(sys.stderr, "dir must not be empty")
+                sys.exit(2)
+        if opt in ("-s", "--skelsrc"):
+            skelsrc = os.path.abspath(os.path.expanduser(arg))
+            if not skelsrc:
+                usage(sys.stderr, "skelsrc must not be empty")
+                sys.exit(2)
+        if opt in ("-h", "--help"):
+            usage(sys.stdout)
+            sys.exit()
+        if opt in ("-u", "--user"):
+            if not arg:
+                usage(sys.stderr, "user must not be empty")
+                sys.exit(2)
+            if not ":" in arg:
+                usage(sys.stderr, "user must be specified as name:password")
+                sys.exit(2)
+            user, password = arg.split(":", 1)
+
+    if not skeltarget:
+        # interactively ask for skeltarget and initial user name/passwd.
+        # cant set custom instancehome in interactive mode, we default
+        # to skeltarget.
+        skeltarget = instancehome = os.path.abspath(
+            os.path.expanduser(get_skeltarget())
+            )
+
+    instancehome = skeltarget
+    zopehome = os.path.dirname(os.path.dirname(script))
+    softwarehome = os.path.join(zopehome, "src")
+    configfile = os.path.join(instancehome, 'etc', 'zope.conf')
+    if skelsrc is None:
+        # default to using stock Zope skeleton source
+        skelsrc = os.path.join(os.path.dirname(__file__), "skel")
+
+    inituser = os.path.join(instancehome, "inituser")
+    if not (user or os.path.exists(inituser)):
+        user, password = get_inituser()
+
+    # we need to distinguish between python.exe and pythonw.exe under
+    # Windows.  Zope is always run using 'python.exe' (even for services),
+    # however, it may be installed via pythonw.exe (as a sub-process of an
+    # installer).  Thus, sys.executable may not be the executable we use.
+    # We still provide both PYTHON and PYTHONW, but PYTHONW should never
+    # need be used.
+    psplit = os.path.split(sys.executable)
+    exedir = os.path.join(*psplit[:-1])
+    pythonexe = os.path.join(exedir, 'python.exe')
+    pythonwexe = os.path.join(exedir, 'pythonw.exe')
+
+    if ( os.path.isfile(pythonwexe) and os.path.isfile(pythonexe) and
+         (sys.executable in [pythonwexe, pythonexe]) ):
+        # we're using a Windows build with both python.exe and pythonw.exe
+        # in the same directory
+        PYTHON = pythonexe
+        PYTHONW = pythonwexe
+    else:
+        # we're on UNIX or we have a nonstandard Windows setup
+        PYTHON = PYTHONW = sys.executable
+
+    kw = {
+        "PYTHON":PYTHON,
+        "PYTHONW":PYTHONW,
+        "INSTANCE_HOME": instancehome,
+        "SOFTWARE_HOME": softwarehome,
+        "ZOPE_HOME": zopehome,
+        }
+
+    copyzopeskel.copyskel(skelsrc, skeltarget, None, None, **kw)
+    if user and password:
+        write_inituser(inituser, user, password)
+
+def usage(stream, msg=None):
+    if msg:
+        print >>stream, msg
+        print >>stream
+    program = os.path.basename(sys.argv[0])
+    print >>stream, __doc__ % {"program": program}
+
+def get_skeltarget():
+    print 'Please choose a directory in which you\'d like to install'
+    print 'Zope "instance home" files such as database files, configuration'
+    print 'files, etc.'
+    print
+    while 1:
+        skeltarget = raw_input("Directory: ").strip()
+        if skeltarget == '':
+            print 'You must specify a directory'
+            continue
+        else:
+            break
+    return skeltarget
+
+def get_inituser():
+    import getpass
+    print 'Please choose a username and password for the initial user.'
+    print 'These will be the credentials you use to initially manage'
+    print 'your new Zope instance.'
+    print
+    user = raw_input("Username: ").strip()
+    if user == '':
+        return None, None
+    while 1:
+        passwd = getpass.getpass("Password: ")
+        verify = getpass.getpass("Verify password: ")
+        if verify == passwd:
+            break
+        else:
+            passwd = verify = ''
+            print "Password mismatch, please try again..."
+    return user, passwd
+
+def write_inituser(fn, user, password):
+    import binascii
+    try:
+        from hashlib import sha1 as sha
+    except:
+        from sha import new as sha
+    fp = open(fn, "w")
+    pw = binascii.b2a_base64(sha(password).digest())[:-1]
+    fp.write('%s:{SHA}%s\n' % (user, pw))
+    fp.close()
+    os.chmod(fn, 0644)
+
+if __name__ == "__main__":
+    main()

src/Zope2/utilities/reindex_catalog.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+""" 
+A utility to perform external reindex operations for ZCatalogs
+
+Usage:
+
+   zopectl run reindex_catalog.py [options]
+
+   Use --help to get a list of all options.
+
+Author: Andreas Jung (andreas@andreas-jung.com)
+
+$Id$
+"""
+
+import sys
+from optparse import OptionParser
+
+import transaction
+from Products.ZCatalog.ProgressHandler import StdoutHandler
+
+
+def path2catalog(path):
+    """ lookup catalog by path """
+
+    catalog = app.restrictedTraverse(path, None)
+    if not catalog:
+        raise ValueError('No catalog found at %s' % path)
+    return catalog
+
+
+def getHandler(options):
+    """ return a progress handler """
+    
+    if options.silent:
+        return None
+    else:
+        return StdoutHandler(options.steps)
+
+
+def listIndexes(options, args):
+    """ print a list of all indexes to stdout """
+
+    catalog = path2catalog(options.catalog)
+    indexes = catalog._catalog.indexes    
+
+    print 'Listing of all indexes at %s' % options.catalog
+    print
+
+    for id, idx in indexes.items():
+        print '%-20s %-50s %d' % (id, idx.meta_type, idx.numObjects())
+
+
+def reindexIndexes(optioons, args):
+    """ reindex given list of indexes """
+
+    catalog = path2catalog(options.catalog)
+    handler = getHandler(options)
+
+    for id in args:
+        print 'Reindexing index %s at %s' % (id, options.catalog)
+        catalog.reindexIndex(id, None, handler)
+    transaction.commit()
+
+
+def refreshMetadata(options, args):
+    """ reindex metadata """
+
+    catalog = path2catalog(options.catalog)
+    handler = getHandler(options)
+
+    print 'Refresh metadata at %s' % options.catalog
+    catalog.refreshMetadata(handler)
+    transaction.commit()
+
+
+def reindexAll(options, args):
+    """ reindex complete catalog """
+
+    catalog = path2catalog(options.catalog)
+    handler = getHandler(options)
+
+    print 'Reindexing complete ZCatalog at %s' % options.catalog
+    catalog.refreshCatalog(options.clearCatalog, handler)
+    transaction.commit()
+
+
+if __name__ == '__main__':
+   
+    parser = OptionParser()
+    parser.add_option('-c', '--catalog', dest='catalog',
+                     help='path to ZCatalog')
+    parser.add_option('-i', '--indexes', action="store_true", dest="listIndexes",
+                     help='list all indexes')
+    parser.add_option('-C', '--clear', action="store_true", dest="clearCatalog", default=False,
+                     help='clear catalog before reindexing the complete catalog (--all only)')
+    parser.add_option('-a', '--all', action="store_true", dest="reindexAll",
+                     help='reindex the complete catalog and update all metadata')
+    parser.add_option('-r', '--reindex', action="store_true", dest="reindexIndexes",
+                     help='reindex specified indexes')
+    parser.add_option('-m', '--metadata', action="store_true", dest="refreshMetadata",
+                     help='refresh all metadata')
+    parser.add_option('-n', '--steps', dest="steps", default=100, type="int",
+                     help='log progress every N objects')
+    parser.add_option('-s', '--silent', action="store_true", dest="silent", default=False,
+                     help='do not log reindexing progress to stdout')
+
+    options,args = parser.parse_args()
+
+    if options.listIndexes: listIndexes(options, args)
+    if options.reindexIndexes: reindexIndexes(options, args)
+    if options.refreshMetadata: refreshMetadata(options, args)
+    if options.reindexAll: reindexAll(options, args)

src/Zope2/utilities/requestprofiler.py

+#!/usr/bin/env python2.4
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Request log profiler script
+
+$Id$
+"""
+import sys, time, getopt, math, cPickle
+from types import StringType
+try: import gzip
+except: pass
+
+class ProfileException(Exception): pass
+
+class Request:
+    def __init__(self):
+        self.url = None
+        self.start = None
+        self.method = None
+        self.t_recdinput = None
+        self.isize = None
+        self.t_recdoutput = None
+        self.osize = None
+        self.httpcode = None
+        self.t_end = None
+        self.elapsed = "I"
+        self.active = 0
+
+    def put(self, code, t, desc):
+        if code not in ('A', 'B', 'I', 'E'):
+            raise "unknown request code %s" % code
+        if code == 'B':
+            self.start = t
+            self.method, self.url = desc.strip().split()
+        elif code == "I":
+            self.t_recdinput = t
+            self.isize = desc.strip()
+        elif code == "A":
+            self.t_recdoutput = t
+            self.httpcode, self.osize = desc.strip().split()
+        elif code == 'E':
+            self.t_end = t
+            self.elapsed = int(self.t_end - self.start)
+
+    def isfinished(self):
+        return not self.elapsed == "I"
+
+    def prettystart(self):
+        if self.start is not None:
+            t = time.localtime(self.start)
+            return time.strftime('%Y-%m-%dT%H:%M:%S', t)
+        else:
+            return "NA"
+
+    def shortprettystart(self):
+        if self.start is not None:
+            t = time.localtime(self.start)
+            return time.strftime('%H:%M:%S', t)
+        else:
+            return "NA"
+
+    def win(self):
+        if self.t_recdinput is not None and self.start is not None:
+            return self.t_recdinput - self.start
+        else:
+            return "NA"
+
+    def wout(self):
+        if self.t_recdoutput is not None and self.t_recdinput is not None:
+            return self.t_recdoutput - self.t_recdinput
+        else:
+            return "NA"
+
+    def wend(self):
+        if self.t_end is not None and self.t_recdoutput is not None:
+            return self.t_end - self.t_recdoutput
+        else:
+            return "NA"
+
+    def endstage(self):
+        if self.t_end is not None:
+            stage = "E"
+        elif self.t_recdoutput is not None:
+            stage = "A"
+        elif self.t_recdinput is not None:
+            stage = "I"
+        else:
+            stage = "B"
+        return stage
+
+    def total(self):
+        stage = self.endstage()
+        if stage == "B": return 0
+        if stage == "I": return self.t_recdinput - self.start
+        if stage == "A": return self.t_recdoutput - self.start
+        if stage == "E": return self.elapsed
+
+    def prettyisize(self):
+        if self.isize is not None:
+            return self.isize
+        else:
+            return "NA"
+
+    def prettyosize(self):
+        if self.osize is not None:
+            return self.osize
+        else:
+            return "NA"
+
+    def prettyhttpcode(self):
+        if self.httpcode is not None:
+            return self.httpcode
+        else:
+            return "NA"
+
+    def __str__(self):
+        body = (
+            self.prettystart(), self.win(), self.wout(), self.wend(),
+            self.total(), self.endstage(), self.prettyosize(),
+            self.prettyhttpcode(), self.active, self.url
+            )
+        return self.fmt % body
+
+    fmt = "%19s %4s %4s %4s %3s %1s %7s %4s %4s %s"
+
+    def getheader(self):
+        body = ('Start', 'WIn', 'WOut', 'WEnd', 'Tot', 'S', 'OSize',
+                'Code', 'Act', 'URL')
+        return self.fmt % body
+
+class StartupRequest(Request):
+    def endstage(self):
+        return "U"
+
+    def total(self):
+        return 0
+
+class Cumulative:
+    def __init__(self, url):
+        self.url = url
+        self.times = []
+        self.hangs = 0
+        self.allelapsed = None
+
+    def put(self, request):
+        elapsed = request.elapsed
+        if elapsed == "I": self.hangs = self.hangs + 1
+        self.times.append(elapsed)
+
+    def all(self):
+        if self.allelapsed == None:
+            self.allelapsed = []
+            for elapsed in self.times:
+                self.allelapsed.append(elapsed)
+            self.allelapsed.sort()
+        return self.allelapsed
+
+    def __str__(self):
+        body = (
+            self.hangs, self.hits(), self.total(), self.max(), self.min(),
+            self.median(), self.mean(), self.url
+            )
+        return self.fmt % body
+
+    def getheader(self):
+        return self.fmt % ('Hangs', 'Hits', 'Total', 'Max', 'Min', 'Median',
+                           'Mean', 'URL')
+
+    fmt = "%5s %5s %5s %5s %5s %6s %5s %s"
+
+    def hits(self):
+        return len(self.times)
+
+    def max(self):
+        return max(self.all())
+
+    def min(self):
+        return min(self.all())
+
+    def mean(self):
+        l = len(self.times)
+        if l == 0:
+            return "I"
+        else:
+            t = self.total()
+            if t == "I": return "I"
+            return t/l
+
+    def median(self):
+        all = self.all()
+        l = len(all)
+        if l == 0:
+            return "I"
+        else:
+            if l == 1:
+                return all[0]
+            elif l % 2 != 0:
+                i = l/2 + 1
+                return all[i]
+            else:
+                i = l/2 - 1
+                i2 = i + 1
+                v1 = all[i]
+                v2 = all[i2]
+                if isinstance(v1, StringType) or isinstance(v2, StringType):
+                    return "I"
+                else: return (v1 + v2) / 2
+
+    def total(self):
+        t = 0
+        all = self.all()
+        for elapsed in all:
+            if elapsed == "I": continue
+            t = t + elapsed
+        return t
+
+def parsebigmlogline(line):
+    tup = line.split(None, 3)
+    if len(tup) == 3:
+        code, id, timestr = tup
+        return code, id, timestr, ''
+    elif len(tup) == 4:
+        return tup
+    else:
+        return None
+
+def get_earliest_file_data(files):
+    temp = {}
+    earliest_fromepoch = 0
+    earliest = None
+    retn = None
+    for file in files:
+        line = file.readline()
+        if not line:
+            continue
+        linelen = len(line)
+        line = line.strip()
+        tup = parsebigmlogline(line)
+        if tup is None:
+            print "Could not interpret line: %s" % line
+            continue
+        code, id, timestr, desc = tup
+        timestr = timestr.strip()
+        fromepoch = getdate(timestr)
+        temp[file] = linelen
+        if earliest_fromepoch == 0 or fromepoch < earliest_fromepoch:
+            earliest_fromepoch = fromepoch
+            earliest = file
+            retn = [code, id, fromepoch, desc]
+
+    for file, linelen in temp.items():
+        if file is not earliest:
+            file.seek(file.tell() - linelen)
+
+    return retn
+
+def get_requests(files, start=None, end=None, statsfname=None,
+                 writestats=None, readstats=None):
+    finished = []
+    unfinished = {}
+    if readstats:
+        fp = open(statsfname, 'r')
+        u = cPickle.Unpickler(fp)
+        requests = u.load()
+        fp.close()
+        del u
+        del fp
+    else:
+        while 1:
+            tup = get_earliest_file_data(files)
+            if tup is None:
+                break
+            code, id, fromepoch, desc = tup
+            if start is not None and fromepoch < start: continue
+            if end is not None and fromepoch > end: break
+            if code == 'U':
+                finished.extend(unfinished.values())
+                unfinished.clear()
+                request = StartupRequest()
+                request.url = desc
+                request.start = int(fromepoch)
+                finished.append(request)
+                continue
+            request = unfinished.get(id)
+            if request is None:
+                if code != "B": continue # garbage at beginning of file
+                request = Request()
+                for pending_req in unfinished.values():
+                    pending_req.active = pending_req.active + 1
+                unfinished[id] = request
+            t = int(fromepoch)
+            try:
+                request.put(code, t, desc)
+            except:
+                print "Unable to handle entry: %s %s %s"%(code, t, desc)
+            if request.isfinished():
+                del unfinished[id]
+                finished.append(request)
+
+        finished.extend(unfinished.values())
+        requests = finished
+
+        if writestats:
+            fp = open(statsfname, 'w')
+            p = cPickle.Pickler(fp)
+            p.dump(requests)
+            fp.close()
+            del p
+            del fp
+
+    return requests
+
+def analyze(requests, top, sortf, start=None, end=None, mode='cumulative',
+            resolution=60, urlfocusurl=None, urlfocustime=60):
+
+    if mode == 'cumulative':
+        cumulative = {}
+        for request in requests:
+            url = request.url
+            stats = cumulative.get(url)
+            if stats is None:
+                stats = Cumulative(url)
+                cumulative[url] = stats
+            stats.put(request)
+        requests = cumulative.values()
+        requests.sort(sortf)
+        write(requests, top)
+
+    elif mode=='timed':
+        computed_start = requests[0].start
+        computed_end = requests[-1].t_end
+        if start and end:
+            timewrite(requests,start,end,resolution)
+        if start and not end:
+            timewrite(requests,start,computed_end,resolution)
+        if end and not start:
+            timewrite(requests,computed_start,end,resolution)
+        if not end and not start:
+            timewrite(requests,computed_start,computed_end,resolution)
+
+    elif mode == 'urlfocus':
+        requests.sort(sortf)
+        urlfocuswrite(requests, urlfocusurl, urlfocustime)
+
+    else:
+        requests.sort(sortf)
+        write(requests, top)
+
+def urlfocuswrite(requests, url, t):
+    l = []
+    i = 0
+    for request in requests:
+        if request.url == url: l.append(i)
+        i = i + 1
+    before = {}
+    after = {}
+    x = 0
+    for n in l:
+        x = x + 1
+        r = requests[n]
+        start = r.start
+        earliest = start - t
+        latest = start + t
+        print 'URLs invoked %s seconds before and after %s (#%s, %s)' % \
+              (t, url, x, r.shortprettystart())
+        print '---'
+        i = -1
+        for request in requests:
+            i = i + 1
+            if request.start < earliest: continue
+            if request.start > latest: break
+            if n == i: # current request
+                print '%3d' % (request.start - start),
+                print '%s' % (request.shortprettystart()),
+                print request.url
+                continue
+            if request.start <= start:
+                if before.get(i):
+                    before[i] = before[i] + 1
+                else:
+                    before[i] = 1
+            if request.start > start:
+                if after.get(i):
+                    after[i] = after[i] + 1
+                else:
+                    after[i] = 1
+            print '%3d' % (request.start - start),
+            print '%s' % (request.shortprettystart()),
+            print request.url
+        print
+    print ('Summary of URLs invoked before (and at the same time as) %s '
+           '(times, url)' % url)
+    before = before.items()
+    before.sort()
+    for k,v in before:
+        print v, requests[k].url
+    print
+    print 'Summary of URLs invoked after %s (times, url)' % url
+    after = after.items()
+    after.sort()
+    for k,v in after:
+        print v, requests[k].url
+
+def write(requests, top=0):
+    if len(requests) == 0:
+        print "No data.\n"
+        return
+    i = 0
+    header = requests[0].getheader()
+    print header
+    for stat in requests:
+        i = i + 1
+        if verbose:
+            print str(stat)
+        else:
+            print str(stat)[:78]
+        if i == top:
+            break
+
+def getdate(val):
+    try:
+        val = val.strip()
+        year, month, day = int(val[:4]), int(val[5:7]), int(val[8:10])
+        hour,minute,second=int(val[11:13]),int(val[14:16]),int(val[17:19])
+        t = time.mktime((year, month, day, hour, minute, second, 0, 0, -1))
+        return t
+    except:
+        raise ProfileException, "bad date %s" % val
+
+def getTimeslice(period, utime):
+    low = int(math.floor(utime)) - period + 1
+    high = int(math.ceil(utime)) + 1
+    for x in range(low, high):
+        if x % period == 0:
+            return x
+
+def timewrite(requests, start, end, resolution):
+    print "Start: %s    End: %s   Resolution: %d secs" % \
+        (tick2str(start), tick2str(end), resolution)
+    print "-" * 78
+    print
+    print "Date/Time                #requests requests/second"
+
+    d = {}
+    max = 0
+    min = None
+    for r in requests:
+        t = r.start
+        slice = getTimeslice(resolution,t)
+        if slice > max: max = slice
+        if (min is None) or (slice < min): min = slice
+        if d.has_key(slice):
+            d[slice] = d[slice] + 1
+        else:
+            d[slice] = 1
+
+    num = 0
+    hits = 0
+    avg_requests = None
+    max_requests = 0
+    for slice in range(min, max, resolution):
+        num = d.get(slice, 0)
+        if num>max_requests: max_requests = num
+        hits = hits + num
+
+        if avg_requests is None:
+            avg_requests = num
+        else:
+            avg_requests = (avg_requests + num) / 2
+
+        s = tick2str(slice)
+        s = s + "     %6d         %4.2lf" % (num,num*1.0/resolution)
+        print s
+
+    print '='*78
+    print " Peak:                  %6d         %4.2lf" % \
+        (max_requests,max_requests*1.0/resolution)
+    print "  Avg:                  %6d         %4.2lf" % \
+        (avg_requests,avg_requests*1.0/resolution)
+    print "Total:                  %6d          n/a " % (hits)
+
+def tick2str(t):
+    return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t))
+
+def codesort(v1, v2):
+    v1 = v1.endstage()
+    v2 = v2.endstage()
+    if v1 == v2:
+        return 0
+
+    if v1 == "B":
+        return -1 # v1 is smaller than v2
+    if v1 == "I":
+        if v2 == "B": return 1 # v1 is larger than v2
+        else: return -1
+    if v1 == "A":
+        if v2 in ['B', 'I']: return 1
+        else: return -1
+    if v1 == "E":
+        return 1
+
+class Sort:
+    def __init__(self, fname, ascending=0):
+        self.fname = fname
+        self.ascending = ascending
+
+    def __call__(self, i1, i2):
+        f1 = getattr(i1, self.fname)
+        f2 = getattr(i2, self.fname)
+        if callable(f1): f1 = f1()
+        if callable(f2): f2 = f2()
+        if f1 < f2:
+            if self.ascending: return -1
+            else: return 1
+        elif f1 == f2:
+            return 0
+        else:
+            if self.ascending: return 1
+            else: return -1
+
+def detailedusage():
+    details = usage(0)
+    pname = sys.argv[0]
+    details = details + """
+Reports are of four types: cumulative, detailed, timed, or urlfocus.  The
+default is cumulative. Data is taken from one or more Zope detailed request
+logs (-M logs, aka 'big M' logs) or from a preprocessed statistics file.
+
+For cumulative reports, each line in the profile indicates information
+about a Zope method (URL) collected via a detailed request log.
+
+For detailed reports, each line in the profile indicates information about
+a single request.
+
+For timed reports, each line in the profile indicates informations about
+the number of requests and the number of requests/second for a period of time.
+
+For urlfocus reports, ad-hoc information about requests surrounding the
+specified url is given.
+
+Each 'filename' is a path to a '-M' log that contains detailed request data.
+Multiple input files can be analyzed at the same time by providing the path
+to each file.  (Analyzing  multiple big M log files at once is useful if you
+have more than one Zope client on a single machine and you'd like to
+get an overview of all Zope activity on that machine).
+
+If you wish to make multiple analysis runs against the same input data, you
+may want to use the --writestats option.  The --writestats option creates a
+file which holds preprocessed data representing the specfified input files.
+Subsequent runs (for example with a different sort spec) will be much
+faster if the  --readstats option is used to specify a preprocessed stats
+file instead of actual input files because the logfile parse step is skipped.
+
+If a 'sort' value is specified, sort the profile info by the spec.  The sort
+order is descending unless indicated.    The default cumulative sort spec is
+'total'.  The default detailed sort spec is 'start'.
+
+For cumulative reports, the following sort specs are accepted:
+
+  'hits'        -- the number of hits against the method
+  'hangs'       -- the number of unfinished requests to the method
+  'max'         -- the maximum time in secs taken by a request to this method
+  'min'         -- the minimum time in secs taken by a request to this method
+  'mean'        -- the mean time in secs taken by a request to this method
+  'median'      -- the median time in secs taken by a request to this method
+  'total'       -- the total time in secs across all requests to this method
+  'url'         -- the URL/method name (ascending)
+
+For detailed (non-cumulative) reports, the following sort specs are accepted:
+
+  'start'       -- the start time of the request to ZServer (ascending)
+  'win'         -- the num of secs ZServer spent waiting for input from client
+  'wout'        -- the secs ZServer spent waiting for output from ZPublisher
+  'wend'        -- the secs ZServer spent sending data to the client
+  'total'       -- the secs taken for the request from begin to end
+  'endstage'    -- the last successfully completed request stage (B, I, A, E)
+  'osize'       -- the size in bytes of output provided by ZPublisher
+  'httpcode'    -- the HTTP response code provided by ZPublisher (ascending)
+  'active'      -- total num of requests pending at the end of this request
+  'url'         -- the URL/method name (ascending)
+
+For timed and urlfocus reports, there are no sort specs allowed.
+
+If the 'top' argument is specified, only report on the top 'n' entries in
+the profile (as per the sort). The default is to show all data in the profile.
+
+If the 'verbose' argument is specified, do not trim url to fit into 80 cols.
+
+If the 'today' argument is specified, limit results to hits received today.
+
+If the 'daysago' argument is specified, limit results to hits received n days ago.
+
+The 'resolution' argument is used only for timed reports and specifies the
+number of seconds between consecutive lines in the report
+(default is 60 seconds).
+
+The 'urlfocustime' argument is used only for urlfocus reports and specifies the
+number of seconds to target before and after the URL provided in urlfocus mode.
+(default is 10 seconds).
+
+If the 'start' argument is specified in the form 'DD/MM/YYYY HH:MM:SS' (UTC),
+limit results to hits received after this date/time.
+
+If the 'end' argument is specified in the form 'DD/MM/YYYY HH:MM:SS' (UTC),
+limit results to hits received before this date/time.
+
+'start' and 'end' arguments are not honored when request stats are obtained
+via the --readstats argument.
+
+Examples:
+
+  %(pname)s debug.log
+
+    Show cumulative report statistics for information in the file 'debug.log',
+    by default sorted by 'total'.
+
+  %(pname)s debug.log --detailed
+
+    Show detailed report statistics sorted by 'start' (by default).
+
+  %(pname)s debug.log debug2.log --detailed
+
+    Show detailed report statistics for both logs sorted by 'start'
+    (by default).
+
+  %(pname)s debug.log --cumulative --sort=mean --today --verbose
+
+    Show cumulative report statistics sorted by mean for entries in the log
+    which happened today, and do not trim the URL in the resulting report.
+
+  %(pname)s debug.log --cumulative --sort=mean --daysago=3 --verbose
+
+    Show cumulative report statistics sorted by mean for entries in the log
+    which happened three days ago, and do not trim the URL in the resulting report.
+
+  %(pname)s debug.log --urlfocus='/manage_main' --urlfocustime=60
+
+    Show 'urlfocus' report which displays statistics about requests
+    surrounding the invocation of '/manage_main'.  Focus on the time periods
+    60 seconds before and after each invocation of the '/manage_main' URL.
+
+  %(pname)s debug.log --detailed --start='2001/05/10 06:00:00'
+    --end='2001/05/11 23:00:00'
+
+    Show detailed report statistics for entries in 'debug.log' which
+    begin after 6am UTC on May 10, 2001 and which end before
+    11pm UTC on May 11, 2001.
+
+  %(pname)s debug.log --timed --resolution=300 --start='2001/05/10 06:00:00'
+    --end='2001/05/11 23:00:00'
+
+    Show timed report statistics for entries in the log for one day
+    with a resolution of 5 minutes
+
+  %(pname)s debug.log --top=100 --sort=max
+
+    Show cumulative report of the the 'top' 100 methods sorted by maximum
+    elapsed time.
+
+  %(pname)s debug.log debug2.log --writestats='requests.stat'
+
+    Write stats file for debug.log and debug2.log into 'requests.stat' and
+    show default report.
+
+  %(pname)s --readstats='requests.stat' --detailed
+
+    Read from 'requests.stat' stats file (instead of actual -M log files)
+    and show detailed report against this data.""" % {'pname':pname}
+    return details
+
+def usage(basic=1):
+    usage = (
+        """
+Usage: %s filename1 [filename2 ...]
+          [--cumulative | --detailed | [--timed --resolution=seconds]]
+          [--sort=spec]
+          [--top=n]
+          [--verbose]
+          [--today | [--start=date] [--end=date] | --daysago=n ]
+          [--writestats=filename | --readstats=filename]
+          [--urlfocus=url]
+          [--urlfocustime=seconds]
+          [--help]
+
+Provides a profile of one or more Zope "-M" request log files.
+""" % sys.argv[0]
+        )
+    if basic == 1:
+        usage = usage + """
+If the --help argument is given, detailed usage docs are provided."""
+    return usage
+
+
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        print usage()
+        sys.exit(0)
+    if sys.argv[1] == '--help': print detailedusage(); sys.exit(0)
+    mode = 'cumulative'
+    sortby = None
+    trim = 0
+    top = 0
+    verbose = 0
+    start = None
+    end = None
+    resolution=60
+    urlfocustime=10
+    urlfocusurl=None
+    statsfname = None
+    readstats = 0
+    writestats = 0
+
+    files = []
+    i = 1
+    for arg in sys.argv[1:]:
+        if arg[:2] != '--':
+            if arg[-3:] == '.gz' and globals().has_key('gzip'):
+                files.append(gzip.GzipFile(arg,'r'))
+            else:
+                files.append(open(arg))
+            sys.argv.remove(arg)
+            i = i + 1
+
+    try:
+        opts, extra = getopt.getopt(
+            sys.argv[1:], '', ['sort=', 'top=', 'help', 'verbose', 'today',
+                               'cumulative', 'detailed', 'timed','start=',
+                               'end=','resolution=', 'writestats=','daysago=',
+                               'readstats=','urlfocus=','urlfocustime=']
+            )
+        for opt, val in opts:
+
+            if opt=='--readstats':
+                statsfname = val
+                readstats = 1
+            elif opt=='--writestats':
+                statsfname = val
+                writestats = 1
+            if opt=='--sort': sortby = val
+            if opt=='--top': top=int(val)
+            if opt=='--help': print detailedusage(); sys.exit(0)
+            if opt=='--verbose':
+                verbose = 1
+            if opt=='--resolution':
+                resolution=int(val)
+            if opt=='--today':
+                now = time.localtime(time.time())
+                # for testing - now = (2001, 04, 19, 0, 0, 0, 0, 0, -1)
+                start = list(now)
+                start[3] = start[4] = start[5] = 0
+                start = time.mktime(start)
+                end = list(now)
+                end[3] = 23; end[4] = 59; end[5] = 59
+                end = time.mktime(end)
+            if opt=='--daysago':
+                now = time.localtime(time.time() - int(val)*3600*24 )
+                # for testing - now = (2001, 04, 19, 0, 0, 0, 0, 0, -1)
+                start = list(now)
+                start[3] = start[4] = start[5] = 0
+                start = time.mktime(start)
+                end = list(now)
+                end[3] = 23; end[4] = 59; end[5] = 59
+                end = time.mktime(end)
+            if opt=='--start':
+                start = getdate(val)
+            if opt=='--end':
+                end = getdate(val)
+            if opt=='--detailed':
+                mode='detailed'
+                d_sortby = sortby
+            if opt=='--cumulative':
+                mode='cumulative'
+            if opt=='--timed':
+                mode='timed'
+            if opt=='--urlfocus':
+                mode='urlfocus'
+                urlfocusurl = val
+            if opt=='--urlfocustime':
+                urlfocustime=int(val)
+
+        validcumsorts = ['url', 'hits', 'hangs', 'max', 'min', 'median',
+                         'mean', 'total']
+        validdetsorts = ['start', 'win', 'wout', 'wend', 'total',
+                         'endstage', 'isize', 'osize', 'httpcode',
+                         'active', 'url']
+
+        if mode == 'cumulative':
+            if sortby is None: sortby = 'total'
+            assert sortby in validcumsorts, (sortby, mode, validcumsorts)
+            if sortby in ['url']:
+                sortf = Sort(sortby, ascending=1)
+            else:
+                sortf = Sort(sortby)
+        elif mode == 'detailed':
+            if sortby is None: sortby = 'start'
+            assert sortby in validdetsorts, (sortby, mode, validdetsorts)
+            if sortby in ['start', 'url', 'httpcode']:
+                sortf = Sort(sortby, ascending=1)
+            elif sortby == 'endstage':
+                sortf = codesort
+            else:
+                sortf = Sort(sortby)
+        elif mode=='timed':
+            sortf = None
+        elif mode=='urlfocus':
+            sortf = Sort('start', ascending=1)
+        else:
+            raise 'Invalid mode'
+
+        req=get_requests(files, start, end, statsfname, writestats, readstats)
+        analyze(req, top, sortf, start, end, mode, resolution, urlfocusurl,
+                urlfocustime)
+
+    except AssertionError, val:
+        a = "%s is not a valid %s sort spec, use one of %s"
+        print a % (val[0], val[1], val[2])
+        sys.exit(0)
+    except getopt.error, val:
+        print val
+        sys.exit(0)
+    except ProfileException, val:
+        print val
+        sys.exit(0)
+    except SystemExit:
+        sys.exit(0)
+    except:
+        import traceback
+        traceback.print_exc()
+        print usage()
+        sys.exit(0)

skel/bin/runzope.in → src/Zope2/utilities/skel/bin/runzope.in

@@ -8,6 +8,7 @@ SOFTWARE_HOME="<<SOFTWARE_HOME>>"
 PYTHONPATH="$SOFTWARE_HOME:$PYTHONPATH"
 export PYTHONPATH INSTANCE_HOME SOFTWARE_HOME
 
-ZOPE_RUN="$SOFTWARE_HOME/Zope2/Startup/run.py"
+ZOPE2PATH=`${PYTHON} -c "import os, Zope2; print os.path.dirname(Zope2.__file__)" `
+ZOPE_RUN="$ZOPE2PATH/Startup/run.py"
 
 exec "$PYTHON" "$ZOPE_RUN" -C "$CONFIG_FILE" "$@"

skel/bin/zopectl.in → src/Zope2/utilities/skel/bin/zopectl.in

@@ -8,6 +8,7 @@ SOFTWARE_HOME="<<SOFTWARE_HOME>>"
 PYTHONPATH="$SOFTWARE_HOME:$PYTHONPATH"
 export PYTHONPATH INSTANCE_HOME SOFTWARE_HOME
 
-ZDCTL="$SOFTWARE_HOME/Zope2/Startup/zopectl.py"
+ZOPE2PATH=`${PYTHON} -c "import os, Zope2; print os.path.dirname(Zope2.__file__)" `
+ZDCTL="$ZOPE2PATH/Startup/zopectl.py"
 
 exec "$PYTHON" "$ZDCTL" -C "$CONFIG_FILE" "$@"

src/Zope2/utilities/tracelog.py

+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Yet another lag analysis tool
+
+
+
+$Id$
+"""
+
+import datetime, optparse, sys
+
+class Request:
+
+    output_bytes = '-'
+
+    def __init__(self, start, method, url):
+        self.method = method
+        self.url = url
+        self.start = start
+        self.state = 'input'
+
+    def I(self, input_time, input_bytes):
+        self.input_time = input_time
+        self.input_bytes = input_bytes
+        self.state = 'app'
+
+    def A(self, app_time, response, output_bytes):
+        self.app_time = app_time
+        self.response = response
+        self.output_bytes = output_bytes
+        self.state = 'output'
+
+    def E(self, end):
+        self.end = end
+
+    @property
+    def app_seconds(self):
+        return (self.app_time - self.input_time).seconds
+
+    @property
+    def total_seconds(self):
+        return (self.end - self.start).seconds
+
+class Times:
+
+    tid = 1l
+
+    def __init__(self):
+        self.times = []
+        self.hangs = 0
+        Times.tid += 1
+        self.tid = Times.tid # generate a unique id
+
+    def finished(self, request):
+        self.times.append(request.app_seconds)
+
+    def hung(self):
+        self.hangs += 1
+
+    def impact(self):
+        times = self.times
+        if not times:
+            self.median = self.mean = self.impact = 0
+            return 0
+        self.times.sort()
+        n = len(times) 
+        if n % 2:
+            m = times[(n+1)/2-1]
+        else:
+            m = .5 * (times[n/2]+times[n/2-1])
+        self.median = m
+        self.mean = sum(times)/n
+        self.impact = self.mean * (n+self.hangs)
+        return self.impact
+
+    def __str__(self):
+        times = self.times
+        if not times:
+            return "              0                             %5d" % (
+                self.hangs)
+            
+        n = len(times)
+        m = self.median
+        return "%9.1f %5d %6.0f %6.2f %6.2f %6.0f %5d" % (
+            self.impact, n, times[0], m, self.mean, times[-1], self.hangs)
+
+    def html(self):
+        times = self.times
+        if not times:
+            print td('', 0, '', '', '', '', self.hangs)
+        else:
+            n = len(times)
+            m = self.median
+            impact = '<a name="u%s">%s' % (self.tid, self.impact)
+            print td(impact, n, times[0], m, self.mean, times[-1],
+                     self.hangs)
+
+def parsedt(s):
+    date, time = s.split('T')
+    return datetime.datetime(*(
+        map(int, date.split('-')) 
+        +
+        map(int, time.split(':'))         
+        ))
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    options, args = parser.parse_args(args)
+    if options.event_log:
+        restarts = find_restarts(options.event_log)
+    else:
+        restarts = []
+
+    restarts.append(datetime.datetime.utcnow()+datetime.timedelta(1000))
+
+    if options.html:
+        print_app_requests = print_app_requests_html
+        output_minute = output_minute_html
+        output_stats = output_stats_html
+        minutes_header = minutes_header_html
+        minutes_footer = minutes_footer_html
+        print '<html title="trace log statistics"><body>'
+    else:
+        print_app_requests = print_app_requests_text
+        output_minute = output_minute_text
+        output_stats = output_stats_text
+        minutes_header = minutes_header_text
+        minutes_footer = minutes_footer_text
+        
+        
+    urls = {}
+    [file] = args
+    lmin = ldt = None
+    requests = {}
+    input = apps = output = n = 0
+    spr = spa = 0.0
+    restart = restarts.pop(0)
+    minutes_header()
+    remove_prefix = options.remove_prefix
+    for record in open(file):
+        record = record.split()
+        typ, rid, dt = record[:3]
+        min = dt[:-3]
+        dt = parsedt(dt)
+        if dt == restart:
+            continue
+        while dt > restart:
+            print_app_requests(requests, ldt,
+                               options.old_requests,
+                               options.app_requests,
+                               urls,
+                               "\nLeft over:")
+            record_hung(urls, requests)
+            requests = {}
+            input = apps = output = n = 0
+            spr = spa = 0.0
+            restart = restarts.pop(0)
+        ldt = dt
+        
+        if min != lmin:
+            if lmin is not None:
+                output_minute(lmin, requests, input, apps, output, n, spr, spa)
+                if apps > options.apps:
+                    print_app_requests(requests, dt,
+                                       options.old_requests,
+                                       options.app_requests,
+                                       urls,
+                                       )
+            lmin = min
+            spr = 0.0
+            spa = 0.0
+            n = 0
+            
+        if typ == 'B':
+            if rid in requests:
+                request = requests[rid]
+                if request.state == 'output':
+                    output -= 1
+                elif request.state == 'app':
+                    apps -= 1
+                else:
+                    input -= 1
+            
+            input += 1
+            request = Request(dt, *record[3:5])
+            if remove_prefix and request.url.startswith(remove_prefix):
+                request.url = request.url[len(remove_prefix):]
+            requests[rid] = request
+            times = urls.get(request.url)
+            if times is None:
+                times = urls[request.url] = Times()
+        elif typ == 'I':
+            if rid in requests:
+                input -= 1
+                apps += 1
+                requests[rid].I(dt, record[3])
+        elif typ == 'A':
+            if rid in requests:
+                apps -= 1
+                output += 1
+                requests[rid].A(dt, *record[3:5])
+        elif typ == 'E':
+            if rid in requests:
+                output -= 1
+                request = requests.pop(rid)
+                request.E(dt)
+                spr += request.total_seconds
+                spa += request.app_seconds
+                n += 1
+                times = urls[request.url]
+                times.finished(request)
+        else:
+            print 'WTF', record
+
+    print_app_requests(requests, dt,
+                       options.old_requests,
+                       options.app_requests,
+                       urls,
+                       "Left over:")
+
+    minutes_footer()
+
+    output_stats(urls)
+
+    if options.html:
+        print '</body></html>'
+
+def output_stats_text(urls):
+    print
+    print 'URL statistics:'
+    print "   Impact count    min median   mean    max hangs"
+    print "========= ===== ====== ====== ====== ====== ====="
+    urls = [(times.impact(), url, times)
+            for (url, times) in urls.iteritems()
+            ]
+    urls.sort()
+    urls.reverse()
+    for (_, url, times) in urls:
+        if times.impact > 0 or times.hangs:
+            print times, url
+
+def output_stats_html(urls):
+    print
+    print 'URL statistics:'
+    print '<table border="1">'
+    print '<tr><th>Impact</th><th>count</th><th>min</th>'
+    print     '<th>median</th><th>mean</th><th>max</th><th>hangs</th></tr>'
+    urls = [(times.impact(), url, times)
+            for (url, times) in urls.iteritems()
+            ]
+    urls.sort()
+    urls.reverse()
+    for (_, url, times) in urls:
+        if times.impact > 0 or times.hangs:
+            print '<tr>'
+            times.html()
+            print td(url)
+            print '</tr>'
+    print '</table>'
+
+def minutes_header_text():
+    print
+    print "          minute   req input   app output"
+    print "================ ===== ===== ===== ======"
+
+def minutes_footer_text():
+    print
+
+def minutes_header_html():
+    print '<table border="2">'
+    print "<tr>"
+    print '<th>Minute</th>'
+    print '<th>Requests</th>'
+    print '<th>Requests inputing</th>'
+    print '<th>Requests executing or waiting</th>'
+    print '<th>Requests outputing</th>'
+    print '<th>Requests completed</th>'
+    print '<th>Mean Seconds Per Request Total</th>'
+    print '<th>Mean Seconds Per Request in App</th>'
+    print "</tr>"
+
+def minutes_footer_html():
+    print '</table>'
+
+def output_minute_text(lmin, requests, input, apps, output, n, spr, spa):
+    print lmin.replace('T', ' '), "%5d I=%3d A=%3d O=%5d " % (
+        len(requests), input, apps, output),
+    if n:
+        print "N=%4d %10.2f %10.2f" % (n, spr/n, spa/n)
+    else:
+        print
+
+def td(*values):
+    return ''.join([("<td>%s</td>" % s) for s in values])
+
+def output_minute_html(lmin, requests, input, apps, output, n, spr, spa):
+    print '<tr>'
+    apps = '<font size="+2"><strong>%s</strong></font>' % apps
+    print td(lmin.replace('T', ' '), len(requests), input, apps, output)
+    if n:
+        print td(n, "%10.2f" % (spr/n), "%10.2f" % (spa/n))
+    print '</tr>'
+
+def find_restarts(event_log):
+    result = []
+    for l in open(event_log):
+        if l.strip().endswith("Zope Ready to handle requests"):
+            result.append(parsedt(l.split()[0]))
+    return result
+
+def record_hung(urls, requests):
+    for request in requests.itervalues():
+        times = urls.get(request.url)
+        if times is None:
+            times = urls[request.url] = Times()
+        times.hung()
+
+def print_app_requests_text(requests, dt, min_seconds, max_requests, urls,
+                       label=''):
+    requests = [
+        ((dt-request.input_time).seconds, request)
+        for request in requests.values()
+        if request.state == 'app'
+    ]
+
+    urls = {}
+    for s, request in requests:
+        urls[request.url] = urls.get(request.url, 0) + 1
+    
+    requests.sort()
+    requests.reverse()
+    for s, request in requests[:max_requests]:
+        if s < min_seconds:
+            continue
+        if label:
+            print label
+            label = ''
+        url = request.url
+        repeat = urls[url]
+        if repeat > 1:
+            print s, "R=%d" % repeat, url
+        else:
+            print s, url
+
+def print_app_requests_html(requests, dt, min_seconds, max_requests, allurls,
+                            label=''):
+    requests = [
+        ((dt-request.input_time).seconds, request)
+        for request in requests.values()
+        if request.state == 'app'
+    ]
+
+    urls = {}
+    for s, request in requests:
+        urls[request.url] = urls.get(request.url, 0) + 1
+    
+    requests.sort()
+    requests.reverse()
+    printed = False
+    for s, request in requests[:max_requests]:
+        if s < min_seconds:
+            continue
+        if label:
+            print label
+            label = ''
+        if not printed:
+            minutes_footer_html()
+            print '<table border="1">'
+            print '<tr><th>age</th><th>R</th><th>url</th></tr>'
+            printed = True
+        url = request.url
+        repeat = urls[url]
+        print '<tr>'
+        if repeat <= 1:
+            repeat = ''
+        url = '<a href="#u%s">%s</a>' % (allurls[url].tid, url)
+        print td(s, repeat, url)
+        print '</tr>'
+
+    if printed:
+        print '</table>'
+        minutes_header_html()
+
+parser = optparse.OptionParser("""
+Usage: %prog [options] trace_log_file
+
+Output trace log data showing:
+
+- number of active requests,
+- number of input requests (requests gathering input),
+- number of application requests,
+- number of output requests,
+- number of requests completed in the minute shown,
+- mean seconds per request and
+- mean application seconds per request.
+
+Note that we don't seem to be logging when a connection to the client
+is broken, so the number of active requests, and especially the number
+of outputing requests tends to grow over time. This is spurious.
+
+Also, note that, unfortunately, application requests include requests
+that are running in application threads and requests waiting to get an
+application thread.
+
+When application threads get above the app request threshold, then we
+show the requests that have been waiting the longest.
+
+""")
+
+parser.add_option("--app-request-threshold", "-a", dest='apps',
+                  type="int", default=10,
+                  help="""
+Number of pending application requests at which detailed request information
+if printed.
+""")
+parser.add_option("--app-requests", "-r", dest='app_requests',
+                  type="int", default=10,
+                  help="""
+How many requests to show when the maximum number of pending
+apps is exceeded.
+""")
+parser.add_option("--old-requests", "-o", dest='old_requests',
+                  type="int", default=10,
+                  help="""
+Number of seconds beyond which a request is considered old.
+""")
+parser.add_option("--event-log", "-e", dest='event_log',
+                  help="""
+The name of an event log that goes with the trace log.  This is used
+to determine when the server is restarted, so that the running trace data structures can be reinitialized.
+""")
+parser.add_option("--html", dest='html', action='store_true',
+                  help="""
+Generate HTML output.
+""")
+parser.add_option("--remove-prefix", dest='remove_prefix',
+                  help="""
+A prefex to be removed from URLS.
+""")
+                  
+
+            
+if __name__ == '__main__':
+    main()

src/Zope2/utilities/zpasswd.py

+#!/usr/bin/env python2.4
+
+##############################################################################
+#
+# Copyright (c) 2001,2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Zope user bootstrap system
+
+Usage: %(PROGRAM)s [options] filename
+
+If this program is called without command-line options, it will prompt
+for all necessary information.  The available options are:
+
+    -u / --username=
+    Set the username to be used for the initial user or the emergency user
+
+    -p / --password=
+    Set the password
+
+    -e / --encoding=
+    Set the encryption/encoding rules.  Defaults to SHA-1. OPTIONAL
+
+    -d / --domains=
+    Set the domain names that the user user can log in from.  Defaults to
+    any. OPTIONAL.
+
+    -h / --help
+    Print this help text and exit.
+
+    Filename is required and should be the name of the file to store the
+    information in (usually "inituser" or "access").
+"""
+import sys,  sha, binascii, random, getopt, getpass, os
+
+try:
+    from crypt import crypt
+except ImportError:
+    crypt = None
+
+PROGRAM = sys.argv[0]
+COMMASPACE = ', '
+
+
+def generate_salt():
+    """Generate a salt value for the crypt function."""
+    salt_choices = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                    "abcdefghijklmnopqrstuvwxyz"
+                    "0123456789./")
+    return random.choice(salt_choices)+random.choice(salt_choices)
+
+def generate_passwd(password, encoding):
+    encoding=encoding.upper()
+    if encoding == 'SHA':
+        pw = '{SHA}' + binascii.b2a_base64(sha.new(password).digest())[:-1]
+    elif encoding == 'CRYPT':
+        pw = '{CRYPT}' + crypt(password, generate_salt())
+    elif encoding == 'CLEARTEXT':
+        pw = password
+    else:
+        raise ValueError('Unsupported encoding: %s' % encoding)
+
+    return pw
+
+def write_generated_password(home, ac_path, username):
+    pw_choices = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                  "abcdefghijklmnopqrstuvwxyz"
+                  "0123456789!")
+    acfile=open(ac_path, 'w')
+    pw = ''
+    for i in range(8):
+        pw = pw + random.choice(pw_choices)
+    acfile.write('%s:%s\n' % (username, generate_passwd(pw, 'SHA')))
+    acfile.close()
+    os.chmod(ac_path, 0644)
+    return pw
+
+def write_access(home, user='', group=''):
+    ac_path=os.path.join(home, 'access')
+    if not os.path.exists(ac_path):
+        print '-'*78
+        print 'creating default access file'
+        pw = write_generated_password(home, ac_path, 'emergency')
+        print """Note:
+        The emergency user name and password are 'emergency'
+        and '%s'.
+
+        You can change the emergency name and password with the
+        zpasswd script.  To find out more, type:
+
+        %s zpasswd.py
+        """ % (pw, sys.executable)
+
+        import do; do.ch(ac_path, user, group)
+
+def get_password():
+    while 1:
+        password = getpass.getpass("Password: ")
+        verify = getpass.getpass("Verify password: ")
+        if verify == password:
+            return password
+        else:
+            password = verify = ''
+            print "Password mismatch, please try again..."
+
+def write_inituser(home, user='', group=''):
+    ac_path=os.path.join(home, 'inituser')
+    if not os.path.exists(ac_path):
+        print '-'*78
+        print 'creating default inituser file'
+        pw = write_generated_password(home, ac_path, 'admin')
+        print """Note:
+        The initial user name and password are 'admin'
+        and '%s'.
+
+        You can change the name and password through the web
+        interface or using the 'zpasswd.py' script.
+        """ % pw
+
+        import do; do.ch(ac_path, user, group)
+
+
+def usage(code, msg=''):
+    print >> sys.stderr, __doc__ % globals()
+    if msg:
+        print >> sys.stderr, msg
+    sys.exit(code)
+
+
+def main():
+    shortopts = 'u:p:e:d:h'
+    longopts = ['username=',
+                'password=',
+                'encoding=',
+                'domains=',
+                'help']
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], shortopts, longopts)
+    except getopt.error, msg:
+        usage(1, msg)
+
+    # Defaults
+    username = password = None
+    domains = ''
+    encoding = 'SHA'
+
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+        elif opt in ('-u', '--username'):
+            username = arg
+        elif opt in ('-p', '--password'):
+            password = arg
+        elif opt in ('-e', '--encoding'):
+            encoding = arg
+        elif opt in ('-d', '--domains'):
+            domains = ':' + arg
+
+    # Extra command line arguments?
+    if len(args) == 0:
+        usage(1, 'filename is required')
+    elif len(args) == 1:
+        access_file = open(args[0], 'w')
+    else:
+        usage(1, 'Extra command line arguments: ' + COMMASPACE.join(args))
+
+    if opts:
+        # There were some command line args, so verify
+        if username is not None and password is None:
+            password = get_password()
+    else:
+        # No command line args, so prompt
+        while 1:
+            username = raw_input("Username: ")
+            if username != '':
+                break
+
+        password = get_password()
+        
+        while 1:
+            print """
+Please choose a format from:
+
+SHA - SHA-1 hashed password (default)
+CRYPT - UNIX-style crypt password
+CLEARTEXT - no protection
+"""
+            encoding = raw_input("Encoding: ")
+            if encoding == '':
+                encoding = 'SHA'
+                break
+            if encoding.upper() in ['SHA', 'CRYPT', 'CLEARTEXT']:
+                break
+
+        domains = raw_input("Domain restrictions: ")
+        if domains:
+            domains = ":" + domains
+
+    # Done with prompts and args
+    access_file.write(username + ":" +
+                      generate_passwd(password, encoding) +
+                      domains)
+
+
+# If called from the command line
+if __name__=='__main__':
+    main()