buildout.py

#############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Buildout main script

$Id$
"""

import logging
import md5
import os
import pprint
import re
import shutil
import sys
import tempfile
import ConfigParser

import pkg_resources
import zc.buildout
import zc.buildout.easy_install

pkg_resources_loc = pkg_resources.working_set.find(
    pkg_resources.Requirement.parse('setuptools')).location


class MissingOption(zc.buildout.UserError, KeyError):
    """A required option was missing
    """

class MissingSection(zc.buildout.UserError, KeyError):
    """A required section is missinh
    """

class Options(dict):

    def __init__(self, buildout, section, data):
        self.buildout = buildout
        self.section = section
        super(Options, self).__init__(data)

    def __getitem__(self, option):
        try:
            return super(Options, self).__getitem__(option)
        except KeyError:
            raise MissingOption("Missing option: %s:%s"
                                % (self.section, option))

    # XXX need test
    def __setitem__(self, option, value):
        if not isinstance(value, str):
            raise TypeError('Option values must be strings', value)
        super(Options, self).__setitem__(option, value)

    def copy(self):
        return Options(self.buildout, self.section, self)

class Buildout(dict):

    def __init__(self, config_file, cloptions, windows_restart=False):
        config_file = os.path.abspath(config_file)
        self._config_file = config_file
        self.__windows_restart = windows_restart
        if not os.path.exists(config_file):
            print 'Warning: creating', config_file
            open(config_file, 'w').write('[buildout]\nparts = \n')

        super(Buildout, self).__init__()

        # default options
        data = dict(buildout={
            'directory': os.path.dirname(config_file),
            'eggs-directory': 'eggs',
            'develop-eggs-directory': 'develop-eggs',
            'bin-directory': 'bin',
            'parts-directory': 'parts',
            'installed': '.installed.cfg',
            'python': 'buildout',
            'executable': sys.executable,
            'log-level': 'INFO',
            'log-format': '%(name)s: %(message)s',
            })

        # load user defaults, which override defaults
        if 'HOME' in os.environ:
            user_config = os.path.join(os.environ['HOME'],
                                       '.buildout', 'default.cfg')
            if os.path.exists(user_config):
                _update(data, _open(os.path.dirname(user_config), user_config,
                                    []))

        # load configuration files
        _update(data, _open(os.path.dirname(config_file), config_file, []))

        # apply command-line options
        for (section, option, value) in cloptions:
            options = data.get(section)
            if options is None:
                options = self[section] = {}
            options[option] = value
                # The egg dire

        # do substitutions
        converted = {}
        for section, options in data.iteritems():
            for option, value in options.iteritems():
                if '$' in value:
                    value = self._dosubs(section, option, value,
                                         data, converted, [])
                    options[option] = value
                converted[(section, option)] = value

        # copy data into self:
        for section, options in data.iteritems():
            self[section] = Options(self, section, options)
        
        # initialize some attrs and buildout directories.
        options = self['buildout']

        links = options.get('find-links', '')
        self._links = links and links.split() or ()

        self._buildout_dir = options['directory']
        for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
            d = self._buildout_path(options[name+'-directory'])
            options[name+'-directory'] = d

        options['installed'] = os.path.join(options['directory'],
                                            options['installed'])

        self._setup_logging()

    def _dosubs(self, section, option, value, data, converted, seen):
        key = section, option
        r = converted.get(key)
        if r is not None:
            return r
        if key in seen:
            raise zc.buildout.UserError(
                "Circular reference in substitutions.\n"
                "We're evaluating %s\nand are referencing: %s.\n"
                % (", ".join([":".join(k) for k in seen]),
                   ":".join(key)
                   )
                )
        seen.append(key)
        value = '$$'.join([self._dosubs_esc(s, data, converted, seen)
                           for s in value.split('$$')
                           ])
        seen.pop()
        return value

    _template_split = re.compile('([$]{[^}]*})').split
    _simple = re.compile('[-a-zA-Z0-9 ._]+$').match
    _valid = re.compile('[-a-zA-Z0-9 ._]+:[-a-zA-Z0-9 ._]+$').match
    def _dosubs_esc(self, value, data, converted, seen):
        value = self._template_split(value)
        subs = []
        for ref in value[1::2]:
            s = tuple(ref[2:-1].split(':'))
            if not self._valid(ref):
                if len(s) < 2:
                    raise zc.buildout.UserError("The substitution, %s,\n"
                                                "doesn't contain a colon."
                                                % ref)
                if len(s) > 2:
                    raise zc.buildout.UserError("The substitution, %s,\n"
                                                "has too many colons."
                                                % ref)
                if not self._simple(s[0]):
                    raise zc.buildout.UserError(
                        "The section name in substitution, %s,\n"
                        "has invalid characters."
                        % ref)
                if not self._simple(s[1]):
                    raise zc.buildout.UserError(
                        "The option name in substitution, %s,\n"
                        "has invalid characters."
                        % ref)
                
            v = converted.get(s)
            if v is None:
                options = data.get(s[0])
                if options is None:
                    raise MissingSection(
                        "Referenced section does not exist", s[0])
                v = options.get(s[1])
                if v is None:
                    raise MissingOption("Referenced option does not exist:",
                                        *s)
                if '$' in v:
                    v = self._dosubs(s[0], s[1], v, data, converted, seen)
                    options[s[1]] = v
                converted[s] = v
            subs.append(v)
        subs.append('')

        return ''.join([''.join(v) for v in zip(value[::2], subs)])

    def _buildout_path(self, *names):
        return os.path.join(self._buildout_dir, *names)

    def bootstrap(self, args):
        self._setup_directories()

        # Now copy buildout and setuptools eggs, amd record destination eggs:
        entries = []
        for name in 'setuptools', 'zc.buildout':
            r = pkg_resources.Requirement.parse(name)
            dist = pkg_resources.working_set.find(r)
            if dist.precedence == pkg_resources.DEVELOP_DIST:
                dest = os.path.join(self['buildout']['develop-eggs-directory'],
                                    name+'.egg-link')
                open(dest, 'w').write(dist.location)
                entries.append(dist.location)
            else:
                dest = os.path.join(self['buildout']['eggs-directory'],
                                    os.path.basename(dist.location))
                entries.append(dest)
                if not os.path.exists(dest):
                    if os.path.isdir(dist.location):
                        shutil.copytree(dist.location, dest)
                    else:
                        shutil.copy2(dist.location, dest)

        # Create buildout script
        ws = pkg_resources.WorkingSet(entries)
        ws.require('zc.buildout')
        zc.buildout.easy_install.scripts(
            ['zc.buildout'], ws, sys.executable,
            self['buildout']['bin-directory'])

    def install(self, install_parts):
        self._load_extensions()
        self._setup_directories()

        # Add develop-eggs directory to path so that it gets searched
        # for eggs:
        sys.path.insert(0, self['buildout']['develop-eggs-directory'])

        # Check for updates. This could cause the process to be rstarted
        self._maybe_upgrade()

        # load installed data
        installed_part_options = self._read_installed_part_options()

        # Remove old develop eggs
        self._uninstall(
            installed_part_options['buildout'].get(
                'installed_develop_eggs', '')
            )

        # Build develop eggs
        installed_develop_eggs = self._develop()

        # get configured and installed part lists
        conf_parts = self['buildout']['parts']
        conf_parts = conf_parts and conf_parts.split() or []
        installed_parts = installed_part_options['buildout']['parts']
        installed_parts = installed_parts and installed_parts.split() or []


        # If install_parts is given, then they must be listed in parts
        # and we don't uninstall anything. Otherwise, we install
        # the configured parts and uninstall anything else.
        if install_parts:
            extra = [p for p in install_parts if p not in conf_parts]
            if extra:
                self._error(
                    'Invalid install parts: %s.\n'
                    'Install parts must be listed in the configuration.',
                    ' '.join(extra))
            uninstall_missing = False
        else:
            install_parts = conf_parts
            uninstall_missing = True

        # load recipes
        recipes = self._load_recipes(install_parts)

        # compute new part recipe signatures
        self._compute_part_signatures(install_parts)

        try:
            # uninstall parts that are no-longer used or who's configs
            # have changed
            for part in reversed(installed_parts):
                if part in install_parts:
                    old_options = installed_part_options[part].copy()
                    installed_files = old_options.pop('__buildout_installed__')
                    new_options = self.get(part)
                    if old_options == new_options:
                        # The options are the same, but are all of the
                        # installed files still there?  If not, we should
                        # reinstall.
                        if not installed_files:
                            continue
                        for f in installed_files.split('\n'):
                            if not os.path.exists(self._buildout_path(f)):
                                break
                        else:
                            continue

                    # output debugging info
                    for k in old_options:
                        if k not in new_options:
                            self._logger.debug("Part: %s, dropped option %s",
                                               part, k)
                        elif old_options[k] != new_options[k]:
                            self._logger.debug(
                                "Part: %s, option %s, %r != %r",
                                part, k, new_options[k], old_options[k],
                                )
                    for k in new_options:
                        if k not in old_options:
                            self._logger.debug("Part: %s, new option %s",
                                               part, k)

                elif not uninstall_missing:
                    continue

                # ununstall part
                self._logger.info('Uninstalling %s', part)
                self._uninstall(
                    installed_part_options[part]['__buildout_installed__'])
                installed_parts = [p for p in installed_parts if p != part]

            # install new parts
            for part in install_parts:
                signature = self[part].pop('__buildout_signature__')
                saved_options = self[part].copy()
                if part in installed_parts:
                    self._logger.info('Updating %s', part)
                    old_options = installed_part_options[part]
                    old_installed_files = old_options['__buildout_installed__']
                    try:
                        update = recipes[part].update
                    except AttributeError:
                        update = recipes[part].install
                        self._logger.warning(
                            "The recipe for %s doesn't define an update "
                            "method. Using it's install method",
                            part)
                        
                    try:
                        installed_files = update()
                    except:
                        installed_parts.remove(part)
                        self._uninstall(old_installed_files)
                        raise
                    
                    if installed_files is None:
                        installed_files = old_installed_files.split('\n')

                else:
                    self._logger.info('Installing %s', part)
                    installed_files = recipes[part].install()
                    if installed_files is None:
                        self._logger.warning(
                            "The %s install returned None.  A path or "
                            "iterable os paths should be returned.",
                            part)
                        installed_files = ()
                    
                if isinstance(installed_files, str):
                    installed_files = [installed_files]

                installed_part_options[part] = saved_options
                saved_options['__buildout_installed__'
                              ] = '\n'.join(installed_files)
                saved_options['__buildout_signature__'] = signature

                if part not in installed_parts:
                    installed_parts.append(part)

        finally:
            installed_part_options['buildout']['parts'] = ' '.join(
                [p for p in conf_parts if p in installed_parts]
                +
                [p for p in installed_parts if p not in conf_parts] 
            )
            installed_part_options['buildout']['installed_develop_eggs'
                                               ] = installed_develop_eggs
            
            self._save_installed_options(installed_part_options)

    def _setup_directories(self):

        # Create buildout directories
        for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
            d = self['buildout'][name+'-directory']
            if not os.path.exists(d):
                self._logger.info('Creating directory %s', d)
                os.mkdir(d)

    def _develop(self):
        """Install sources by running setup.py develop on them
        """
        develop = self['buildout'].get('develop')
        if not develop:
            return ''

        dest = self['buildout']['develop-eggs-directory']
        old_files = os.listdir(dest)

        env = dict(os.environ, PYTHONPATH=pkg_resources_loc)
        here = os.getcwd()
        try:
            try:
                for setup in develop.split():
                    setup = self._buildout_path(setup)
                    if os.path.isdir(setup):
                        setup = os.path.join(setup, 'setup.py')

                    self._logger.info("Develop: %s", setup)


                    fd, tsetup = tempfile.mkstemp()
                    try:
                        os.write(fd, runsetup_template % dict(
                            setuptools=pkg_resources_loc,
                            setupdir=os.path.dirname(setup),
                            setup=setup,
                            __file__ = setup,
                            ))

                        args = [
                            zc.buildout.easy_install._safe_arg(tsetup),
                            '-q', 'develop', '-mxN',
                            '-f', zc.buildout.easy_install._safe_arg(
                                ' '.join(self._links)
                                ),
                            '-d', zc.buildout.easy_install._safe_arg(dest),
                            ]

                        if self._log_level <= logging.DEBUG:
                            if self._log_level == logging.DEBUG:
                                del args[1]
                            else:
                                args[1] == '-v'
                            self._logger.debug("in: %s\n%r",
                                               os.path.dirname(setup), args)

                        assert os.spawnl(
                            os.P_WAIT, sys.executable, sys.executable,
                            *args) == 0

                    finally:
                        os.close(fd)
                        os.remove(tsetup)

            except:
                # if we had an error, we need to roll back changes, by
                # removing any files we created.
                self._sanity_check_develop_eggs_files(dest, old_files)
                self._uninstall('\n'.join(
                    [os.path.join(dest, f)
                     for f in os.listdir(dest)
                     if f not in old_files
                     ]))
                raise
                     
            else:
                self._sanity_check_develop_eggs_files(dest, old_files)
                return '\n'.join([os.path.join(dest, f)
                                  for f in os.listdir(dest)
                                  if f not in old_files
                                  ])

        finally:
            os.chdir(here)


    def _sanity_check_develop_eggs_files(self, dest, old_files):
        for f in os.listdir(dest):
            if f in old_files:
                continue
            if not (os.path.isfile(os.path.join(dest, f))
                    and f.endswith('.egg-link')):
                self._logger.warning(
                    "Unexpected entry, %s, in develop-eggs directory", f)

    def _load_recipes(self, parts):
        recipes = {}
        if not parts:
            return recipes
        
        recipes_requirements = []
        pkg_resources.working_set.add_entry(
            self['buildout']['develop-eggs-directory'])
        pkg_resources.working_set.add_entry(self['buildout']['eggs-directory'])

        # Gather requirements
        for part in parts:
            options = self.get(part)
            if options is None:
                raise MissingSection("No section was specified for part", part)

            recipe, entry = self._recipe(part, options)
            if recipe not in recipes_requirements:
                recipes_requirements.append(recipe)

        # Install the recipe distros
        offline = self['buildout'].get('offline', 'false')
        if offline not in ('true', 'false'):
            self._error('Invalid value for offline option: %s', offline)
            
        if offline == 'false':
            dest = self['buildout']['eggs-directory']
        else:
            dest = None

        ws = zc.buildout.easy_install.install(
            recipes_requirements, dest,
            links=self._links,
            index=self['buildout'].get('index'),
            path=[self['buildout']['develop-eggs-directory'],
                  self['buildout']['eggs-directory'],
                  ],
            working_set=pkg_resources.working_set,
            )

        # instantiate the recipes
        for part in parts:
            options = self[part]
            recipe, entry = self._recipe(part, options)
            recipe_class = pkg_resources.load_entry_point(
                recipe, 'zc.buildout', entry)
            recipes[part] = recipe_class(self, part, options)
        
        return recipes

    def _compute_part_signatures(self, parts):
        # Compute recipe signature and add to options
        for part in parts:
            options = self.get(part)
            if options is None:
                options = self[part] = {}
            recipe, entry = self._recipe(part, options)
            req = pkg_resources.Requirement.parse(recipe)
            sig = _dists_sig(pkg_resources.working_set.resolve([req]))
            options['__buildout_signature__'] = ' '.join(sig)

    def _recipe(self, part, options):
        recipe = options['recipe']
        if ':' in recipe:
            recipe, entry = recipe.split(':')
        else:
            entry = 'default'

        return recipe, entry

    def _read_installed_part_options(self):
        old = self._installed_path()
        if os.path.isfile(old):
            parser = ConfigParser.SafeConfigParser(_spacey_defaults)
            parser.optionxform = lambda s: s
            parser.read(old)
            return dict([
                (section,
                 Options(self, section,
                         [item for item in parser.items(section)
                          if item[0] not in _spacey_defaults]
                         )
                 )
                for section in parser.sections()])
        else:
            return {'buildout': Options(self, 'buildout', {'parts': ''})}

    def _installed_path(self):        
        return self._buildout_path(self['buildout']['installed'])

    def _uninstall(self, installed):
        for f in installed.split('\n'):
            if not f:
                continue
            f = self._buildout_path(f)
            if os.path.isdir(f):
                shutil.rmtree(f)
            elif os.path.isfile(f):
                os.remove(f)
                
    def _install(self, part):
        options = self[part]
        recipe, entry = self._recipe(part, options)
        recipe_class = pkg_resources.load_entry_point(
            recipe, 'zc.buildout', entry)
        installed = recipe_class(self, part, options).install()
        if installed is None:
            installed = []
        elif isinstance(installed, basestring):
            installed = [installed]
        base = self._buildout_path('')
        installed = [d.startswith(base) and d[len(base):] or d
                     for d in installed]
        return ' '.join(installed)


    def _save_installed_options(self, installed_options):
        f = open(self._installed_path(), 'w')
        _save_options('buildout', installed_options['buildout'], f)
        for part in installed_options['buildout']['parts'].split():
            print >>f
            _save_options(part, installed_options[part], f)
        f.close()

    def _error(self, message, *args, **kw):
        self._logger.error(message, *args, **kw)
        sys.exit(1)

    def _setup_logging(self):
        root_logger = logging.getLogger()
        handler = logging.StreamHandler(sys.stdout)
        handler.setFormatter(logging.Formatter(self['buildout']['log-format']))
        root_logger.addHandler(handler)
        self._logger = logging.getLogger('buildout')
        level = self['buildout']['log-level']
        if level in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
            level = getattr(logging, level)
        else:
            try:
                level = int(level)
            except ValueError:
                self._error("Invalid logging level %s", level)
        verbosity = self['buildout'].get('verbosity', 0)
        try:
            verbosity = int(verbosity)
        except ValueError:
            self._error("Invalid verbosity %s", verbosity)

        level -= verbosity
        root_logger.setLevel(level)
        self._log_level = level

        if level <= logging.DEBUG:
            sections = list(self)
            sections.sort()
            print 'Configuration data:'
            for section in sections:
                _save_options(section, self[section], sys.stdout)
            print    

    def _maybe_upgrade(self):
        # See if buildout or setuptools need to be upgraded.
        # If they do, do the upgrade and restart the buildout process.

        if self['buildout'].get('offline') == 'true':
            return # skip upgrade in offline mode:
        
        ws = zc.buildout.easy_install.install(
            [
            (spec + ' ' + self['buildout'].get(spec+'-version', '')).strip()
            for spec in ('zc.buildout', 'setuptools')
            ],
            self['buildout']['eggs-directory'],
            links = self['buildout'].get('find-links', '').split(),
            index = self['buildout'].get('index'),
            path = [self['buildout']['develop-eggs-directory']],
            )

        upgraded = []
        for project in 'zc.buildout', 'setuptools':
            req = pkg_resources.Requirement.parse(project)
            if ws.find(req) != pkg_resources.working_set.find(req):
                upgraded.append(ws.find(req))

        if not upgraded:
            return

        if (os.path.abspath(sys.argv[0])
            != os.path.join(os.path.abspath(self['buildout']['bin-directory']),
                            'buildout')
            ):
            self._logger.warn("Not upgrading because not running a local "
                              "buildout command")
            return

        if sys.platform == 'win32' and not self.__windows_restart:
            args = map(zc.buildout.easy_install._safe_arg, sys.argv)
            args.insert(1, '-W')
            if not __debug__:
                args.insert(0, '-O')
            args.insert(0, sys.executable)
            os.execv(sys.executable, args)            
        
        self._logger.info("Upgraded:\n  %s;\nrestarting.",
                          ",\n  ".join([("%s version %s"
                                       % (dist.project_name, dist.version)
                                       )
                                      for dist in upgraded
                                      ]
                                     ),
                          )
                
        # the new dist is different, so we've upgraded.
        # Update the scripts and return True
        zc.buildout.easy_install.scripts(
            ['zc.buildout'], ws, sys.executable,
            self['buildout']['bin-directory'],
            )

        # Restart
        args = map(zc.buildout.easy_install._safe_arg, sys.argv)
        if not __debug__:
            args.insert(0, '-O')
        args.insert(0, sys.executable)
        sys.exit(os.spawnv(os.P_WAIT, sys.executable, args))

    def _load_extensions(self):
        specs = self['buildout'].get('extensions', '').split()
        if specs:
            if self['buildout'].get('offline') == 'true':
                dest = None
            else:
                dest = self['buildout']['eggs-directory']
                if not os.path.exists(dest):
                    self._logger.info('Creating directory %s', dest)
                    os.mkdir(dest)
                    
            zc.buildout.easy_install.install(
                specs, dest,
                path=[self['buildout']['develop-eggs-directory']],
                working_set=pkg_resources.working_set,
                )
            for ep in pkg_resources.iter_entry_points('zc.buildout.extension'):
                ep.load()(self)

    def setup(self, args):
        setup = args.pop(0)
        if os.path.isdir(setup):
            setup = os.path.join(setup, 'setup.py')

        self._logger.info("Running setup script %s", setup)
        setup = os.path.abspath(setup)

        fd, tsetup = tempfile.mkstemp()
        try:
            os.write(fd, runsetup_template % dict(
                setuptools=pkg_resources_loc,
                setupdir=os.path.dirname(setup),
                setup=setup,
                __file__ = setup,
                ))
            os.spawnl(os.P_WAIT, sys.executable, sys.executable, tsetup,
                      *[zc.buildout.easy_install._safe_arg(a)
                        for a in args])
        finally:
            os.close(fd)
            os.remove(tsetup)

    runsetup = setup # backward compat
        
runsetup_template = """
import sys
sys.path.insert(0, %(setuptools)r)
import os, setuptools

__file__ = %(__file__)r

os.chdir(%(setupdir)r)
sys.argv[0] = %(setup)r
execfile(%(setup)r)
"""


_spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*'
                        '|'
                        '^[ \t\r\f\v]+'
                        '|'
                        '[ \t\r\f\v]+$'
                        )

def _quote_spacey_nl(match):
    match = match.group(0).split('\n', 1)
    result = '\n\t'.join(
        [(s
          .replace(' ', '%(__buildout_space__)s')
          .replace('\r', '%(__buildout_space_r__)s')
          .replace('\f', '%(__buildout_space_f__)s')
          .replace('\v', '%(__buildout_space_v__)s')
          .replace('\n', '%(__buildout_space_n__)s')
          )
         for s in match]
        )
    return result

_spacey_defaults = dict(
    __buildout_space__   = ' ',
    __buildout_space_r__ = '\r',
    __buildout_space_f__ = '\f',
    __buildout_space_v__ = '\v',
    __buildout_space_n__ = '\n',
    )

def _save_options(section, options, f):
    print >>f, '[%s]' % section
    items = options.items()
    items.sort()
    for option, value in items:
        value = value.replace('%', '%%')
        value = _spacey_nl.sub(_quote_spacey_nl, value)
        if value.startswith('\n\t'):
            value = '%(__buildout_space_n__)s' + value[2:]
        if value.endswith('\n\t'):
            value = value[:-2] + '%(__buildout_space_n__)s'
        print >>f, option, '=', value
            
    

def _open(base, filename, seen):
    """Open a configuration file and return the result as a dictionary,

    Recursively open other files based on buildout options found.
    """

    filename = os.path.join(base, filename)
    if filename in seen:
        raise zc.buildout.UserError("Recursive file include", seen, filename)

    base = os.path.dirname(filename)
    seen.append(filename)

    result = {}

    parser = ConfigParser.SafeConfigParser()
    parser.optionxform = lambda s: s
    parser.readfp(open(filename))
    extends = extended_by = None
    for section in parser.sections():
        options = dict(parser.items(section))
        if section == 'buildout':
            extends = options.pop('extends', extends)
            extended_by = options.pop('extended-by', extended_by)
        result[section] = options

    if extends:
        extends = extends.split()
        extends.reverse()
        for fname in extends:
            result = _update(_open(base, fname, seen), result)

    if extended_by:
        for fname in extended_by.split():
            result = _update(result, _open(base, fname, seen))

    seen.pop()
    return result
    

def _dir_hash(dir):
    hash = md5.new()
    for (dirpath, dirnames, filenames) in os.walk(dir):
        filenames[:] = [f for f in filenames
                        if not (f.endswith('pyc') or f.endswith('pyo'))
                        ]
        hash.update(' '.join(dirnames))
        hash.update(' '.join(filenames))
        for name in filenames:
            hash.update(open(os.path.join(dirpath, name)).read())
    return hash.digest().encode('base64').strip()
    
def _dists_sig(dists):
    result = []
    for dist in dists:
        location = dist.location
        if dist.precedence == pkg_resources.DEVELOP_DIST:
            result.append(dist.project_name + '-' + _dir_hash(location))
        else:
            result.append(os.path.basename(location))
    return result

def _update(d1, d2):
    for section in d2:
        if section in d1:
            d1[section].update(d2[section])
        else:
            d1[section] = d2[section]
    return d1

def _error(*message):
    sys.stderr.write('Error: ' + ' '.join(message) +'\n')
    sys.exit(1)

_usage = """\
Usage: buildout [options] [assignments] [command [command arguments]]

Options:

  -h, --help

     Print this message and exit.

  -v

     Increase the level of verbosity.  This option can be used multiple times.

  -q

     Decrease the level of verbosity.  This option can be used multiple times.

  -c config_file

     Specify the path to the buildout configuration file to be used.
     This defaults to the file named "buildout.cfg" in the current
     working directory.

Assignments are of the form: section:option=value and are used to
provide configuration options that override those given in the
configuration file.  For example, to run the buildout in offline mode,
use buildout:offline=true.

Options and assignments can be interspersed.

Commands:

  install [parts]

    Install parts.  If no command arguments are given, then the parts
    definition from the configuration file is used.  Otherwise, the
    arguments specify the parts to be installed.

  bootstrap

    Create a new buildout in the current working directory, copying
    the buildout and setuptools eggs and, creating a basic directory
    structure and a buildout-local buildout script.

"""
def _help():
    print _usage
    sys.exit(0)

def main(args=None):
    if args is None:
        args = sys.argv[1:]

    config_file = 'buildout.cfg'
    verbosity = 0
    options = []
    windows_restart = False
    while args:
        if args[0][0] == '-':
            op = orig_op = args.pop(0)
            op = op[1:]
            while op and op[0] in 'vqhW':
                if op[0] == 'v':
                    verbosity += 10
                elif op[0] == 'q':
                    verbosity -= 10
                elif op[0] == 'W':
                    windows_restart = True
                else:
                    _help()
                op = op[1:]
                
            if op[:1] == 'c':
                op = op[1:]
                if op:
                    config_file = op
                else:
                    if args:
                        config_file = args.pop(0)
                    else:
                        _error("No file name specified for option", orig_op)
            elif op:
                if orig_op == '--help':
                    _help()
                _error("Invalid option", '-'+op[0])
        elif '=' in args[0]:
            option, value = args.pop(0).split('=', 1)
            if len(option.split(':')) != 2:
                _error('Invalid option:', option)
            section, option = option.split(':')
            options.append((section.strip(), option.strip(), value.strip()))
        else:
            # We've run out of command-line options and option assignnemnts
            # The rest should be commands, so we'll stop here
            break

    if verbosity:
        options.append(('buildout', 'verbosity', str(verbosity)))

    if args:
        command = args.pop(0)
        if command not in ('install', 'bootstrap', 'runsetup', 'setup'):
            _error('invalid command:', command)
    else:
        command = 'install'

    try:
        try:
            buildout = Buildout(config_file, options, windows_restart)
            getattr(buildout, command)(args)
        except zc.buildout.UserError, v:
            _error(str(v))
            
    finally:
            logging.shutdown()

if sys.version_info[:2] < (2, 4):
    def reversed(iterable):
        result = list(iterable);
        result.reverse()
        return result