buildout.py 94.6 KB
Newer Older
1
##############################################################################
2
#
3
# Copyright (c) 2005-2009 Zope Foundation and Contributors.
4 5 6 7 8 9 10 11 12 13 14 15 16
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Buildout main script
"""

17
from zc.buildout.rmtree import rmtree
18
import zc.buildout.easy_install
19

20 21
from functools import partial

22
try:
23
    from hashlib import md5 as md5_original
24
except ImportError:
25
    from md5 import md5 as md5_original
Jim Fulton's avatar
Jim Fulton committed
26

27
try:
28
    from collections.abc import MutableMapping as DictMixin
29
except ImportError:
30
    from UserDict import DictMixin
31

32 33 34 35 36
try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO

37 38 39 40 41
try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

42
import zc.buildout.configparser
Jim Fulton's avatar
Jim Fulton committed
43
import copy
44
import datetime
45
import distutils.errors
46
import errno
Jim Fulton's avatar
Jim Fulton committed
47
import glob
48 49
import importlib
import inspect
Jim Fulton's avatar
Jim Fulton committed
50
import itertools
Jim Fulton's avatar
Jim Fulton committed
51
import logging
52
import os
Jim Fulton's avatar
Jim Fulton committed
53
import pkg_resources
54 55
import re
import shutil
56
import subprocess
57
import sys
58
import tempfile
59
import pprint
60
import zc.buildout
61
import zc.buildout.download
62
from functools import partial
63

64 65 66 67 68 69
PY3 = sys.version_info[0] == 3
if PY3:
    text_type = str
else:
    text_type = unicode

70 71 72 73 74 75
try:
    hashed = md5_original(b'test')
    md5 = md5_original
except ValueError:
    md5 = partial(md5_original, usedforsecurity=False)

76 77 78 79 80 81 82 83 84 85 86 87 88

def command(method):
    method.buildout_command = True
    return method


def commands(cls):
    for name, method in cls.__dict__.items():
        if hasattr(method, "buildout_command"):
            cls.COMMANDS.add(name)
    return cls


89 90 91 92 93 94 95 96
def _print_options(sep=' ', end='\n', file=None):
    return sep, end, file

def print_(*args, **kw):
    sep, end, file = _print_options(**kw)
    if file is None:
        file = sys.stdout
    file.write(sep.join(map(str, args))+end)
97

98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
_MARKER = []

class BuildoutSerialiser(object):
    # XXX: I would like to access pprint._safe_repr, but it's not
    # officially available. PrettyPrinter class has a functionally-speaking
    # static method "format" which just calls _safe_repr, but it is not
    # declared as static... So I must create an instance of it.
    _format = pprint.PrettyPrinter().format
    _dollar = '\\x%02x' % ord('$')
    _semicolon = '\\x%02x' % ord(';')
    _safe_globals = {'__builtins__': {
        # Types which are represented as calls to their constructor.
        'bytearray': bytearray,
        'complex': complex,
        'frozenset': frozenset,
        'set': set,
        # Those buildins are available through keywords, which allow creating
        # instances which in turn give back access to classes. So no point in
        # hiding them.
        'dict': dict,
        'list': list,
        'str': str,
        'tuple': tuple,
        'False': False,
        'True': True,
        'None': None,
    }}

    def loads(self, value):
        return eval(value, self._safe_globals)

    def dumps(self, value):
        value, isreadable, _ = self._format(value, {}, 0, 0)
        if not isreadable:
            raise ValueError('Value cannot be serialised: %s' % (value, ))
        return value.replace('$', self._dollar).replace(';', self._semicolon)

SERIALISED_VALUE_MAGIC = '!py'
SERIALISED = re.compile(SERIALISED_VALUE_MAGIC + '([^!]*)!(.*)')
SERIALISER_REGISTRY = {
    '': BuildoutSerialiser(),
}
SERIALISER_VERSION = ''
SERIALISER = SERIALISER_REGISTRY[SERIALISER_VERSION]
# Used only to compose data
SERIALISER_PREFIX = SERIALISED_VALUE_MAGIC + SERIALISER_VERSION + '!'
assert SERIALISED.match(SERIALISER_PREFIX).groups() == (
    SERIALISER_VERSION, ''), SERIALISED.match(SERIALISER_PREFIX).groups()

def dumps(value):
    orig_value = value
    value = SERIALISER.dumps(value)
    assert SERIALISER.loads(value) == orig_value, (repr(value), orig_value)
    return SERIALISER_PREFIX + value

def loads(value):
    assert value.startswith(SERIALISED_VALUE_MAGIC), repr(value)
    version, data = SERIALISED.match(value).groups()
    return SERIALISER_REGISTRY[version].loads(data)

158
realpath = zc.buildout.easy_install.realpath
159

160
_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
161

162
class MissingOption(zc.buildout.UserError, KeyError):
163
    """A required option was missing.
164 165
    """

166
class MissingSection(zc.buildout.UserError, KeyError):
167
    """A required section is missing.
168 169
    """

170
    def __str__(self):
Jim Fulton's avatar
Jim Fulton committed
171
        return "The referenced section, %r, was not defined." % self.args[0]
172

173

174
def _annotate_section(section, source):
175
    for key in section:
176
        section[key] = SectionKey(section[key], source)
177 178
    return section

179 180 181 182 183 184 185 186 187 188 189 190 191 192 193

class SectionKey(object):
    def __init__(self, value, source):
        self.history = []
        self.value = value
        self.addToHistory("SET", value, source)

    @property
    def source(self):
        return self.history[-1].source

    def overrideValue(self, sectionkey):
        self.value = sectionkey.value
        if sectionkey.history[-1].operation not in ['ADD', 'REMOVE']:
            self.addToHistory("OVERRIDE", sectionkey.value, sectionkey.source)
194 195
        else:
            self.history = copy.deepcopy(sectionkey.history)
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312

    def setDirectory(self, value):
        self.value = value
        self.addToHistory("DIRECTORY", value, self.source)

    def addToValue(self, added, source):
        subvalues = self.value.split('\n') + added.split('\n')
        self.value = "\n".join(subvalues)
        self.addToHistory("ADD", added, source)

    def removeFromValue(self, removed, source):
        subvalues = [
            v
            for v in self.value.split('\n')
            if v not in removed.split('\n')
        ]
        self.value = "\n".join(subvalues)
        self.addToHistory("REMOVE", removed, source)

    def addToHistory(self, operation, value, source):
        item = HistoryItem(operation, value, source)
        self.history.append(item)

    def printAll(self, key, basedir, verbose):
        self.printKeyAndValue(key)
        if verbose:
            self.printVerbose(basedir)
        else:
            self.printTerse(basedir)

    def printKeyAndValue(self, key):
        lines = self.value.splitlines()
        if len(lines) <= 1:
            args = [key, "="]
            if self.value:
                args.append(" ")
                args.append(self.value)
            print_(*args, sep='')
        else:
            print_(key, "= ", lines[0], sep='')
            for line in lines[1:]:
                print_(line)

    def printVerbose(self, basedir):
        print_()
        for item in reversed(self.history):
            item.printAll(basedir)
        print_()

    def printTerse(self, basedir):
        toprint = []
        history = copy.deepcopy(self.history)
        while history:
            next = history.pop()
            if next.operation in ["ADD", "REMOVE"]:
                next.printShort(toprint, basedir)
            else:
                next.printShort(toprint, basedir)
                break

        for line in reversed(toprint):
            if line.strip():
                print_(line)

    def __repr__(self):
        return "<SectionKey value=%s source=%s>" % (
            " ".join(self.value.split('\n')), self.source)


class HistoryItem(object):
    def __init__(self, operation, value, source):
        self.operation = operation
        self.value = value
        self.source = source

    def printShort(self, toprint, basedir):
        source = self.source_for_human(basedir)
        if self.operation in ["OVERRIDE", "SET", "DIRECTORY"]:
            toprint.append("    " + source)
        elif self.operation == "ADD":
            toprint.append("+=  " + source)
        elif self.operation == "REMOVE":
            toprint.append("-=  " + source)

    def printOperation(self):
        lines = self.value.splitlines()
        if len(lines) <= 1:
            print_("  ", self.operation, "VALUE =", self.value)
        else:
            print_("  ", self.operation, "VALUE =")
            for line in lines:
                print_("  ", "  ", line)

    def printSource(self, basedir):
        if self.source in (
            'DEFAULT_VALUE', 'COMPUTED_VALUE', 'COMMAND_LINE_VALUE'
        ):
            prefix = "AS"
        else:
            prefix = "IN"
        print_("  ", prefix, self.source_for_human(basedir))

    def source_for_human(self, basedir):
        if self.source.startswith(basedir):
            return os.path.relpath(self.source, basedir)
        else:
            return self.source

    def printAll(self, basedir):
        self.printSource(basedir)
        self.printOperation()

    def __repr__(self):
        return "<HistoryItem operation=%s value=%s source=%s>" % (
            self.operation, " ".join(self.value.split('\n')), self.source)


313 314 315 316 317
def _annotate(data, note):
    for key in data:
        data[key] = _annotate_section(data[key], note)
    return data

318

Godefroid Chapelle's avatar
Godefroid Chapelle committed
319
def _print_annotate(data, verbose, chosen_sections, basedir):
320
    sections = list(data.keys())
321
    sections.sort()
322 323 324
    print_()
    print_("Annotated sections")
    print_("="*len("Annotated sections"))
325
    for section in sections:
Godefroid Chapelle's avatar
Godefroid Chapelle committed
326
        if (not chosen_sections) or (section in chosen_sections):
327 328 329 330 331 332 333
            print_()
            print_('[%s]' % section)
            keys = list(data[section].keys())
            keys.sort()
            for key in keys:
                sectionkey = data[section][key]
                sectionkey.printAll(key, basedir, verbose)
334

335 336 337 338 339 340
def _remove_ignore_missing(path):
    try:
        os.remove(path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise
341

342
def _unannotate_section(section):
343 344 345
    for key in section:
        section[key] = section[key].value
    return section
346

347
def _unannotate(data):
348 349 350
    for key in data:
        _unannotate_section(data[key])
    return data
351

352 353 354 355 356 357 358 359 360 361 362 363 364 365 366

def _format_picked_versions(picked_versions, required_by):
    output = ['[versions]']
    required_output = []
    for dist_, version in picked_versions:
        if dist_ in required_by:
            required_output.append('')
            required_output.append('# Required by:')
            for req_ in sorted(required_by[dist_]):
                required_output.append('# '+req_)
            target = required_output
        else:
            target = output
        target.append("%s = %s" % (dist_, version))
    output.extend(required_output)
367
    return output
368 369


370
_buildout_default_options = _annotate_section({
371 372
    'allow-hosts': '*',
    'allow-picked-versions': 'true',
373
    'bin-directory': 'bin',
374 375 376 377 378
    'develop-eggs-directory': 'develop-eggs',
    'eggs-directory': 'eggs',
    'executable': sys.executable,
    'find-links': '',
    'install-from-cache': 'false',
379
    'installed': '.installed.cfg',
380
    'log-format': '',
381 382 383
    'log-level': 'INFO',
    'newest': 'true',
    'offline': 'false',
384
    'dry-run': 'false',
385
    'check-signature': 'true',
386
    'parts-directory': 'parts',
Jim Fulton's avatar
Jim Fulton committed
387
    'prefer-final': 'true',
388
    'python': 'buildout',
389
    'show-picked-versions': 'false',
390
    'socket-timeout': '',
391
    'update-versions-file': '',
392
    'use-dependency-links': 'true',
Jason Madden's avatar
Jason Madden committed
393
    'allow-unknown-extras': 'false',
394 395
    }, 'DEFAULT_VALUE')

396

397 398 399 400 401 402
def _get_user_config():
    buildout_home = os.path.join(os.path.expanduser('~'), '.buildout')
    buildout_home = os.environ.get('BUILDOUT_HOME', buildout_home)
    return os.path.join(buildout_home, 'default.cfg')


403 404 405
network_cache_parameter_dict = {}


406
@commands
407
class Buildout(DictMixin):
408

409 410
    COMMANDS = set()

411 412
    installed_part_options = None

Jim Fulton's avatar
Jim Fulton committed
413
    def __init__(self, config_file, cloptions,
414
                 use_user_defaults=True,
415
                 command=None, args=()):
416

417
        __doing__ = 'Initializing.'
418

419
        # default options
420 421 422
        _buildout_default_options_copy = copy.deepcopy(
            _buildout_default_options)
        data = dict(buildout=_buildout_default_options_copy)
423
        self._buildout_dir = os.getcwd()
424

Jim Fulton's avatar
Jim Fulton committed
425
        if config_file and not _isurl(config_file):
426 427
            config_file = os.path.abspath(config_file)
            if not os.path.exists(config_file):
428
                if command == 'init':
429
                    self._init_config(config_file, args)
Jim Fulton's avatar
Jim Fulton committed
430
                elif command == 'setup':
431 432
                    # Sigh. This model of a buildout instance
                    # with methods is breaking down. :(
Jim Fulton's avatar
Jim Fulton committed
433
                    config_file = None
434 435
                    data['buildout']['directory'] = SectionKey(
                        '.', 'COMPUTED_VALUE')
436 437 438
                else:
                    raise zc.buildout.UserError(
                        "Couldn't open %s" % config_file)
439 440 441
            elif command == 'init':
                raise zc.buildout.UserError(
                    "%r already exists." % config_file)
442 443

            if config_file:
444 445
                data['buildout']['directory'] = SectionKey(
                    os.path.dirname(config_file), 'COMPUTED_VALUE')
Jim Fulton's avatar
Jim Fulton committed
446

447 448 449
        result = {}
        for section, option, value in cloptions:
          result.setdefault(section, {})[option] = value
450

451 452 453
        options = result.setdefault('buildout', {})

        extends = []
454 455
        user_config = _get_user_config()
        if use_user_defaults and os.path.exists(user_config):
456
            extends.append(user_config)
457
        if config_file:
458 459 460 461 462
            extends.append(config_file)
        clextends = options.get('extends')
        if clextends:
            extends.append(clextends)
        options['extends'] = '\n'.join(extends)
463

464
        data = _extends(data, result, os.getcwd(), 'COMMAND_LINE_VALUE')
465

466 467
        # Set up versions section, if necessary
        if 'versions' not in data['buildout']:
468 469
            data['buildout']['versions'] = SectionKey(
                'versions', 'DEFAULT_VALUE')
470 471 472 473
            if 'versions' not in data:
                data['versions'] = {}

        # Default versions:
474
        versions_section_name = data['buildout']['versions'].value
475 476 477 478 479
        if versions_section_name:
            versions = data[versions_section_name]
        else:
            versions = {}
        versions.update(
480 481
            dict((k, SectionKey(v(), 'DEFAULT_VALUE'))
                 # Use lambdas to compute values only if needed
482 483 484
                 for (k, v) in (
                     # Prevent downgrading due to prefer-final:
                     ('zc.buildout',
485
                      lambda: '>='+pkg_resources.working_set.find(
486
                          pkg_resources.Requirement.parse('zc.buildout')
487 488
                          # Skip local part because ">=x.y.z+abc" is invalid
                          ).parsed_version.public),
489
                     # Use 2, even though not final
490
                     ('zc.recipe.egg', lambda: '>=2.0.6'),
491 492 493 494
                     )
                 if k not in versions
                 ))

495 496 497 498
        # Absolutize some particular directory, handling also the ~/foo form,
        # and considering the location of the configuration file that generated
        # the setting as the base path, falling back to the main configuration
        # file location
499
        for name in ('download-cache', 'eggs-directory', 'extends-cache'):
500
            if name in data['buildout']:
501 502 503
                sectionkey = data['buildout'][name]
                origdir = sectionkey.value
                src = sectionkey.source
504 505 506
                if not origdir:
                    del data['buildout'][name]
                    continue
507 508 509 510 511 512 513
                if '${' in origdir:
                    continue
                if not os.path.isabs(origdir):
                    if src in ('DEFAULT_VALUE',
                               'COMPUTED_VALUE',
                               'COMMAND_LINE_VALUE'):
                        if 'directory' in data['buildout']:
514
                            basedir = data['buildout']['directory'].value
515 516 517 518 519 520 521 522 523 524 525 526 527
                        else:
                            basedir = self._buildout_dir
                    else:
                        if _isurl(src):
                            raise zc.buildout.UserError(
                                'Setting "%s" to a non absolute location ("%s") '
                                'within a\n'
                                'remote configuration file ("%s") is ambiguous.' % (
                                    name, origdir, src))
                        basedir = os.path.dirname(src)
                    absdir = os.path.expanduser(origdir)
                    if not os.path.isabs(absdir):
                        absdir = os.path.join(basedir, absdir)
528
                    absdir = os.path.abspath(absdir)
529
                    sectionkey.setDirectory(absdir)
530

531 532
        self._annotated = copy.deepcopy(data)
        self._raw = _unannotate(data)
533 534
        self._data = {}
        self._parts = []
535 536
        self._initializing = []
        self._signature_cache = {}
537
        self._default_requirement = None
538

539 540
        # provide some defaults before options are parsed
        # because while parsing options those attributes might be
541
        # used already (Gottfried Ganssauge)
542
        buildout_section = self._raw['buildout']
543

544 545 546 547 548
        # Try to make sure we have absolute paths for standard
        # directories. We do this before doing substitutions, in case
        # a one of these gets read by another section.  If any
        # variable references are used though, we leave it as is in
        # _buildout_path.
549 550 551 552 553 554
        if 'directory' in buildout_section:
            self._buildout_dir = buildout_section['directory']
            for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
                d = self._buildout_path(buildout_section[name+'-directory'])
                buildout_section[name+'-directory'] = d

555 556 557 558 559 560 561 562
        # Attributes on this buildout object shouldn't be used by
        # recipes in their __init__.  It can cause bugs, because the
        # recipes will be instantiated below (``options = self['buildout']``)
        # before this has completed initializing.  These attributes are
        # left behind for legacy support but recipe authors should
        # beware of using them.  A better practice is for a recipe to
        # use the buildout['buildout'] options.
        links = buildout_section['find-links']
563
        self._links = links and links.split() or ()
564
        allow_hosts = buildout_section['allow-hosts'].split('\n')
565
        self._allow_hosts = tuple([host.strip() for host in allow_hosts
566
                                   if host.strip() != ''])
567
        self._logger = logging.getLogger('zc.buildout')
Jim Fulton's avatar
Jim Fulton committed
568 569 570 571
        self.offline = bool_option(buildout_section, 'offline')
        self.newest = ((not self.offline) and
                       bool_option(buildout_section, 'newest')
                       )
572
        self.dry_run = (buildout_section['dry-run'] == 'true')
573
        self.check_signature = (buildout_section['check-signature'] == 'true')
574

575 576 577
        ##################################################################
        ## WARNING!!!
        ## ALL ATTRIBUTES MUST HAVE REASONABLE DEFAULTS AT THIS POINT
578 579 580
        ## OTHERWISE ATTRIBUTEERRORS MIGHT HAPPEN ANY TIME FROM RECIPES.
        ## RECIPES SHOULD GENERALLY USE buildout['buildout'] OPTIONS, NOT
        ## BUILDOUT ATTRIBUTES.
581
        ##################################################################
582 583 584
        # initialize some attrs and buildout directories.
        options = self['buildout']

585
        # now reinitialize
586 587
        links = options.get('find-links', '')
        self._links = links and links.split() or ()
588

589
        allow_hosts = options['allow-hosts'].split('\n')
590
        self._allow_hosts = tuple([host.strip() for host in allow_hosts
Tarek Ziad's avatar
Tarek Ziad committed
591
                                   if host.strip() != ''])
592 593

        self._buildout_dir = options['directory']
594 595 596

        # Make sure we have absolute paths for standard directories.  We do this
        # a second time here in case someone overrode these in their configs.
597
        for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
598 599
            d = self._buildout_path(options[name+'-directory'])
            options[name+'-directory'] = d
600

Jim Fulton's avatar
Jim Fulton committed
601 602 603
        if options['installed']:
            options['installed'] = os.path.join(options['directory'],
                                                options['installed'])
Jim Fulton's avatar
Jim Fulton committed
604

Jim Fulton's avatar
Jim Fulton committed
605
        self._setup_logging()
606
        self._setup_socket_timeout()
Jim Fulton's avatar
Jim Fulton committed
607

608 609
        # finish w versions
        if versions_section_name:
610
            # refetching section name just to avoid a warning
611 612 613
            versions = self[versions_section_name]
        else:
            # remove annotations
614
            versions = dict((k, v.value) for (k, v) in versions.items())
615
        options['versions'] # refetching section name just to avoid a warning
616
        self.versions = versions
Gary Poster's avatar
Gary Poster committed
617
        zc.buildout.easy_install.default_versions(versions)
618

Jim Fulton's avatar
Jim Fulton committed
619 620
        zc.buildout.easy_install.prefer_final(
            bool_option(options, 'prefer-final'))
621
        zc.buildout.easy_install.use_dependency_links(
Jim Fulton's avatar
Jim Fulton committed
622
            bool_option(options, 'use-dependency-links'))
Jim Fulton's avatar
Jim Fulton committed
623
        zc.buildout.easy_install.allow_picked_versions(
Jim Fulton's avatar
Jim Fulton committed
624
                bool_option(options, 'allow-picked-versions'))
625 626
        self.show_picked_versions = bool_option(options,
                                                'show-picked-versions')
627
        self.update_versions_file = options['update-versions-file']
628 629
        zc.buildout.easy_install.store_required_by(self.show_picked_versions or
                                                   self.update_versions_file)
Jim Fulton's avatar
Jim Fulton committed
630

631
        download_cache = options.get('download-cache')
632
        extends_cache = options.get('extends-cache')
633 634 635

        if bool_option(options, 'abi-tag-eggs', 'false'):
            from zc.buildout.pep425tags import get_abi_tag
636 637 638
            options['eggs-directory'] = os.path.join(
                options['eggs-directory'], get_abi_tag())

639
        eggs_cache = options.get('eggs-directory')
640

641 642 643 644 645
        for cache in [download_cache, extends_cache, eggs_cache]:
            if cache:
                cache = os.path.join(options['directory'], cache)
                if not os.path.exists(cache):
                    self._logger.info('Creating directory %r.', cache)
646
                    os.makedirs(cache)
647

648
        if download_cache:
Reinout van Rees's avatar
Reinout van Rees committed
649
            # Actually, we want to use a subdirectory in there called 'dist'.
650
            download_cache = os.path.join(download_cache, 'dist')
651
            if not os.path.exists(download_cache):
652 653 654
                os.mkdir(download_cache)
            zc.buildout.easy_install.download_cache(download_cache)

655 656 657 658
        if bool_option(options, 'install-from-cache'):
            if self.offline:
                raise zc.buildout.UserError(
                    "install-from-cache can't be used with offline mode.\n"
pombredanne's avatar
pombredanne committed
659
                    "Nothing is installed, even from cache, in offline\n"
660 661 662
                    "mode, which might better be called 'no-install mode'.\n"
                    )
            zc.buildout.easy_install.install_from_cache(True)
663

664 665 666 667
        # "Use" each of the defaults so they aren't reported as unused options.
        for name in _buildout_default_options:
            options[name]

668 669
        os.chdir(options['directory'])

670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
        networkcache_section_name = options.get('networkcache-section')
        if networkcache_section_name:
            networkcache_section = self[networkcache_section_name]
            for k in (
                'download-cache-url',
                'download-dir-url',
                'upload-cache-url',
                'upload-dir-url',
                'signature-certificate-list',
                'signature-private-key-file',
                'shacache-ca-file',
                'shacache-cert-file',
                'shacache-key-file',
                'shadir-ca-file',
                'shadir-cert-file',
                'shadir-key-file',
            ):
                network_cache_parameter_dict[k] = networkcache_section.get(k, '')
                # parse signature list
                cert_marker = '-----BEGIN CERTIFICATE-----'
            network_cache_parameter_dict['signature-certificate-list'] = \
                [cert_marker + '\n' + q.strip() \
                    for q in network_cache_parameter_dict['signature-certificate-list'].split(cert_marker) \
                    if q.strip()]

695 696 697 698
    def _buildout_path(self, name):
        if '${' in name:
            return name
        return os.path.join(self._buildout_dir, name)
699

700
    @command
Jim Fulton's avatar
Jim Fulton committed
701
    def bootstrap(self, args):
702
        __doing__ = 'Bootstrapping.'
703

704
        self._setup_directories()
Jim Fulton's avatar
Jim Fulton committed
705

706 707 708 709 710 711 712 713
        # Hack: propagate libnetworkcache soft dependency
        specs = ['zc.buildout']
        try:
            import slapos.libnetworkcache
            specs.append('slapos.libnetworkcache')
        except ImportError:
            pass

714 715 716 717 718 719 720 721
        # Install buildout and dependent eggs following pinned versions.
        dest = self['buildout']['eggs-directory']
        path = [self['buildout']['develop-eggs-directory']]
        if self.offline:
            # Cannot install: just check requirements are already met
            path.append(dest)
            dest = None
        ws = zc.buildout.easy_install.install(
722
                specs,
723 724 725 726 727 728 729 730 731 732 733
                dest,
                links=self._links,
                index=self['buildout'].get('index'),
                path=path,
                newest=self.newest,
                allow_hosts=self._allow_hosts,
                )
        # If versions aren't pinned or if current modules match,
        # nothing will be installed, but then we'll copy them to
        # the local eggs or develop-eggs folder just after this.

734
        # Now copy buildout and setuptools eggs, and record destination eggs:
Jim Fulton's avatar
Jim Fulton committed
735
        entries = []
736
        for dist in ws:
Jim Fulton's avatar
Jim Fulton committed
737
            if dist.precedence == pkg_resources.DEVELOP_DIST:
738
                dest = os.path.join(self['buildout']['develop-eggs-directory'],
739
                                    dist.key + '.egg-link')
Guido Stevens's avatar
Guido Stevens committed
740 741
                with open(dest, 'w') as fh:
                    fh.write(dist.location)
Jim Fulton's avatar
Jim Fulton committed
742 743 744 745 746 747
                entries.append(dist.location)
            else:
                dest = os.path.join(self['buildout']['eggs-directory'],
                                    os.path.basename(dist.location))
                entries.append(dest)
                if not os.path.exists(dest):
Jim Fulton's avatar
Jim Fulton committed
748 749 750 751
                    if os.path.isdir(dist.location):
                        shutil.copytree(dist.location, dest)
                    else:
                        shutil.copy2(dist.location, dest)
Jim Fulton's avatar
Jim Fulton committed
752 753 754 755

        # Create buildout script
        ws = pkg_resources.WorkingSet(entries)
        ws.require('zc.buildout')
756
        options = self['buildout']
757 758 759 760 761 762 763
        eggs_dir = options['eggs-directory']
        develop_eggs_dir = options['develop-eggs-directory']
        ws = zc.buildout.easy_install.sort_working_set(
                ws,
                eggs_dir=eggs_dir,
                develop_eggs_dir=develop_eggs_dir
                )
Jim Fulton's avatar
Jim Fulton committed
764 765
        zc.buildout.easy_install.scripts(
            ['zc.buildout'], ws, sys.executable,
766
            options['bin-directory'],
767 768 769 770 771
            relative_paths = (
                bool_option(options, 'relative-paths', False)
                and options['directory']
                or ''),
            )
Jim Fulton's avatar
Jim Fulton committed
772

773
    def _init_config(self, config_file, args):
774
        print_('Creating %r.' % config_file)
775 776 777 778
        f = open(config_file, 'w')
        sep = re.compile(r'[\\/]')
        if args:
            eggs = '\n  '.join(a for a in args if not sep.search(a))
779
            sepsub = os.path.sep == '/' and '/' or re.escape(os.path.sep)
780
            paths = '\n  '.join(
781
                sep.sub(sepsub, a)
782
                for a in args if sep.search(a))
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802
            f.write('[buildout]\n'
                    'parts = py\n'
                    '\n'
                    '[py]\n'
                    'recipe = zc.recipe.egg\n'
                    'interpreter = py\n'
                    'eggs =\n'
                    )
            if eggs:
                f.write('  %s\n' % eggs)
            if paths:
                f.write('extra-paths =\n  %s\n' % paths)
                for p in [a for a in args if sep.search(a)]:
                    if not os.path.exists(p):
                        os.mkdir(p)

        else:
            f.write('[buildout]\nparts =\n')
        f.close()

803
    @command
804 805 806 807
    def init(self, args):
        self.bootstrap(())
        if args:
            self.install(())
808

809
    @command
810
    def install(self, install_args):
811 812 813 814 815 816 817 818 819 820 821 822 823
        try:
            self._install_parts(install_args)
        finally:
            if self.installed_part_options is not None:
                try:
                    self._save_installed_options()
                finally:
                    del self.installed_part_options
        if self.show_picked_versions or self.update_versions_file:
            self._print_picked_versions()
        self._unload_extensions()

    def _install_parts(self, install_args):
824
        __doing__ = 'Installing.'
825

Jim Fulton's avatar
Jim Fulton committed
826
        self._load_extensions()
827
        self._setup_directories()
Jim Fulton's avatar
Jim Fulton committed
828

829 830 831 832
        # Add develop-eggs directory to path so that it gets searched
        # for eggs:
        sys.path.insert(0, self['buildout']['develop-eggs-directory'])

pombredanne's avatar
pombredanne committed
833
        # Check for updates. This could cause the process to be restarted
834 835
        self._maybe_upgrade()

836
        # load installed data
837 838
        installed_part_options = self._read_installed_part_options()
        installed_parts = installed_part_options['buildout']['parts'].split()
839

840 841 842 843 844 845 846 847
        # Remove old develop eggs
        self._uninstall(
            installed_part_options['buildout'].get(
                'installed_develop_eggs', '')
            )

        # Build develop eggs
        installed_develop_eggs = self._develop()
848 849
        installed_part_options['buildout']['installed_develop_eggs'
                                           ] = installed_develop_eggs
850

851 852
        # From now, the caller will update the .installed.cfg at return.
        self.installed_part_options = installed_part_options
853

854
        install_parts = self['buildout']['parts']
855 856
        if install_args:
            install_parts = install_args
857 858
            uninstall_missing = False
        else:
859
            install_parts = install_parts.split()
860 861
            uninstall_missing = True

862 863
        # load and initialize recipes
        [self[part]['recipe'] for part in install_parts]
864 865
        if not install_args:
            install_parts = self._parts
866

867
        if self._log_level < logging.DEBUG:
868 869
            sections = list(self)
            sections.sort()
870 871
            print_()
            print_('Configuration data:')
872
            for section in sorted(self._data):
873
                _save_options(section, self[section], sys.stdout)
874
            print_()
875

876
        del self._signature_cache
877

878 879
        # uninstall parts that are no-longer used or who's configs
        # have changed
880 881 882 883 884 885 886 887 888 889 890 891
        if self._logger.getEffectiveLevel() < logging.DEBUG:
            reinstall_reason_score = -1
        elif int(os.getenv('BUILDOUT_INFO_REINSTALL_REASON') or 1):
            # We rely on the fact that installed_parts is sorted according to
            # dependencies (unless install_args). This is not the case of
            # installed_parts.
            reinstall_reason_score = len(installed_parts)
        else:
            # Provide a way to disable in tests
            # or we'd have to update all recipe eggs.
            reinstall_reason_score = 0
        reinstall_reason = None
892
        for part in reversed(installed_parts):
893 894 895 896 897 898
            try:
                part_index = install_parts.index(part)
            except ValueError:
                if not uninstall_missing:
                    continue
            else:
899 900
                old_options = installed_part_options[part].copy()
                installed_files = old_options.pop('__buildout_installed__')
901
                new_options = self.get(part).copy()
902 903 904 905 906 907 908 909 910 911 912 913 914
                if not self.check_signature:
                    old_signature = old_options.get(
                        '__buildout_signature__', None)
                    new_signature = new_options.get(
                        '__buildout_signature__', None)
                    if old_signature != new_signature:
                        self._logger.info(
                            '[%s]: __buildout_signature__ is different '
                            'but ignored.' % part)
                        if new_signature:
                            old_options['__buildout_signature__'] = new_signature
                        else:
                            del old_options['__buildout_signature__']
915 916 917 918 919 920
                if old_options == new_options:
                    # The options are the same, but are all of the
                    # installed files still there?  If not, we should
                    # reinstall.
                    if not installed_files:
                        continue
921 922 923
                    for installed_path in installed_files.split('\n'):
                        if not os.path.exists(
                                self._buildout_path(installed_path)):
924
                            break
Jim Fulton's avatar
Jim Fulton committed
925
                    else:
926
                        continue
927 928
                else:
                    installed_path = None
929

930 931 932 933 934 935 936
                if part_index < reinstall_reason_score:
                    reinstall_reason_score = part_index
                    reinstall_reason = (
                        part, old_options, new_options, installed_path)
                elif reinstall_reason_score < 0:
                    self._log_reinstall_reason(logging.DEBUG,
                        part, old_options, new_options, installed_path)
Jim Fulton's avatar
Jim Fulton committed
937

938 939
            self._uninstall_part(part, installed_part_options)
            installed_parts = [p for p in installed_parts if p != part]
940 941
            installed_part_options['buildout']['parts'] = (
                ' '.join(installed_parts))
942 943
        if reinstall_reason:
            self._log_reinstall_reason(logging.INFO, *reinstall_reason)
944 945 946 947

        # Check for unused buildout options:
        _check_for_unused_options_in_section(self, 'buildout')

948

949
        # install new parts
950
        all_installed_paths = {}
951 952 953 954 955
        for part in install_parts:
            signature = self[part].pop('__buildout_signature__')
            saved_options = self[part].copy()
            recipe = self[part].recipe
            if part in installed_parts: # update
956
                __doing__ = 'Updating %s.', part
957
                self._logger.info(*__doing__)
958 959
                if self.dry_run:
                    continue
960
                old_options = installed_part_options[part]
961
                installed_files = old_options['__buildout_installed__']
962 963 964 965 966 967 968 969 970 971 972

                try:
                    update = recipe.update
                except AttributeError:
                    update = recipe.install
                    self._logger.warning(
                        "The recipe for %s doesn't define an update "
                        "method. Using its install method.",
                        part)

                try:
973 974
                    updated_files = self[part]._call(update)
                except Exception:
975
                    installed_parts.remove(part)
976 977 978
                    self._uninstall(installed_files)
                    installed_part_options['buildout']['parts'] = (
                        ' '.join(installed_parts))
979 980
                    raise

981 982
                installed_files = set(installed_files.split('\n')) \
                    if installed_files else set()
983 984 985
                if updated_files:
                    (installed_files.add if isinstance(updated_files, str) else
                     installed_files.update)(updated_files)
986 987

            else: # install
988
                __doing__ = 'Installing %s.', part
989
                self._logger.info(*__doing__)
990 991
                if self.dry_run:
                    continue
992
                installed_files = self[part]._call(recipe.install)
993 994 995 996 997
                if installed_files is None:
                    self._logger.warning(
                        "The %s install returned None.  A path or "
                        "iterable os paths should be returned.",
                        part)
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
                elif installed_files:
                    installed_files = ({installed_files}
                        if isinstance(installed_files, str) else
                        set(installed_files))

            if installed_files:
                conflicts = installed_files.intersection(
                    all_installed_paths)
                if conflicts:
                    self._error(
                        "The following paths are already"
                        " installed by other sections: %r",
                        {x: all_installed_paths[x] for x in conflicts})
                all_installed_paths.update(dict.fromkeys(installed_files, part))
                installed_files = '\n'.join(sorted(installed_files))
            else:
                installed_files = ''
1015

1016
            saved_options['__buildout_installed__'] = installed_files
1017
            saved_options['__buildout_signature__'] = signature
1018
            installed_part_options[part] = saved_options
1019

1020 1021
            if part not in installed_parts:
                installed_parts.append(part)
1022
                installed_part_options['buildout']['parts'] = (
1023
                    ' '.join(installed_parts))
1024
                _check_for_unused_options_in_section(self, part)
1025

1026 1027 1028
            if self._log_level < logging.INFO:
                self._save_installed_options()

1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
    def _log_reinstall_reason(self, level, part,
                              old_options, new_options, missing):
        log = self._logger.log
        if missing:
            log(level, "Part %s, missing path: %s", part, missing)
            return
        for k in old_options:
            if k not in new_options:
                log(level, "Part %s, dropped option %s.", part, k)
            elif old_options[k] != new_options[k]:
                log(level, "Part %s, option %s changed: %r != %r",
                           part, k, new_options[k], old_options[k])
        for k in new_options:
            if k not in old_options:
                log(level, "Part %s, new option %s.", part, k)

1045
    def _uninstall_part(self, part, installed_part_options):
pombredanne's avatar
pombredanne committed
1046
        # uninstall part
1047
        __doing__ = 'Uninstalling %s.', part
1048
        self._logger.info(*__doing__)
1049 1050
        if self.dry_run:
            return
1051

1052
        # run uninstall recipe
1053 1054 1055 1056
        recipe, entry = _recipe(installed_part_options[part])
        try:
            uninstaller = _install_and_load(
                recipe, 'zc.buildout.uninstall', entry, self)
1057
            self._logger.info('Running uninstall recipe.')
1058
            uninstaller(part, installed_part_options[part])
1059
        except (ImportError, pkg_resources.DistributionNotFound):
1060 1061 1062 1063 1064 1065
            pass

        # remove created files and directories
        self._uninstall(
            installed_part_options[part]['__buildout_installed__'])

1066
    def _setup_directories(self):
1067
        __doing__ = 'Setting up buildout directories'
1068 1069

        # Create buildout directories
1070
        for name in ('bin', 'parts', 'develop-eggs'):
1071 1072
            d = self['buildout'][name+'-directory']
            if not os.path.exists(d):
1073
                self._logger.info('Creating directory %r.', d)
1074 1075
                os.mkdir(d)

1076 1077 1078
    def _develop(self):
        """Install sources by running setup.py develop on them
        """
1079 1080
        __doing__ = 'Processing directories listed in the develop option'

1081
        develop = self['buildout'].get('develop')
1082 1083 1084 1085 1086 1087
        if not develop:
            return ''

        dest = self['buildout']['develop-eggs-directory']
        old_files = os.listdir(dest)

1088 1089
        env = dict(os.environ,
                   PYTHONPATH=zc.buildout.easy_install.setuptools_pythonpath)
1090 1091
        here = os.getcwd()
        try:
1092 1093
            try:
                for setup in develop.split():
1094
                    setup = self._buildout_path(setup)
1095 1096
                    files = glob.glob(setup)
                    if not files:
1097 1098
                        self._logger.warning("Couldn't develop %r (not found)",
                                             setup)
Shane Hathaway's avatar
Shane Hathaway committed
1099 1100
                    else:
                        files.sort()
1101 1102 1103 1104
                    for setup in files:
                        self._logger.info("Develop: %r", setup)
                        __doing__ = 'Processing develop directory %r.', setup
                        zc.buildout.easy_install.develop(setup, dest)
1105 1106 1107 1108 1109 1110 1111 1112 1113
            except:
                # if we had an error, we need to roll back changes, by
                # removing any files we created.
                self._sanity_check_develop_eggs_files(dest, old_files)
                self._uninstall('\n'.join(
                    [os.path.join(dest, f)
                     for f in os.listdir(dest)
                     if f not in old_files
                     ]))
1114
                raise
1115

1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
            else:
                self._sanity_check_develop_eggs_files(dest, old_files)
                return '\n'.join([os.path.join(dest, f)
                                  for f in os.listdir(dest)
                                  if f not in old_files
                                  ])

        finally:
            os.chdir(here)


    def _sanity_check_develop_eggs_files(self, dest, old_files):
        for f in os.listdir(dest):
            if f in old_files:
                continue
            if not (os.path.isfile(os.path.join(dest, f))
                    and f.endswith('.egg-link')):
                self._logger.warning(
1134
                    "Unexpected entry, %r, in develop-eggs directory.", f)
1135

1136
    _compute_part_signatures = None # BBB: monkey-patched by slapos.rebootstrap
1137 1138

    def _read_installed_part_options(self):
Jim Fulton's avatar
Jim Fulton committed
1139 1140
        old = self['buildout']['installed']
        if old and os.path.isfile(old):
1141 1142 1143
            fp = open(old)
            sections = zc.buildout.configparser.parse(fp, old)
            fp.close()
1144
            result = {}
1145 1146
            for section, options in sections.items():
                for option, value in options.items():
1147 1148 1149
                    if '%(' in value:
                        for k, v in _spacey_defaults:
                            value = value.replace(k, v)
1150
                        options[option] = value
Jim Fulton's avatar
Jim Fulton committed
1151
                result[section] = self.Options(self, section, options)
1152

1153
            return result
1154
        else:
1155
            return {'buildout': self.Options(self, 'buildout', {'parts': ''})}
1156 1157

    def _uninstall(self, installed):
Jim Fulton's avatar
Jim Fulton committed
1158 1159 1160
        for f in installed.split('\n'):
            if not f:
                continue
1161
            f = self._buildout_path(f)
1162
            if os.path.isdir(f):
1163
                rmtree(f)
1164
            elif os.path.isfile(f):
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
                try:
                    os.remove(f)
                except OSError:
                    if not (
                        sys.platform == 'win32' and
                        (realpath(os.path.join(os.path.dirname(sys.argv[0]),
                                               'buildout.exe'))
                         ==
                         realpath(f)
                         )
pombredanne's avatar
pombredanne committed
1175
                        # Sigh. This is the executable used to run the buildout
1176 1177
                        # and, of course, it's in use. Leave it.
                        ):
1178 1179
                        raise

1180 1181
    def _install(self, part):
        options = self[part]
1182
        recipe, entry = _recipe(options)
1183 1184 1185 1186 1187
        recipe_class = pkg_resources.load_entry_point(
            recipe, 'zc.buildout', entry)
        installed = recipe_class(self, part, options).install()
        if installed is None:
            installed = []
1188
        elif isinstance(installed, str):
1189
            installed = [installed]
1190
        base = self._buildout_path('')
1191 1192 1193 1194
        installed = [d.startswith(base) and d[len(base):] or d
                     for d in installed]
        return ' '.join(installed)

1195

1196
    def _save_installed_options(self):
1197 1198
        if self.dry_run:
            return
1199 1200
        installed_path = self['buildout']['installed']
        if not installed_path:
Jim Fulton's avatar
Jim Fulton committed
1201
            return
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
        installed_part_options = self.installed_part_options
        buildout = installed_part_options['buildout']
        installed_parts = buildout['parts']
        if installed_parts or buildout['installed_develop_eggs']:
            new = StringIO()
            _save_options('buildout', buildout, new)
            for part in installed_parts.split():
                new.write('\n')
                _save_options(part, installed_part_options[part], new)
            new = new.getvalue()
            try:
                with open(installed_path) as f:
                    save = f.read(1+len(new)) != new
            except IOError as e:
                if e.errno != errno.ENOENT:
                    raise
                save = True
            if save:
                installed_tmp = installed_path + ".tmp"
                try:
                    with open(installed_tmp, "w") as f:
                        f.write(new)
                        f.flush()
                        os.fsync(f.fileno())
                    os.rename(installed_tmp, installed_path)
                finally:
                    _remove_ignore_missing(installed_tmp)
        else:
            _remove_ignore_missing(installed_path)
Jim Fulton's avatar
Jim Fulton committed
1231

1232 1233
    def _error(self, message, *args):
        raise zc.buildout.UserError(message % args)
Jim Fulton's avatar
Jim Fulton committed
1234

1235 1236
    def _setup_socket_timeout(self):
        timeout = self['buildout']['socket-timeout']
1237
        if timeout != '':
1238 1239 1240
            try:
                timeout = int(timeout)
                import socket
Jim Fulton's avatar
Jim Fulton committed
1241 1242
                self._logger.info(
                    'Setting socket time out to %d seconds.', timeout)
1243 1244 1245 1246 1247 1248
                socket.setdefaulttimeout(timeout)
            except ValueError:
                self._logger.warning("Default socket timeout is used !\n"
                    "Value in configuration is not numeric: [%s].\n",
                    timeout)

Jim Fulton's avatar
Jim Fulton committed
1249 1250
    def _setup_logging(self):
        root_logger = logging.getLogger()
1251
        self._logger = logging.getLogger('zc.buildout')
Jim Fulton's avatar
Jim Fulton committed
1252
        handler = logging.StreamHandler(sys.stdout)
1253 1254 1255 1256 1257 1258 1259 1260 1261
        log_format = self['buildout']['log-format']
        if not log_format:
            # No format specified. Use different formatter for buildout
            # and other modules, showing logger name except for buildout
            log_format = '%(name)s: %(message)s'
            buildout_handler = logging.StreamHandler(sys.stdout)
            buildout_handler.setFormatter(logging.Formatter('%(message)s'))
            self._logger.propagate = False
            self._logger.addHandler(buildout_handler)
1262

1263
        handler.setFormatter(logging.Formatter(log_format))
Jim Fulton's avatar
Jim Fulton committed
1264
        root_logger.addHandler(handler)
1265

Jim Fulton's avatar
Jim Fulton committed
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
        level = self['buildout']['log-level']
        if level in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
            level = getattr(logging, level)
        else:
            try:
                level = int(level)
            except ValueError:
                self._error("Invalid logging level %s", level)
        verbosity = self['buildout'].get('verbosity', 0)
        try:
            verbosity = int(verbosity)
        except ValueError:
            self._error("Invalid verbosity %s", verbosity)
1279 1280 1281

        level -= verbosity
        root_logger.setLevel(level)
1282
        self._log_level = level
1283

1284
    def _maybe_upgrade(self):
1285
        # See if buildout or setuptools need to be upgraded.
1286
        # If they do, do the upgrade and restart the buildout process.
1287
        __doing__ = 'Checking for upgrades.'
1288

1289
        if 'BUILDOUT_RESTART_AFTER_UPGRADE' in os.environ:
1290 1291
            return

Jim Fulton's avatar
Jim Fulton committed
1292 1293
        if not self.newest:
            return
1294

1295
        ws = zc.buildout.easy_install.install(
1296
            ('zc.buildout', 'setuptools', 'pip', 'wheel'),
1297 1298 1299
            self['buildout']['eggs-directory'],
            links = self['buildout'].get('find-links', '').split(),
            index = self['buildout'].get('index'),
Tarek Ziad's avatar
Tarek Ziad committed
1300 1301
            path = [self['buildout']['develop-eggs-directory']],
            allow_hosts = self._allow_hosts
1302 1303 1304
            )

        upgraded = []
1305

1306
        for project in 'zc.buildout', 'setuptools', 'pip', 'wheel':
1307
            req = pkg_resources.Requirement.parse(project)
1308 1309 1310 1311
            dist = ws.find(req)
            importlib.import_module(project)
            if not inspect.getfile(sys.modules[project]).startswith(dist.location):
                upgraded.append(dist)
1312 1313 1314

        if not upgraded:
            return
Jim Fulton's avatar
Jim Fulton committed
1315

1316
        __doing__ = 'Upgrading.'
1317

1318 1319 1320 1321 1322 1323 1324 1325
        should_run = realpath(
            os.path.join(os.path.abspath(self['buildout']['bin-directory']),
                         'buildout')
            )
        if sys.platform == 'win32':
            should_run += '-script.py'

        if (realpath(os.path.abspath(sys.argv[0])) != should_run):
1326 1327
            self._logger.debug("Running %r.", realpath(sys.argv[0]))
            self._logger.debug("Local buildout is %r.", should_run)
1328 1329
            self._logger.warning("Not upgrading because not running a local "
                                 "buildout command.")
1330 1331
            return

1332
        self._logger.info("Upgraded:\n  %s;\nRestarting.",
1333
                          ",\n  ".join([("%s version %s"
1334 1335 1336 1337 1338 1339
                                       % (dist.project_name, dist.version)
                                       )
                                      for dist in upgraded
                                      ]
                                     ),
                          )
1340

1341 1342
        # the new dist is different, so we've upgraded.
        # Update the scripts and return True
1343
        options = self['buildout']
1344 1345 1346 1347 1348 1349 1350
        eggs_dir = options['eggs-directory']
        develop_eggs_dir = options['develop-eggs-directory']
        ws = zc.buildout.easy_install.sort_working_set(
                ws,
                eggs_dir=eggs_dir,
                develop_eggs_dir=develop_eggs_dir
                )
1351 1352
        zc.buildout.easy_install.scripts(
            ['zc.buildout'], ws, sys.executable,
1353
            options['bin-directory'],
1354 1355 1356 1357
            relative_paths = (
                bool_option(options, 'relative-paths', False)
                and options['directory']
                or ''),
1358 1359 1360
            )

        # Restart
1361
        args = sys.argv[:]
1362 1363
        if not __debug__:
            args.insert(0, '-O')
1364
        args.insert(0, sys.executable)
1365 1366
        env=dict(os.environ, BUILDOUT_RESTART_AFTER_UPGRADE='1')
        sys.exit(subprocess.call(args, env=env))
1367

1368
    def _load_extensions(self):
1369
        __doing__ = 'Loading extensions.'
1370
        specs = self['buildout'].get('extensions', '').split()
1371 1372 1373
        for superceded_extension in ['buildout-versions',
                                     'buildout.dumppickedversions']:
            if superceded_extension in specs:
1374 1375
                msg = ("Buildout now includes 'buildout-versions' (and part "
                       "of the older 'buildout.dumppickedversions').\n"
1376
                       "Remove the extension from your configuration and "
1377 1378 1379
                       "look at the 'show-picked-versions' option in "
                       "buildout's documentation.")
                raise zc.buildout.UserError(msg)
1380
        if specs:
Jim Fulton's avatar
Jim Fulton committed
1381
            path = [self['buildout']['develop-eggs-directory']]
Jim Fulton's avatar
Jim Fulton committed
1382
            if self.offline:
1383
                dest = None
Jim Fulton's avatar
Jim Fulton committed
1384
                path.append(self['buildout']['eggs-directory'])
1385 1386
            else:
                dest = self['buildout']['eggs-directory']
Jim Fulton's avatar
Jim Fulton committed
1387

1388
            zc.buildout.easy_install.install(
Jim Fulton's avatar
Jim Fulton committed
1389
                specs, dest, path=path,
1390
                working_set=pkg_resources.working_set,
1391 1392
                links = self['buildout'].get('find-links', '').split(),
                index = self['buildout'].get('index'),
Tarek Ziad's avatar
Tarek Ziad committed
1393
                newest=self.newest, allow_hosts=self._allow_hosts)
1394 1395 1396 1397 1398

            # Clear cache because extensions might now let us read pages we
            # couldn't read before.
            zc.buildout.easy_install.clear_index_cache()

1399 1400
            for ep in pkg_resources.iter_entry_points('zc.buildout.extension'):
                ep.load()(self)
1401

1402 1403 1404 1405 1406 1407 1408 1409
    def _unload_extensions(self):
        __doing__ = 'Unloading extensions.'
        specs = self['buildout'].get('extensions', '').split()
        if specs:
            for ep in pkg_resources.iter_entry_points(
                'zc.buildout.unloadextension'):
                ep.load()(self)

1410
    def _print_picked_versions(self):
1411 1412 1413
        picked_versions, required_by = (zc.buildout.easy_install
                                        .get_picked_versions())
        if not picked_versions:
1414 1415 1416
            # Don't print empty output.
            return

1417
        output = _format_picked_versions(picked_versions, required_by)
1418

1419 1420 1421 1422 1423
        if self.show_picked_versions:
            print_("Versions had to be automatically picked.")
            print_("The following part definition lists the versions picked:")
            print_('\n'.join(output))

1424
        if self.update_versions_file:
1425
            # Write to the versions file.
1426 1427 1428
            if os.path.exists(self.update_versions_file):
                output[:1] = [
                    '',
1429 1430
                    '# Added by buildout at %s' % datetime.datetime.now()
                ]
1431 1432
            output.append('')
            f = open(self.update_versions_file, 'a')
1433
            f.write(('\n'.join(output)))
1434
            f.close()
1435
            print_("Picked versions have been written to " +
1436
                   self.update_versions_file)
1437

1438
    @command
1439
    def setup(self, args):
1440 1441
        if not args:
            raise zc.buildout.UserError(
Jim Fulton's avatar
Jim Fulton committed
1442
                "The setup command requires the path to a setup script or \n"
1443
                "directory containing a setup script, and its arguments."
1444
                )
1445 1446 1447 1448
        setup = args.pop(0)
        if os.path.isdir(setup):
            setup = os.path.join(setup, 'setup.py')

1449
        self._logger.info("Running setup script %r.", setup)
1450 1451 1452 1453
        setup = os.path.abspath(setup)

        fd, tsetup = tempfile.mkstemp()
        try:
1454
            os.write(fd, (zc.buildout.easy_install.runsetup_template % dict(
1455 1456
                setupdir=os.path.dirname(setup),
                setup=setup,
1457
                path_list=[],
1458
                __file__ = setup,
1459
                )).encode())
Jim Fulton's avatar
Jim Fulton committed
1460 1461
            args = [sys.executable, tsetup] + args
            zc.buildout.easy_install.call_subprocess(args)
1462 1463 1464
        finally:
            os.close(fd)
            os.remove(tsetup)
1465

1466 1467 1468
    @command
    def runsetup(self, args):
        self.setup(args)
1469

1470
    @command
Godefroid Chapelle's avatar
Godefroid Chapelle committed
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
    def query(self, args=None):
        if args is None or len(args) != 1:
            _error('The query command requires a single argument.')
        option = args[0]
        option = option.split(':')
        if len(option) == 1:
            option = 'buildout', option[0]
        elif len(option) != 2:
            _error('Invalid option:', args[0])
        section, option = option
        verbose = self['buildout'].get('verbosity', 0) != 0
        if verbose:
            print_('${%s:%s}' % (section, option))
        try:
            print_(self._raw[section][option])
        except KeyError:
            if section in self._raw:
                _error('Key not found:', option)
            else:
                _error('Section not found:', section)

1492
    @command
Jim Fulton's avatar
Jim Fulton committed
1493
    def annotate(self, args=None):
Godefroid Chapelle's avatar
Godefroid Chapelle committed
1494
        verbose = self['buildout'].get('verbosity', 0) != 0
1495
        section = None
Godefroid Chapelle's avatar
Godefroid Chapelle committed
1496 1497 1498 1499 1500
        if args is None:
            sections = []
        else:
            sections = args
        _print_annotate(self._annotated, verbose, sections, self._buildout_dir)
1501

Jim Fulton's avatar
Jim Fulton committed
1502
    def print_options(self, base_path=None):
Jim Fulton's avatar
Jim Fulton committed
1503 1504 1505 1506 1507 1508 1509 1510 1511
        for section in sorted(self._data):
            if section == 'buildout' or section == self['buildout']['versions']:
                continue
            print_('['+section+']')
            for k, v in sorted(self._data[section].items()):
                if '\n' in v:
                    v = '\n  ' + v.replace('\n', '\n  ')
                else:
                    v = ' '+v
Jim Fulton's avatar
Jim Fulton committed
1512 1513 1514

                if base_path:
                    v = v.replace(os.getcwd(), base_path)
Jim Fulton's avatar
Jim Fulton committed
1515 1516
                print_("%s =%s" % (k, v))

1517 1518
    def initialize(self, options, reqs, entry):
        recipe_class = _install_and_load(reqs, 'zc.buildout', entry, self)
1519
        try:
1520
            sig = self._signature_cache[reqs]
1521
        except KeyError:
1522 1523 1524 1525 1526 1527 1528 1529 1530 1531
            req = pkg_resources.Requirement.parse(reqs)
            sig = self._signature_cache[reqs] = sorted(set(
                _dists_sig(pkg_resources.working_set.resolve([req]))))
        self._initializing.append((options, sig))
        try:
            recipe = recipe_class(self, options.name, options)
            options['__buildout_signature__']
        finally:
            del self._initializing[-1]
        return recipe
1532

1533 1534
    def __getitem__(self, section):
        __doing__ = 'Getting section %s.', section
1535
        try:
1536
            options = self._data[section]
1537
        except KeyError:
1538 1539 1540 1541
            try:
                data = self._raw[section]
            except KeyError:
                raise MissingSection(section)
1542 1543 1544
            e = data.get('__unsupported_conditional_expression__')
            if e:
                raise e
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556
            options = self.Options(self, section, data)
            self._data[section] = options
            options._initialize()

        if self._initializing:
            caller = self._initializing[-1][0]
            if 'buildout' != section and not (
                  section in caller.depends or
                  # Do not only check the caller,
                  # because of circular dependencies during substitutions.
                  section in (x[0].name for x in self._initializing)):
                caller.depends.add(section)
1557
        return options
1558

Jim Fulton's avatar
Jim Fulton committed
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568
    def __setitem__(self, name, data):
        if name in self._raw:
            raise KeyError("Section already exists", name)
        self._raw[name] = dict((k, str(v)) for (k, v) in data.items())
        self[name] # Add to parts

    def parse(self, data):
        import textwrap

        sections = zc.buildout.configparser.parse(
1569
            StringIO(textwrap.dedent(data)), '', _default_globals)
Jim Fulton's avatar
Jim Fulton committed
1570 1571 1572 1573 1574 1575 1576 1577
        for name in sections:
            if name in self._raw:
                raise KeyError("Section already exists", name)
            self._raw[name] = dict((k, str(v))
                                   for (k, v) in sections[name].items())

        for name in sections:
            self[name] # Add to parts
1578 1579 1580 1581 1582

    def __delitem__(self, key):
        raise NotImplementedError('__delitem__')

    def keys(self):
1583
        return list(self._raw.keys())
1584 1585 1586 1587

    def __iter__(self):
        return iter(self._raw)

1588 1589 1590
    def __len__(self):
        return len(self._raw)

1591
_install_and_load_cache = {}
Jim Fulton's avatar
Jim Fulton committed
1592
def _install_and_load(spec, group, entry, buildout):
1593
    __doing__ = 'Loading recipe %r.', spec
1594 1595 1596 1597 1598 1599 1600

    key = spec, group, entry
    try:
        return _install_and_load_cache[key]
    except KeyError:
        pass

Jim Fulton's avatar
Jim Fulton committed
1601 1602 1603 1604 1605
    try:
        req = pkg_resources.Requirement.parse(spec)

        buildout_options = buildout['buildout']
        if pkg_resources.working_set.find(req) is None:
1606
            __doing__ = 'Installing recipe %s.', spec
Jim Fulton's avatar
Jim Fulton committed
1607
            if buildout.offline:
Jim Fulton's avatar
Jim Fulton committed
1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
                dest = None
                path = [buildout_options['develop-eggs-directory'],
                        buildout_options['eggs-directory'],
                        ]
            else:
                dest = buildout_options['eggs-directory']
                path = [buildout_options['develop-eggs-directory']]

            zc.buildout.easy_install.install(
                [spec], dest,
                links=buildout._links,
                index=buildout_options.get('index'),
                path=path,
                working_set=pkg_resources.working_set,
Jim Fulton's avatar
Jim Fulton committed
1622
                newest=buildout.newest,
Tarek Ziad's avatar
Tarek Ziad committed
1623
                allow_hosts=buildout._allow_hosts
Jim Fulton's avatar
Jim Fulton committed
1624 1625
                )

1626
        __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
1627
        result = _install_and_load_cache[key] = pkg_resources.load_entry_point(
Jim Fulton's avatar
Jim Fulton committed
1628
            req.project_name, group, entry)
1629
        return result
Jim Fulton's avatar
Jim Fulton committed
1630

1631 1632
    except Exception:
        v = sys.exc_info()[1]
Jim Fulton's avatar
Jim Fulton committed
1633 1634
        buildout._logger.log(
            1,
Kian-Meng Ang's avatar
Kian-Meng Ang committed
1635
            "Couldn't load %s entry point %s\nfrom %s:\n%s.",
Jim Fulton's avatar
Jim Fulton committed
1636 1637 1638
            group, entry, spec, v)
        raise

1639
class Options(DictMixin):
1640 1641 1642 1643 1644

    def __init__(self, buildout, section, data):
        self.buildout = buildout
        self.name = section
        self._raw = data
1645
        self._cooked = {}
1646
        self._data = {}
1647
        self.depends = set()
1648 1649

    def _initialize(self):
1650
        name = self.name
1651
        __doing__ = 'Initializing section %s.', name
1652

Jim Fulton's avatar
Jim Fulton committed
1653 1654 1655
        if '<' in self._raw:
            self._raw = self._do_extend_raw(name, self._raw, [])

1656 1657
        default = self.buildout._default_requirement

1658
        # force substitutions
1659
        for k, v in sorted(self._raw.items()):
1660 1661
            if '${' in v:
                self._dosub(k, v)
1662

Jim Fulton's avatar
Jim Fulton committed
1663
        if name == 'buildout':
1664
            return # buildout section can never be a part
1665

1666 1667 1668 1669
        for dname in self.get('<part-dependencies>', '').split():
            # force use of dependencies in buildout:
            self.buildout[dname]

Jim Fulton's avatar
Jim Fulton committed
1670
        if self.get('recipe'):
1671 1672
            if default:
                self.depends.add(default)
1673
            self.recipe = self.buildout.initialize(self, *_recipe(self._data))
Jim Fulton's avatar
Jim Fulton committed
1674
            self.buildout._parts.append(name)
1675

1676
        m = md5()
1677 1678 1679
        # _profile_base_location_ is ignored in signatures, so that two sections
        # at different URLs can have same signature
        _profile_base_location_ = self.get('_profile_base_location_')
1680 1681
        # access values through .get() instead of .items() to detect unused keys
        for key in sorted(self.keys()):
1682 1683
            if key == '_profile_base_location_':
                continue
1684
            value = self._data.get(key, self._cooked.get(key, self._raw.get(key)))
1685 1686
            if _profile_base_location_:
                value = value.replace(_profile_base_location_, '${:_profile_base_location_}')
1687 1688
            m.update(('%r\0%r\0' % (key, value)).encode())
        self.items_signature = '%s:%s' % (name, m.hexdigest())
1689

Jim Fulton's avatar
Jim Fulton committed
1690 1691 1692 1693 1694 1695 1696
    def _do_extend_raw(self, name, data, doing):
        if name == 'buildout':
            return data
        if name in doing:
            raise zc.buildout.UserError("Infinite extending loop %r" % name)
        doing.append(name)
        try:
1697
            to_do = data.get('<', None)
Jim Fulton's avatar
Jim Fulton committed
1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
            if to_do is None:
                return data
            __doing__ = 'Loading input sections for %r', name

            result = {}
            for iname in to_do.split('\n'):
                iname = iname.strip()
                if not iname:
                    continue
                raw = self.buildout._raw.get(iname)
                if raw is None:
                    raise zc.buildout.UserError("No section named %r" % iname)
                result.update(self._do_extend_raw(iname, raw, doing))

1712
            _annotate_section(result, "")
1713
            data = _annotate_section(copy.deepcopy(data), "")
1714
            _update_section(result, data)
1715
            _unannotate_section(result)
Patrick Strawderman's avatar
Patrick Strawderman committed
1716
            result.pop('<', None)
Jim Fulton's avatar
Jim Fulton committed
1717 1718 1719 1720
            return result
        finally:
            assert doing.pop() == name

1721
    def _dosub(self, option, v):
1722
        __doing__ = 'Getting option %s:%s.', self.name, option
1723
        seen = [(self.name, option)]
1724 1725
        v = '$$'.join([self._sub(s, seen, last=False)
                       for s in v.split('$$')])
1726 1727
        self._cooked[option] = v

1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
    def get(self, *args, **kw):
      v = self._get(*args, **kw)
      if hasattr(v, 'startswith') and v.startswith(SERIALISED_VALUE_MAGIC):
        v = loads(v)
      return v

    def _get(self, option, default=None, seen=None, last=True):
        # TODO: raise instead of handling a default parameter,
        #       so that get() never tries to deserialize a default value
        #       (and then: move deserialization to __getitem__
        #                  and make get() use __getitem__)
1739
        try:
1740 1741 1742 1743
            if last:
                return self._data[option].replace('$${', '${')
            else:
                return self._data[option]
1744 1745 1746
        except KeyError:
            pass

1747
        v = self._cooked.get(option)
1748
        if v is None:
1749 1750
            v = self._raw.get(option)
            if v is None:
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
                if option == '__buildout_signature__':
                    buildout = self.buildout
                    options, sig = buildout._initializing[-1]
                    if options is self:
                        self.depends = frozenset(self.depends)
                        v = self._data[option] = ' '.join(sig + [
                            buildout[dependency].items_signature
                            for dependency in sorted(self.depends)])
                        return v
                    raise zc.buildout.UserError("premature access to " + option)
1761
                return default
1762

1763
        __doing__ = 'Getting option %s:%s.', self.name, option
1764

1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
        if '${' in v:
            key = self.name, option
            if seen is None:
                seen = [key]
            elif key in seen:
                raise zc.buildout.UserError(
                    "Circular reference in substitutions.\n"
                    )
            else:
                seen.append(key)
1775 1776
            v = '$$'.join([self._sub(s, seen, last=False)
                           for s in v.split('$$')])
1777
            seen.pop()
1778

1779
        self._data[option] = v
1780 1781 1782 1783
        if last:
            return v.replace('$${', '${')
        else:
            return v
1784 1785 1786

    _template_split = re.compile('([$]{[^}]*})').split
    _simple = re.compile('[-a-zA-Z0-9 ._]+$').match
1787
    _valid = re.compile(r'\${[-a-zA-Z0-9 ._]*:[-a-zA-Z0-9 ._]+}$').match
1788
    def _sub(self, template, seen, last=True):
1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811
        value = self._template_split(template)
        subs = []
        for ref in value[1::2]:
            s = tuple(ref[2:-1].split(':'))
            if not self._valid(ref):
                if len(s) < 2:
                    raise zc.buildout.UserError("The substitution, %s,\n"
                                                "doesn't contain a colon."
                                                % ref)
                if len(s) > 2:
                    raise zc.buildout.UserError("The substitution, %s,\n"
                                                "has too many colons."
                                                % ref)
                if not self._simple(s[0]):
                    raise zc.buildout.UserError(
                        "The section name in substitution, %s,\n"
                        "has invalid characters."
                        % ref)
                if not self._simple(s[1]):
                    raise zc.buildout.UserError(
                        "The option name in substitution, %s,\n"
                        "has invalid characters."
                        % ref)
1812

1813 1814 1815
            section, option = s
            if not section:
                section = self.name
1816 1817 1818 1819 1820 1821 1822
                options = self
            else:
                self.buildout._initializing.append((self,))
                try:
                    options = self.buildout[section]
                finally:
                    del self.buildout._initializing[-1]
1823
            v = options._get(option, None, seen, last=last)
1824
            if v is None:
1825 1826 1827 1828 1829
                if option == '_buildout_section_name_':
                    v = self.name
                else:
                    raise MissingOption("Referenced option does not exist:",
                                        section, option)
1830 1831 1832 1833
            subs.append(v)
        subs.append('')

        return ''.join([''.join(v) for v in zip(value[::2], subs)])
1834

1835
    def __getitem__(self, key):
1836 1837
        v = self.get(key, _MARKER)
        if v is _MARKER:
1838
            raise MissingOption("Missing option: %s:%s" % (self.name, key))
1839 1840 1841
        return v

    def __setitem__(self, option, value):
1842 1843
        if not re.match(zc.buildout.configparser.option_name_re + '$', option):
            raise zc.buildout.UserError("Invalid option name %r" % (option, ))
1844
        if not isinstance(value, str):
1845
            value = dumps(value)
1846
        self._data[option] = value.replace('${', '$${')
1847 1848 1849 1850 1851 1852

    def __delitem__(self, key):
        if key in self._raw:
            del self._raw[key]
            if key in self._data:
                del self._data[key]
1853 1854
            if key in self._cooked:
                del self._cooked[key]
1855 1856 1857
        elif key in self._data:
            del self._data[key]
        else:
1858
            raise KeyError(key)
1859 1860 1861 1862 1863

    def keys(self):
        raw = self._raw
        return list(self._raw) + [k for k in self._data if k not in raw]

1864 1865 1866 1867 1868 1869
    def __iter__(self):
        return iter(self.keys())

    def __len__(self):
        return len(self.keys())

1870
    def copy(self):
1871
        result = copy.deepcopy(self._raw)
1872 1873
        result.update(self._cooked)
        result.update(self._data)
1874 1875 1876
        for key, value in result.items():
            if value.startswith(SERIALISED_VALUE_MAGIC):
                result[key] = loads(value)
1877
        return result
1878

1879
    def _call(self, f):
1880
        buildout_directory = self.buildout['buildout']['directory']
1881 1882 1883
        self._created = []
        try:
            try:
1884
                os.chdir(buildout_directory)
1885 1886 1887 1888
                return f()
            except:
                for p in self._created:
                    if os.path.isdir(p):
1889
                        rmtree(p)
1890 1891 1892
                    elif os.path.isfile(p):
                        os.remove(p)
                    else:
1893
                        self.buildout._logger.warning("Couldn't clean up %r.", p)
1894 1895 1896
                raise
        finally:
            self._created = None
1897
            os.chdir(buildout_directory)
1898 1899 1900 1901 1902 1903 1904 1905 1906 1907

    def created(self, *paths):
        try:
            self._created.extend(paths)
        except AttributeError:
            raise TypeError(
                "Attempt to register a created path while not installing",
                self.name)
        return self._created

1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
    def barrier(self):
        """Set self as a default requirement for not-yet processed parts

        This method must be called if this part may alter the processing
        of other parts in any way, like modifying environment variables.
        In other words, it sets an implicit dependency for these parts.
        """
        buildout = self.buildout
        if not buildout._initializing:
            raise zc.buildout.UserError(
                "Options.barrier() shall only be used during initialization")
        buildout._default_requirement = self.name

1921 1922 1923
    def __repr__(self):
        return repr(dict(self))

1924 1925 1926 1927 1928 1929
    def __eq__(self, other):
        try:
            return sorted(self.items()) == sorted(other.items())
        except Exception:
            return super(Options, self).__eq__(other)

Jim Fulton's avatar
Jim Fulton committed
1930 1931
Buildout.Options = Options

1932 1933 1934
_spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*'
                        '|'
                        '^[ \t\r\f\v]+'
1935 1936 1937 1938
                        '|'
                        '[ \t\r\f\v]+$'
                        )

1939 1940 1941 1942 1943 1944 1945 1946
_spacey_defaults = [
    ('%(__buildout_space__)s',   ' '),
    ('%(__buildout_space_n__)s', '\n'),
    ('%(__buildout_space_r__)s', '\r'),
    ('%(__buildout_space_f__)s', '\f'),
    ('%(__buildout_space_v__)s', '\v'),
    ]

1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960
def _quote_spacey_nl(match):
    match = match.group(0).split('\n', 1)
    result = '\n\t'.join(
        [(s
          .replace(' ', '%(__buildout_space__)s')
          .replace('\r', '%(__buildout_space_r__)s')
          .replace('\f', '%(__buildout_space_f__)s')
          .replace('\v', '%(__buildout_space_v__)s')
          .replace('\n', '%(__buildout_space_n__)s')
          )
         for s in match]
        )
    return result

1961
def _save_option(option, value, f):
1962 1963
    if not isinstance(value, str):
        value = dumps(value)
1964 1965 1966 1967 1968
    value = _spacey_nl.sub(_quote_spacey_nl, value)
    if value.startswith('\n\t'):
        value = '%(__buildout_space_n__)s' + value[2:]
    if value.endswith('\n\t'):
        value = value[:-2] + '%(__buildout_space_n__)s'
1969
    print_(option, '=', value, file=f)
1970

1971
def _save_options(section, options, f):
1972
    print_('[%s]' % section, file=f)
1973 1974 1975
    try:
        get_option = partial(options._get, last=False)
    except AttributeError:
1976 1977 1978
        get_option = options.get
    for option in sorted(options):
        _save_option(option, get_option(option), f)
1979

1980 1981 1982 1983 1984
def _default_globals():
    """Return a mapping of default and precomputed expressions.
    These default expressions are convenience defaults available when eveluating
    section headers expressions.
    NB: this is wrapped in a function so that the computing of these expressions
1985 1986 1987
    is lazy and done only if needed (ie if there is at least one section with
    an expression) because the computing of some of these expressions can be
    expensive.
1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016
    """
    # partially derived or inspired from its.py
    # Copyright (c) 2012, Kenneth Reitz All rights reserved.
    # Redistribution and use in source and binary forms, with or without modification,
    # are permitted provided that the following conditions are met:
    # Redistributions of source code must retain the above copyright notice, this list
    # of conditions and the following disclaimer. Redistributions in binary form must
    # reproduce the above copyright notice, this list of conditions and the following
    # disclaimer in the documentation and/or other materials provided with the
    # distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
    # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
    # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
    # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
    # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
    # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
    # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
    # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
    # OF SUCH DAMAGE.

    # default available modules, explicitly re-imported locally here on purpose
    import sys
    import os
    import platform
    import re

    globals_defs = {'sys': sys, 'os': os, 'platform': platform, 're': re,}

2017
    # major python major_python_versions as python2 and python3
2018
    major_python_versions = tuple(map(str, platform.python_version_tuple()))
2019 2020
    globals_defs.update({'python2': major_python_versions[0] == '2',
                         'python3': major_python_versions[0] == '3'})
2021

2022
    # minor python major_python_versions as python24, python25 ... python39
2023
    minor_python_versions = ('24', '25', '26', '27',
2024 2025
                             '30', '31', '32', '33', '34', '35', '36', '37', '38', '39',
                             '310', '311', '312', '313', '314', '315')
2026 2027
    for v in minor_python_versions:
        globals_defs['python' + v] = ''.join(major_python_versions[:2]) == v
2028 2029

    # interpreter type
2030 2031 2032 2033
    sys_version = sys.version.lower()
    pypy = 'pypy' in sys_version
    jython = 'java' in sys_version
    ironpython ='iron' in sys_version
2034 2035 2036 2037 2038 2039
    # assume CPython, if nothing else.
    cpython = not any((pypy, jython, ironpython,))
    globals_defs.update({'cpython': cpython,
                         'pypy': pypy,
                         'jython': jython,
                         'ironpython': ironpython})
2040

2041
    # operating system
2042 2043 2044 2045 2046 2047
    sys_platform = str(sys.platform).lower()
    globals_defs.update({'linux': 'linux' in sys_platform,
                         'windows': 'win32' in sys_platform,
                         'cygwin': 'cygwin' in sys_platform,
                         'solaris': 'sunos' in sys_platform,
                         'macosx': 'darwin' in sys_platform,
2048
                         'posix': 'posix' in os.name.lower()})
2049

2050 2051 2052 2053 2054 2055 2056 2057 2058 2059
    #bits and endianness
    import struct
    void_ptr_size = struct.calcsize('P') * 8
    globals_defs.update({'bits32': void_ptr_size == 32,
                         'bits64': void_ptr_size == 64,
                         'little_endian': sys.byteorder == 'little',
                         'big_endian': sys.byteorder == 'big'})

    return globals_defs

2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082

class _default_globals(dict):
    """
    Make sure parser context is computed at most once,
    even if several files are parsed.
    And compute some values only if accessed.

    If __getitem__ raises, _doing() calls .get('__doing__'),
    but that's not the only reason to subclass dict:
    CPython requests it (for performance reasons?). PyPy does not care.
    """
    # XXX: The following line is only to keep access to the overridden global.
    #      If pushed upstream, proper naming would avoid such hack.
    #      Meanwhile, the patch consists only in this drop-in class
    #      and that's easier to maintain.
    _default_globals = staticmethod(_default_globals)

    def __getitem__(self, key):
        cls = self.__class__
        try:
            context = self.context
        except AttributeError:
            context = self.context = cls._default_globals()
2083
            context['sys'] = _sysproxy(self)
2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102
        try:
            return context[key]
        except KeyError as e:
            try:
                value = getattr(self, key)
            except AttributeError:
                pass
            else:
                value = context[key] = value()
                return value
            raise e # BBB: On Python 3, a bare 'raise' is enough.

    def multiarch(self):
        args = os.getenv('CC') or 'gcc', '-dumpmachine'
        self['__doing__'] = '%r', args
        m = subprocess.check_output(args, universal_newlines=True).rstrip()
        del self['__doing__']
        return m

2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114
class _sysproxy(object): # BBB: alternate/temporary way to get multiarch value

    def __init__(self, default_globals):
        self.__default_globals = default_globals

    def __getattr__(self, name):
        if name == '_multiarch':
            default_globals = self.__default_globals
            setattr(sys, name, getattr(default_globals, name[1:])())
            default_globals.context['sys'] = sys
        return getattr(sys, name)

2115

2116
variable_template_split = re.compile('([$]{[^}]*})').split
2117

2118

2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159
class _extends(object):

    def __new__(cls, defaults, *args):
        self = super(_extends, cls).__new__(cls)
        self.seen = set()
        self.processing = []
        self.extends = [defaults]
        self._download_options = []
        self.collect(*args)
        return self.merge()

    def merge(self):
        result = {}
        for d in self.extends:
          _update(result, d)
        return result

    def __getattr__(self, attr):
        if attr == 'download_options':
            # Compute processed options
            result_so_far = self.merge()
            self.extends[:] = [result_so_far]
            value = copy.deepcopy(result_so_far.get('buildout')) or {}
            # Update with currently-being-processed options
            for options in reversed(self._download_options):
                _update_section(value, options)
            value = _unannotate_section(value)
            setattr(self, attr, value)
            return value
        return self.__getattribute__(attr)

    def collect(self, result, base, filename):
        options = result.get('buildout', {})
        extends = options.pop('extends', '')

        # Sanitize buildout options
        if 'extended-by' in options:
            raise zc.buildout.UserError(
                'No-longer supported "extended-by" option found in %s.' %
                filename)

2160 2161 2162 2163 2164 2165 2166
        # Find and expose _profile_base_location_
        for section in result.values():
            for value in section.values():
                if '${:_profile_base_location_}' in value:
                    section['_profile_base_location_'] = base
                    break

2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183
        _annotate(result, filename)

        # Collect extends and unprocessed download options
        self.processing.append(filename)
        self._download_options.append(options)
        for fextends in extends.split():
            self.open(base, fextends)
        self.extends.append(result)
        del self.processing[-1], self._download_options[-1]

    def open(self, base, filename):
        # Determine file location
        if _isurl(filename):
            download = True
        elif _isurl(base):
            download = True
            filename = urljoin(base + '/', filename)
2184
        else:
2185 2186 2187 2188 2189 2190 2191 2192 2193 2194
            download = False
            filename = os.path.realpath(
                os.path.join(base, os.path.expanduser(filename)))

        # Detect repetitions and loops
        if filename in self.seen:
            if filename in self.processing:
                raise zc.buildout.UserError("circular extends: %s" % filename)
            return
        self.seen.add(filename)
2195

2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225
        # Fetch file
        is_temp = False
        try:
            if download:
                download_options = self.download_options
                extends_cache = download_options.get('extends-cache')
                if extends_cache and variable_template_split(extends_cache)[1::2]:
                    raise ValueError(
                        "extends-cache '%s' may not contain ${section:variable} to expand."
                        % extends_cache
                    )
                downloaded_filename, is_temp = zc.buildout.download.Download(
                    download_options, cache=extends_cache,
                    fallback=bool_option(download_options, 'newest'),
                    hash_name=True)(filename)
                filename_for_logging = '%s (downloaded as %s)' % (
                    filename, downloaded_filename)
                base = filename[:filename.rfind('/')]
            else:
                downloaded_filename = filename_for_logging = filename
                base = os.path.dirname(filename)

            with open(downloaded_filename) as fp:
                result = zc.buildout.configparser.parse(
                    fp, filename_for_logging, _default_globals)
        finally:
            if is_temp:
                os.remove(downloaded_filename)

        return self.collect(result, base, filename)
2226

2227

2228
ignore_directories = '.svn', 'CVS', '__pycache__', '.git'
2229
_dir_hashes = {}
2230
def _dir_hash(dir):
2231 2232 2233
    dir_hash = _dir_hashes.get(dir, None)
    if dir_hash is not None:
        return dir_hash
2234
    hash = md5()
2235
    for (dirpath, dirnames, filenames) in os.walk(dir):
2236 2237 2238 2239
        dirnames[:] = sorted(n for n in dirnames if n not in ignore_directories)
        filenames[:] = sorted(f for f in filenames
                              if (not (f.endswith('pyc') or f.endswith('pyo'))
                                  and os.path.exists(os.path.join(dirpath, f)))
2240
                          )
2241 2242 2243 2244
        for_hash = ' '.join(dirnames + filenames)
        if isinstance(for_hash, text_type):
            for_hash = for_hash.encode()
        hash.update(for_hash)
2245
        for name in filenames:
2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262
            path = os.path.join(dirpath, name)
            if name == 'entry_points.txt':
                f = open(path)
                # Entry points aren't written in stable order. :(
                try:
                    sections = zc.buildout.configparser.parse(f, path)
                    data = repr([(sname, sorted(sections[sname].items()))
                                 for sname in sorted(sections)]).encode('utf-8')
                except Exception:
                    f.close()
                    f = open(path, 'rb')
                    data = f.read()
            else:
                f = open(path, 'rb')
                data = f.read()
            f.close()
            hash.update(data)
2263
    _dir_hashes[dir] = dir_hash = hash.hexdigest()
2264
    return dir_hash
2265

2266
def _dists_sig(dists):
2267
    seen = set()
2268
    result = []
2269
    for dist in sorted(dists):
2270 2271 2272
        if dist in seen:
            continue
        seen.add(dist)
2273
        location = dist.location
2274 2275
        if (dist.precedence == pkg_resources.DEVELOP_DIST
            and location != zc.buildout.easy_install.python_lib):
2276 2277
            result.append(dist.project_name + '-' + _dir_hash(location))
        else:
2278
            result.append(dist.project_name + '-' + dist.version)
2279 2280
    return result

2281 2282
def _update_section(s1, s2):
    # Base section 2 on section 1; key-value pairs in s2 override those in s1.
2283 2284 2285 2286
    # If there are += or -= operators in s2, process these to add or subtract
    # items (delimited by newlines) from the preexisting values.
    # Sort on key, then on + and - operators, so that KEY < KEY + < KEY -, to
    # process them in this order if several are defined in the same section.
2287
    # Section s1 is modified in place.
2288 2289
    keysort = lambda x: (x[0].rstrip(' +'), x[0].endswith('+'))
    for k, v in sorted(s2.items(), key=keysort):
2290 2291
        if k.endswith('+'):
            key = k.rstrip(' +')
2292 2293 2294
            value = s1.get(key, SectionKey("", "IMPLICIT_VALUE"))
            value.addToValue(v.value, v.source)
            s1[key] = value
2295 2296
        elif k.endswith('-'):
            key = k.rstrip(' -')
2297 2298 2299
            value = s1.get(key, SectionKey("", "IMPLICIT_VALUE"))
            value.removeFromValue(v.value, v.source)
            s1[key] = value
2300
        else:
2301 2302 2303 2304 2305 2306
            if k in s1:
                v1 = s1[k]
                v1.overrideValue(v)
            else:
                s1[k] = v
    return s1
2307

2308
def _update(d1, d2):
2309 2310
    for section in d2:
        if section in d1:
2311
            _update_section(d1[section], d2[section])
2312
        else:
2313
            # XXX: In order to process += (and -=) correctly when
2314
            # <key> = <value> and <key> += <value> are in the same section
2315 2316 2317 2318 2319 2320 2321 2322 2323
            # _update_section should be called even when section is not in d1
            # Hack: When <= is used in the section, _update_section will be
            # called later anyway, so we can avoid calling it now which will
            # enable brittle and partial support for += (or -=) with keys that
            # come from the <= sections.
            # TODO: Either implement += and -= support with <= fully or call
            # _update_section systematically and give up <= compatibility.
            s2 = d2[section]
            d1[section] = s2 if '<' in s2 else _update_section({}, s2)
2324 2325
    return d1

2326 2327 2328 2329 2330 2331 2332 2333 2334
def _recipe(options):
    recipe = options['recipe']
    if ':' in recipe:
        recipe, entry = recipe.split(':')
    else:
        entry = 'default'

    return recipe, entry

2335 2336 2337 2338 2339 2340 2341 2342 2343
def _doing():
    _, v, tb = sys.exc_info()
    message = str(v)
    doing = []
    while tb is not None:
        d = tb.tb_frame.f_locals.get('__doing__')
        if d:
            doing.append(d)
        tb = tb.tb_next
2344

2345 2346 2347 2348 2349 2350 2351
    if doing:
        sys.stderr.write('While:\n')
        for d in doing:
            if not isinstance(d, str):
                d = d[0] % d[1:]
            sys.stderr.write('  %s\n' % d)

2352
def _error(*message):
2353
    sys.stderr.write('Error: ' + ' '.join(message) +'\n')
2354 2355
    sys.exit(1)

2356
_internal_error_template = """
pombredanne's avatar
pombredanne committed
2357
An internal error occurred due to a bug in either zc.buildout or in a
2358 2359 2360
recipe being used:
"""

2361 2362
def _check_for_unused_options_in_section(buildout, section):
    options = buildout[section]
2363 2364
    unused = [option for option in sorted(options._raw)
              if option not in options._data]
2365
    if unused:
2366 2367 2368 2369 2370
        buildout._logger.warning(
            "Section `%s` contains unused option(s): %s.\n"
            "This may be an indication for either a typo in the option's name "
            "or a bug in the used recipe." %
            (section, ' '.join(map(repr, unused)))
2371
        )
2372

2373 2374 2375 2376 2377
_usage = """\
Usage: buildout [options] [assignments] [command [command arguments]]

Options:

2378
  -c config_file
2379

2380 2381 2382
    Specify the path to the buildout configuration file to be used.
    This defaults to the file named "buildout.cfg" in the current
    working directory.
2383

2384
  -D
Marius Gedminas's avatar
Marius Gedminas committed
2385

2386
    Debug errors.  If an error occurs, then the post-mortem debugger
Kian-Meng Ang's avatar
Kian-Meng Ang committed
2387
    will be started. This is especially useful for debugging recipe
2388
    problems.
Marius Gedminas's avatar
Marius Gedminas committed
2389

2390
  -h, --help
2391

2392
    Print this message and exit.
2393

2394
  -N
2395

2396 2397 2398 2399
    Run in non-newest mode.  This is equivalent to the assignment
    buildout:newest=false.  With this setting, buildout will not seek
    new distributions if installed distributions satisfy it's
    requirements.
2400

2401
  -q
2402

2403
    Decrease the level of verbosity.  This option can be used multiple times.
2404

2405 2406
  -t socket_timeout

2407
    Specify the socket timeout in seconds.
2408

Jim Fulton's avatar
Jim Fulton committed
2409 2410
  -U

2411
    Don't read user defaults.
2412

2413
  -v
Jim Fulton's avatar
Jim Fulton committed
2414

2415
    Increase the level of verbosity.  This option can be used multiple times.
Jim Fulton's avatar
Jim Fulton committed
2416

2417
  --version
2418

2419
    Print buildout version number and exit.
2420

2421 2422 2423 2424 2425 2426
  --dry-run

    Dry-run mode.  With this setting, buildout will display what will
    be uninstalled and what will be installed without doing anything
    in reality.

2427 2428 2429 2430 2431 2432 2433
  --skip-signature-check

    Ignore __buildout_signature__ difference in comparing with the
    previously installed state. Note that __buildout_signature__ is
    updated with new dependencies by using this option.


2434
Assignments are of the form: section:option=value and are used to
2435
provide configuration options that override those given in the
2436 2437 2438 2439 2440
configuration file.  For example, to run the buildout in offline mode,
use buildout:offline=true.

Options and assignments can be interspersed.

2441
Commands:
2442

2443
  install
2444

2445 2446
    Install the parts specified in the buildout configuration.  This is
    the default command if no command is specified.
2447

2448 2449 2450
  bootstrap

    Create a new buildout in the current working directory, copying
2451
    the buildout and setuptools eggs and, creating a basic directory
2452 2453
    structure and a buildout-local buildout script.

2454
  init [requirements]
2455

2456 2457
    Initialize a buildout, creating a minimal buildout.cfg file if it doesn't
    exist and then performing the same actions as for the bootstrap
2458 2459
    command.

2460 2461 2462 2463 2464
    If requirements are supplied, then the generated configuration
    will include an interpreter script that requires them.  This
    provides an easy way to quickly set up a buildout to experiment
    with some packages.

2465 2466
  setup script [setup command and options]

2467
    Run a given setup script arranging that setuptools is in the
2468
    script's path and and that it has been imported so that
2469
    setuptools-provided commands (like bdist_egg) can be used even if
Jim Fulton's avatar
Jim Fulton committed
2470
    the setup script doesn't import setuptools.
2471 2472 2473

    The script can be given either as a script path or a path to a
    directory containing a setup.py script.
2474

2475 2476 2477 2478 2479 2480
  annotate

    Display annotated sections. All sections are displayed, sorted
    alphabetically. For each section, all key-value pairs are displayed,
    sorted alphabetically, along with the origin of the value (file name or
    COMPUTED_VALUE, DEFAULT_VALUE, COMMAND_LINE_VALUE).
Godefroid Chapelle's avatar
Godefroid Chapelle committed
2481 2482

  query section:key
2483

Godefroid Chapelle's avatar
Godefroid Chapelle committed
2484
    Display value of given section key pair.
2485
"""
Marius Gedminas's avatar
Marius Gedminas committed
2486

2487
def _help():
2488
    print_(_usage)
2489 2490
    sys.exit(0)

Marius Gedminas's avatar
Marius Gedminas committed
2491 2492 2493 2494 2495 2496
def _version():
    version = pkg_resources.working_set.find(
                pkg_resources.Requirement.parse('zc.buildout')).version
    print_("buildout version %s" % version)
    sys.exit(0)

2497 2498 2499 2500
def main(args=None):
    if args is None:
        args = sys.argv[1:]

Jim Fulton's avatar
Jim Fulton committed
2501 2502
    config_file = 'buildout.cfg'
    verbosity = 0
2503
    options = []
2504
    use_user_defaults = True
2505
    debug = False
Jim Fulton's avatar
Jim Fulton committed
2506 2507 2508 2509
    while args:
        if args[0][0] == '-':
            op = orig_op = args.pop(0)
            op = op[1:]
2510
            while op and op[0] in 'vqhWUoOnNDA':
Jim Fulton's avatar
Jim Fulton committed
2511 2512
                if op[0] == 'v':
                    verbosity += 10
2513
                elif op[0] == 'q':
Jim Fulton's avatar
Jim Fulton committed
2514
                    verbosity -= 10
Jim Fulton's avatar
Jim Fulton committed
2515
                elif op[0] == 'U':
2516
                    use_user_defaults = False
Jim Fulton's avatar
Jim Fulton committed
2517 2518
                elif op[0] == 'o':
                    options.append(('buildout', 'offline', 'true'))
Jim Fulton's avatar
Jim Fulton committed
2519 2520 2521 2522 2523 2524
                elif op[0] == 'O':
                    options.append(('buildout', 'offline', 'false'))
                elif op[0] == 'n':
                    options.append(('buildout', 'newest', 'true'))
                elif op[0] == 'N':
                    options.append(('buildout', 'newest', 'false'))
2525 2526
                elif op[0] == 'D':
                    debug = True
2527 2528
                else:
                    _help()
Jim Fulton's avatar
Jim Fulton committed
2529
                op = op[1:]
2530

2531 2532
            if op[:1] in  ('c', 't'):
                op_ = op[:1]
Jim Fulton's avatar
Jim Fulton committed
2533
                op = op[1:]
2534 2535 2536 2537

                if op_ == 'c':
                    if op:
                        config_file = op
Jim Fulton's avatar
Jim Fulton committed
2538
                    else:
2539 2540 2541 2542 2543 2544
                        if args:
                            config_file = args.pop(0)
                        else:
                            _error("No file name specified for option", orig_op)
                elif op_ == 't':
                    try:
2545 2546
                        timeout_string = args.pop(0)
                        timeout = int(timeout_string)
Jim Fulton's avatar
Jim Fulton committed
2547 2548
                        options.append(
                            ('buildout', 'socket-timeout', timeout_string))
2549 2550 2551
                    except IndexError:
                        _error("No timeout value specified for option", orig_op)
                    except ValueError:
2552
                        _error("Timeout value must be numeric", orig_op)
2553 2554 2555

            elif orig_op == '--dry-run':
                    options.append(('buildout', 'dry-run', 'true'))
2556 2557
            elif orig_op == '--skip-signature-check':
                    options.append(('buildout', 'check-signature', 'false'))
Jim Fulton's avatar
Jim Fulton committed
2558
            elif op:
2559 2560
                if orig_op == '--help':
                    _help()
Marius Gedminas's avatar
Marius Gedminas committed
2561 2562
                elif orig_op == '--version':
                    _version()
2563
                else:
2564
                    _error("Invalid option", orig_op)
Jim Fulton's avatar
Jim Fulton committed
2565 2566
        elif '=' in args[0]:
            option, value = args.pop(0).split('=', 1)
2567 2568 2569 2570
            option = option.split(':')
            if len(option) == 1:
                option = 'buildout', option[0]
            elif len(option) != 2:
Jim Fulton's avatar
Jim Fulton committed
2571
                _error('Invalid option:', option)
2572
            section, option = option
Jim Fulton's avatar
Jim Fulton committed
2573 2574
            options.append((section.strip(), option.strip(), value.strip()))
        else:
pombredanne's avatar
pombredanne committed
2575
            # We've run out of command-line options and option assignments
Jim Fulton's avatar
Jim Fulton committed
2576 2577 2578 2579 2580
            # The rest should be commands, so we'll stop here
            break

    if verbosity:
        options.append(('buildout', 'verbosity', str(verbosity)))
2581 2582 2583

    if args:
        command = args.pop(0)
2584
        if command not in Buildout.COMMANDS:
2585 2586 2587 2588
            _error('invalid command:', command)
    else:
        command = 'install'

Jim Fulton's avatar
Jim Fulton committed
2589
    try:
2590
        try:
Jim Fulton's avatar
Jim Fulton committed
2591
            buildout = Buildout(config_file, options,
2592
                                use_user_defaults, command, args)
2593
            getattr(buildout, command)(args)
2594
        except SystemExit:
2595 2596 2597
            logging.shutdown()
            # Make sure we properly propagate an exit code from a restarted
            # buildout process.
2598
            raise
2599 2600
        except Exception:
            v = sys.exc_info()[1]
2601
            _doing()
2602 2603
            exc_info = sys.exc_info()
            import pdb, traceback
2604 2605 2606 2607 2608
            if debug:
                traceback.print_exception(*exc_info)
                sys.stderr.write('\nStarting pdb:\n')
                pdb.post_mortem(exc_info[2])
            else:
2609
                if isinstance(v, (zc.buildout.UserError,
2610
                                  distutils.errors.DistutilsError
2611 2612
                                  )
                              ):
2613 2614
                    _error(str(v))
                else:
2615 2616
                    sys.stderr.write(_internal_error_template)
                    traceback.print_exception(*exc_info)
2617
            sys.exit(1)
2618

Jim Fulton's avatar
Jim Fulton committed
2619
    finally:
2620
        logging.shutdown()
2621

2622

2623 2624 2625 2626 2627
if sys.version_info[:2] < (2, 4):
    def reversed(iterable):
        result = list(iterable);
        result.reverse()
        return result
Jim Fulton's avatar
Jim Fulton committed
2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638

_bool_names = {'true': True, 'false': False, True: True, False: False}
def bool_option(options, name, default=None):
    value = options.get(name, default)
    if value is None:
        raise KeyError(name)
    try:
        return _bool_names[value]
    except KeyError:
        raise zc.buildout.UserError(
            'Invalid value for %r option: %r' % (name, value))