easy_install.py 49.6 KB
Newer Older
1
#############################################################################
2
#
3
# Copyright (c) 2005 Zope Foundation and Contributors.
4 5 6 7 8 9 10 11 12 13 14 15 16
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Python easy_install API

This module provides a high-level Python API for installing packages.
17
It doesn't install scripts.  It uses setuptools and requires it to be
18 19 20
installed.
"""

21
import distutils.errors
22
import errno
23 24
import glob
import logging
25
import os
26
import pkg_resources
27 28 29
import py_compile
import re
import setuptools.archive_util
Jason R. Coombs's avatar
Jason R. Coombs committed
30
import setuptools.command.easy_install
31 32
import setuptools.command.setopt
import setuptools.package_index
33
import shutil
34
import subprocess
35
import sys
36
import tempfile
37
import zc.buildout
38

39
_oprp = getattr(os.path, 'realpath', lambda path: path)
40
def realpath(path):
41
    return os.path.normcase(os.path.abspath(_oprp(path)))
42

43 44
default_index_url = os.environ.get(
    'buildout-testing-index-url',
45
    'http://pypi.python.org/simple',
46
    )
47

48 49
logger = logging.getLogger('zc.buildout.easy_install')

50
url_match = re.compile('[a-z0-9+.-]+://').match
Reinout van Rees's avatar
Reinout van Rees committed
51
is_source_encoding_line = re.compile('coding[:=]\s*([-\w.]+)').search
52
# Source encoding regex from http://www.python.org/dev/peps/pep-0263/
53

54
is_win32 = sys.platform == 'win32'
Georgy Berdyshev's avatar
Georgy Berdyshev committed
55 56 57 58 59 60
is_jython = sys.platform.startswith('java')

if is_jython:
    import java.lang.System
    jython_os_name = (java.lang.System.getProperties()['os.name']).lower()

61
# Make sure we're not being run with an older bootstrap.py that gives us
62
# setuptools instead of setuptools
63 64 65 66
has_distribute = pkg_resources.working_set.find(
        pkg_resources.Requirement.parse('distribute')) is not None
has_setuptools = pkg_resources.working_set.find(
        pkg_resources.Requirement.parse('setuptools')) is not None
67 68
if has_distribute and not has_setuptools:
    sys.exit("zc.buildout 2 needs setuptools, not distribute."
69 70 71
             "  Are you using an outdated bootstrap.py?  Make sure"
             " you have the latest version downloaded from"
             " http://downloads.buildout.org/2/bootstrap.py")
Georgy Berdyshev's avatar
Georgy Berdyshev committed
72

73 74
setuptools_loc = pkg_resources.working_set.find(
    pkg_resources.Requirement.parse('setuptools')
75 76
    ).location

77 78 79
# Include buildout and setuptools eggs in paths
buildout_and_setuptools_path = [
    setuptools_loc,
80 81
    pkg_resources.working_set.find(
        pkg_resources.Requirement.parse('zc.buildout')).location,
82 83
    ]

Tarek Ziad's avatar
Tarek Ziad committed
84 85
FILE_SCHEME = re.compile('file://', re.I).match

86 87 88 89 90 91 92 93 94 95 96 97 98
class _Monkey(object):
    def __init__(self, module, **kw):
        mdict = self._mdict = module.__dict__
        self._before = mdict.copy()
        self._overrides = kw

    def __enter__(self):
        self._mdict.update(self._overrides)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self._mdict.clear()
        self._mdict.update(self._before)
99

100 101 102 103 104
class _NoWarn(object):
    def warn(self, *args, **kw):
        pass

_no_warn = _NoWarn()
105

Tarek Ziad's avatar
Tarek Ziad committed
106 107 108 109 110 111 112 113
class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
    """Will allow urls that are local to the system.

    No matter what is allow_hosts.
    """
    def url_ok(self, url, fatal=False):
        if FILE_SCHEME(url):
            return True
114 115 116 117 118 119
        # distutils has its own logging, which can't be hooked / suppressed,
        # so we monkey-patch the 'log' submodule to suppress the stupid
        # "Link to <URL> ***BLOCKED*** by --allow-hosts" message.
        with _Monkey(setuptools.package_index, log=_no_warn):
            return setuptools.package_index.PackageIndex.url_ok(
                                                self, url, False)
120

Tarek Ziad's avatar
Tarek Ziad committed
121

122
_indexes = {}
Jim Fulton's avatar
Jim Fulton committed
123 124
def _get_index(index_url, find_links, allow_hosts=('*',)):
    key = index_url, tuple(find_links)
125 126 127 128
    index = _indexes.get(key)
    if index is not None:
        return index

129 130
    if index_url is None:
        index_url = default_index_url
131 132
    if index_url.startswith('file://'):
        index_url = index_url[7:]
Jim Fulton's avatar
Jim Fulton committed
133
    index = AllowHostsPackageIndex(index_url, hosts=allow_hosts)
134

135 136 137 138 139 140
    if find_links:
        index.add_find_links(find_links)

    _indexes[key] = index
    return index

141
clear_index_cache = _indexes.clear
Jim Fulton's avatar
Jim Fulton committed
142

143
if is_win32:
Jim Fulton's avatar
Jim Fulton committed
144 145 146 147 148 149 150
    # work around spawn lamosity on windows
    # XXX need safe quoting (see the subproces.list2cmdline) and test
    def _safe_arg(arg):
        return '"%s"' % arg
else:
    _safe_arg = str

151 152 153 154 155
def call_subprocess(args, **kw):
    if subprocess.call(args, **kw) != 0:
        raise Exception(
            "Failed to run command:\n%s"
            % repr(args)[1:-1])
Jim Fulton's avatar
Jim Fulton committed
156

157 158

def _execute_permission():
159 160 161 162 163 164
    current_umask = os.umask(0o022)
    # os.umask only returns the current umask if you also give it one, so we
    # have to give it a dummy one and immediately set it back to the real
    # value...  Distribute does the same.
    os.umask(current_umask)
    return 0o777 - current_umask
165 166


167
_easy_install_cmd = 'from setuptools.command.easy_install import main; main()'
168

169 170
class Installer:

171
    _versions = {}
172 173
    _required_by = {}
    _picked_versions = {}
174
    _download_cache = None
175
    _install_from_cache = False
176
    _prefer_final = True
177
    _use_dependency_links = True
Jim Fulton's avatar
Jim Fulton committed
178
    _allow_picked_versions = True
179
    _store_required_by = False
180

181 182 183 184 185
    def __init__(self,
                 dest=None,
                 links=(),
                 index=None,
                 executable=sys.executable,
Jim Fulton's avatar
Jim Fulton committed
186
                 always_unzip=None, # Backward compat :/
187 188
                 path=None,
                 newest=True,
Jim Fulton's avatar
Jim Fulton committed
189
                 versions=None,
190
                 use_dependency_links=None,
Tarek Ziad's avatar
Tarek Ziad committed
191
                 allow_hosts=('*',)
192
                 ):
Jim Fulton's avatar
Jim Fulton committed
193
        assert executable == sys.executable, (executable, sys.executable)
194
        self._dest = dest
Tarek Ziad's avatar
Tarek Ziad committed
195
        self._allow_hosts = allow_hosts
196 197 198 199 200 201 202

        if self._install_from_cache:
            if not self._download_cache:
                raise ValueError("install_from_cache set to true with no"
                                 " download cache")
            links = ()
            index = 'file://' + self._download_cache
203

204 205
        if use_dependency_links is not None:
            self._use_dependency_links = use_dependency_links
206
        self._links = links = list(_fix_file_links(links))
207 208 209
        if self._download_cache and (self._download_cache not in links):
            links.insert(0, self._download_cache)

210
        self._index_url = index
211
        path = (path and path[:] or []) + buildout_and_setuptools_path
212 213 214
        if dest is not None and dest not in path:
            path.insert(0, dest)
        self._path = path
215 216
        if self._dest is None:
            newest = False
217
        self._newest = newest
Jim Fulton's avatar
Jim Fulton committed
218 219
        self._env = pkg_resources.Environment(path)
        self._index = _get_index(index, links, self._allow_hosts)
220 221

        if versions is not None:
222
            self._versions = normalize_versions(versions)
223

224
    def _satisfied(self, req, source=None):
225 226
        dists = [dist for dist in self._env[req.project_name] if dist in req]
        if not dists:
227 228
            logger.debug('We have no distributions for %s that satisfies %r.',
                         req.project_name, str(req))
229

230
            return None, self._obtain(req, source)
231 232 233 234 235 236

        # Note that dists are sorted from best to worst, as promised by
        # env.__getitem__

        for dist in dists:
            if (dist.precedence == pkg_resources.DEVELOP_DIST):
237
                logger.debug('We have a develop egg: %s', dist)
238
                return dist, None
239

240 241 242 243 244 245
        # Special common case, we have a specification for a single version:
        specs = req.specs
        if len(specs) == 1 and specs[0][0] == '==':
            logger.debug('We have the distribution that satisfies %r.',
                         str(req))
            return dists[0], None
246

247 248 249 250 251 252 253 254 255 256 257 258 259 260
        if self._prefer_final:
            fdists = [dist for dist in dists
                      if _final_version(dist.parsed_version)
                      ]
            if fdists:
                # There are final dists, so only use those
                dists = fdists

        if not self._newest:
            # We don't need the newest, so we'll use the newest one we
            # find, which is the first returned by
            # Environment.__getitem__.
            return dists[0], None

261
        best_we_have = dists[0] # Because dists are sorted from best to worst
Jim Fulton's avatar
Jim Fulton committed
262

263 264 265 266
        # We have some installed distros.  There might, theoretically, be
        # newer ones.  Let's find out which ones are available and see if
        # any are newer.  We only do this if we're willing to install
        # something, which is only true if dest is not None:
267

Jim Fulton's avatar
Jim Fulton committed
268
        best_available = self._obtain(req, source)
269 270 271 272 273

        if best_available is None:
            # That's a bit odd.  There aren't any distros available.
            # We should use the best one we have that meets the requirement.
            logger.debug(
274 275 276
                'There are no distros available that meet %r.\n'
                'Using our best, %s.',
                str(req), best_available)
277
            return best_we_have, None
278

279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
        if self._prefer_final:
            if _final_version(best_available.parsed_version):
                if _final_version(best_we_have.parsed_version):
                    if (best_we_have.parsed_version
                        <
                        best_available.parsed_version
                        ):
                        return None, best_available
                else:
                    return None, best_available
            else:
                if (not _final_version(best_we_have.parsed_version)
                    and
                    (best_we_have.parsed_version
                     <
                     best_available.parsed_version
                     )
                    ):
                    return None, best_available
        else:
            if (best_we_have.parsed_version
                <
                best_available.parsed_version
                ):
                return None, best_available
304

305 306 307 308
        logger.debug(
            'We have the best distribution that satisfies %r.',
            str(req))
        return best_we_have, None
309

310
    def _load_dist(self, dist):
Jim Fulton's avatar
Jim Fulton committed
311
        dists = pkg_resources.Environment(dist.location)[dist.project_name]
312 313
        assert len(dists) == 1
        return dists[0]
314

315
    def _call_easy_install(self, spec, ws, dest, dist):
316

317 318
        tmp = tempfile.mkdtemp(dir=dest)
        try:
319
            path = setuptools_loc
320

Jim Fulton's avatar
Jim Fulton committed
321
            args = [sys.executable, '-c', _easy_install_cmd, '-mZUNxd', tmp]
322 323
            level = logger.getEffectiveLevel()
            if level > 0:
324
                args.append('-q')
325
            elif level < 0:
326
                args.append('-v')
327

328
            args.append(spec)
329 330

            if level <= logging.DEBUG:
Jim Fulton's avatar
Jim Fulton committed
331 332
                logger.debug('Running easy_install:\n"%s"\npath=%s\n',
                             '" "'.join(args), path)
333 334

            sys.stdout.flush() # We want any pending output first
335

336 337 338
            exit_code = subprocess.call(
                list(args),
                env=dict(os.environ, PYTHONPATH=path))
339 340

            dists = []
Jim Fulton's avatar
Jim Fulton committed
341
            env = pkg_resources.Environment([tmp])
342 343
            for project in env:
                dists.extend(env[project])
344

345 346
            if exit_code:
                logger.error(
pombredanne's avatar
pombredanne committed
347
                    "An error occurred when trying to install %s. "
348
                    "Look above this message for any errors that "
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
                    "were output by easy_install.",
                    dist)

            if not dists:
                raise zc.buildout.UserError("Couldn't install: %s" % dist)

            if len(dists) > 1:
                logger.warn("Installing %s\n"
                            "caused multiple distributions to be installed:\n"
                            "%s\n",
                            dist, '\n'.join(map(str, dists)))
            else:
                d = dists[0]
                if d.project_name != dist.project_name:
                    logger.warn("Installing %s\n"
                                "Caused installation of a distribution:\n"
                                "%s\n"
                                "with a different project name.",
                                dist, d)
                if d.version != dist.version:
                    logger.warn("Installing %s\n"
                                "Caused installation of a distribution:\n"
                                "%s\n"
                                "with a different version.",
                                dist, d)

            result = []
            for d in dists:
                newloc = os.path.join(dest, os.path.basename(d.location))
                if os.path.exists(newloc):
Jim Fulton's avatar
Jim Fulton committed
379
                    if os.path.isdir(newloc):
380 381 382 383 384
                        shutil.rmtree(newloc)
                    else:
                        os.remove(newloc)
                os.rename(d.location, newloc)

Jim Fulton's avatar
Jim Fulton committed
385
                [d] = pkg_resources.Environment([newloc])[d.project_name]
386

387 388 389
                result.append(d)

            return result
390

391 392
        finally:
            shutil.rmtree(tmp)
393

394
    def _obtain(self, requirement, source=None):
395
        # initialize out index for this project:
396
        index = self._index
397

398
        if index.obtain(requirement) is None:
399
            # Nothing is available.
400
            return None
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422

        # Filter the available dists for the requirement and source flag
        dists = [dist for dist in index[requirement.project_name]
                 if ((dist in requirement)
                     and
                     ((not source) or
                      (dist.precedence == pkg_resources.SOURCE_DIST)
                      )
                     )
                 ]

        # If we prefer final dists, filter for final and use the
        # result if it is non empty.
        if self._prefer_final:
            fdists = [dist for dist in dists
                      if _final_version(dist.parsed_version)
                      ]
            if fdists:
                # There are final dists, so only use those
                dists = fdists

        # Now find the best one:
423 424
        best = []
        bestv = ()
425
        for dist in dists:
426 427 428 429 430 431 432 433 434
            distv = dist.parsed_version
            if distv > bestv:
                best = [dist]
                bestv = distv
            elif distv == bestv:
                best.append(dist)

        if not best:
            return None
435

436 437
        if len(best) == 1:
            return best[0]
438

439 440
        if self._download_cache:
            for dist in best:
441 442
                if (realpath(os.path.dirname(dist.location))
                    ==
443
                    self._download_cache
444
                    ):
445
                    return dist
446

447 448
        best.sort()
        return best[-1]
449

Jim Fulton's avatar
Jim Fulton committed
450 451
    def _fetch(self, dist, tmp, download_cache):
        if (download_cache
452
            and (realpath(os.path.dirname(dist.location)) == download_cache)
Jim Fulton's avatar
Jim Fulton committed
453 454 455 456 457
            ):
            return dist

        new_location = self._index.download(dist.location, tmp)
        if (download_cache
458
            and (realpath(new_location) == realpath(dist.location))
Jim Fulton's avatar
Jim Fulton committed
459 460
            and os.path.isfile(new_location)
            ):
461
            # setuptools avoids making extra copies, but we want to copy
Jim Fulton's avatar
Jim Fulton committed
462 463 464
            # to the download cache
            shutil.copy2(new_location, tmp)
            new_location = os.path.join(tmp, os.path.basename(new_location))
465

Jim Fulton's avatar
Jim Fulton committed
466
        return dist.clone(location=new_location)
467

Jim Fulton's avatar
Jim Fulton committed
468
    def _get_dist(self, requirement, ws):
469

470
        __doing__ = 'Getting distribution for %r.', str(requirement)
471

472 473
        # Maybe an existing dist is already the best dist that satisfies the
        # requirement
474
        dist, avail = self._satisfied(requirement)
475

476
        if dist is None:
477 478 479 480 481 482 483
            if self._dest is None:
                raise zc.buildout.UserError(
                    "We don't have a distribution for %s\n"
                    "and can't install one in offline (no-install) mode.\n"
                    % requirement)

            logger.info(*__doing__)
484

485 486
            # Retrieve the dist:
            if avail is None:
487
                self._index.obtain(requirement)
488
                raise MissingDistribution(requirement, ws)
489

490 491 492
            # We may overwrite distributions, so clear importer
            # cache.
            sys.path_importer_cache.clear()
493

494
            tmp = self._download_cache
Jim Fulton's avatar
Jim Fulton committed
495 496 497
            if tmp is None:
                tmp = tempfile.mkdtemp('get_dist')

498
            try:
Jim Fulton's avatar
Jim Fulton committed
499
                dist = self._fetch(avail, tmp, self._download_cache)
500

501 502 503
                if dist is None:
                    raise zc.buildout.UserError(
                        "Couln't download distribution %s." % avail)
504

505 506
                if dist.precedence == pkg_resources.EGG_DIST:
                    # It's already an egg, just fetch it into the dest
507

508 509
                    newloc = os.path.join(
                        self._dest, os.path.basename(dist.location))
510

511 512 513 514 515
                    if os.path.isdir(dist.location):
                        # we got a directory. It must have been
                        # obtained locally.  Just copy it.
                        shutil.copytree(dist.location, newloc)
                    else:
516

Jim Fulton's avatar
Jim Fulton committed
517 518 519

                        setuptools.archive_util.unpack_archive(
                            dist.location, newloc)
520

521 522
                    redo_pyc(newloc)

523 524 525
                    # Getting the dist from the environment causes the
                    # distribution meta data to be read.  Cloning isn't
                    # good enough.
Jim Fulton's avatar
Jim Fulton committed
526 527
                    dists = pkg_resources.Environment([newloc])[
                        dist.project_name]
528 529 530 531 532
                else:
                    # It's some other kind of dist.  We'll let easy_install
                    # deal with it:
                    dists = self._call_easy_install(
                        dist.location, ws, self._dest, dist)
533 534
                    for dist in dists:
                        redo_pyc(dist.location)
535

536 537 538
            finally:
                if tmp != self._download_cache:
                    shutil.rmtree(tmp)
539

540 541
            self._env.scan([self._dest])
            dist = self._env.best_match(requirement, ws)
542
            logger.info("Got %s.", dist)
543

544 545
        else:
            dists = [dist]
546

547
        for dist in dists:
548 549
            if (dist.has_metadata('dependency_links.txt')
                and not self._install_from_cache
550
                and self._use_dependency_links
551
                ):
552 553 554 555 556
                for link in dist.get_metadata_lines('dependency_links.txt'):
                    link = link.strip()
                    if link not in self._links:
                        logger.debug('Adding find link %r from %s', link, dist)
                        self._links.append(link)
Jim Fulton's avatar
Jim Fulton committed
557
                        self._index = _get_index(self._index_url, self._links,
Tarek Ziad's avatar
Tarek Ziad committed
558
                                                 self._allow_hosts)
559 560 561 562 563 564 565 566 567 568

        for dist in dists:
            # Check whether we picked a version and, if we did, report it:
            if not (
                dist.precedence == pkg_resources.DEVELOP_DIST
                or
                (len(requirement.specs) == 1
                 and
                 requirement.specs[0][0] == '==')
                ):
569 570
                logger.debug('Picked: %s = %s',
                             dist.project_name, dist.version)
571
                self._picked_versions[dist.project_name] = dist.version
572

Jim Fulton's avatar
Jim Fulton committed
573 574 575 576
                if not self._allow_picked_versions:
                    raise zc.buildout.UserError(
                        'Picked: %s = %s' % (dist.project_name, dist.version)
                        )
577 578

        return dists
579

580
    def _maybe_add_setuptools(self, ws, dist):
581 582
        if dist.has_metadata('namespace_packages.txt'):
            for r in dist.requires():
583
                if r.project_name in ('setuptools', 'setuptools'):
584 585
                    break
            else:
586
                # We have a namespace package but no requirement for setuptools
587 588
                if dist.precedence == pkg_resources.DEVELOP_DIST:
                    logger.warn(
589
                        "Develop distribution: %s\n"
590
                        "uses namespace packages but the distribution "
591
                        "does not require setuptools.",
592
                        dist)
Jim Fulton's avatar
Jim Fulton committed
593
                requirement = self._constrain(
594
                    pkg_resources.Requirement.parse('setuptools')
Jim Fulton's avatar
Jim Fulton committed
595
                    )
596
                if ws.find(requirement) is None:
Jim Fulton's avatar
Jim Fulton committed
597
                    for dist in self._get_dist(requirement, ws):
598
                        ws.add(dist)
599 600


Jim Fulton's avatar
Jim Fulton committed
601
    def _constrain(self, requirement):
602
        constraint = self._versions.get(requirement.project_name.lower())
603 604
        if constraint:
            requirement = _constrained_requirement(constraint, requirement)
Jim Fulton's avatar
Jim Fulton committed
605 606
        return requirement

607 608
    def install(self, specs, working_set=None):

609
        logger.debug('Installing %s.', repr(specs)[1:-1])
610 611

        path = self._path
612 613 614
        dest = self._dest
        if dest is not None and dest not in path:
            path.insert(0, dest)
615

Jim Fulton's avatar
Jim Fulton committed
616
        requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
617 618 619 620
                        for spec in specs]

        if working_set is None:
            ws = pkg_resources.WorkingSet([])
Jim Fulton's avatar
Jim Fulton committed
621
        else:
622
            ws = working_set
623

624
        for requirement in requirements:
Jim Fulton's avatar
Jim Fulton committed
625
            for dist in self._get_dist(requirement, ws):
626
                ws.add(dist)
627
                self._maybe_add_setuptools(ws, dist)
628 629

        # OK, we have the requested distributions and they're in the working
630 631 632 633 634 635 636 637 638 639
        # set, but they may have unmet requirements.  We'll resolve these
        # requirements. This is code modified from
        # pkg_resources.WorkingSet.resolve.  We can't reuse that code directly
        # because we have to constrain our requirements (see
        # versions_section_ignored_for_dependency_in_favor_of_site_packages in
        # zc.buildout.tests).
        requirements.reverse() # Set up the stack.
        processed = {}  # This is a set of processed requirements.
        best = {}  # This is a mapping of key -> dist.
        # Note that we don't use the existing environment, because we want
640 641
        # to look for new eggs unless what we have is the best that
        # matches the requirement.
642 643 644 645 646 647 648 649 650 651 652 653 654 655
        env = pkg_resources.Environment(ws.entries)
        while requirements:
            # Process dependencies breadth-first.
            req = self._constrain(requirements.pop(0))
            if req in processed:
                # Ignore cyclic or redundant dependencies.
                continue
            dist = best.get(req.key)
            if dist is None:
                # Find the best distribution and add it to the map.
                dist = ws.by_key.get(req.key)
                if dist is None:
                    try:
                        dist = best[req.key] = env.best_match(req, ws)
Alex Clark's avatar
Alex Clark committed
656
                    except pkg_resources.VersionConflict as err:
657 658
                        raise VersionConflict(err, ws)
                    if dist is None:
659
                        if dest:
660 661 662 663 664 665 666 667 668 669 670 671 672
                            logger.debug('Getting required %r', str(req))
                        else:
                            logger.debug('Adding required %r', str(req))
                        _log_requirement(ws, req)
                        for dist in self._get_dist(req, ws,):
                            ws.add(dist)
                            self._maybe_add_setuptools(ws, dist)
            if dist not in req:
                # Oops, the "best" so far conflicts with a dependency.
                raise VersionConflict(
                    pkg_resources.VersionConflict(dist, req), ws)
            requirements.extend(dist.requires(req.extras)[::-1])
            processed[req] = True
673
        return ws
674

675
    def build(self, spec, build_ext):
676

Jim Fulton's avatar
Jim Fulton committed
677
        requirement = self._constrain(pkg_resources.Requirement.parse(spec))
678

679
        dist, avail = self._satisfied(requirement, 1)
680
        if dist is not None:
681
            return [dist.location]
682

683 684 685
        # Retrieve the dist:
        if avail is None:
            raise zc.buildout.UserError(
686 687
                "Couldn't find a source distribution for %r."
                % str(requirement))
688

689 690 691 692 693 694 695
        if self._dest is None:
            raise zc.buildout.UserError(
                "We don't have a distribution for %s\n"
                "and can't build one in offline (no-install) mode.\n"
                % requirement
                )

696
        logger.debug('Building %r', spec)
697

698
        tmp = self._download_cache
Jim Fulton's avatar
Jim Fulton committed
699 700 701
        if tmp is None:
            tmp = tempfile.mkdtemp('get_dist')

702
        try:
Jim Fulton's avatar
Jim Fulton committed
703
            dist = self._fetch(avail, tmp, self._download_cache)
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724

            build_tmp = tempfile.mkdtemp('build')
            try:
                setuptools.archive_util.unpack_archive(dist.location,
                                                       build_tmp)
                if os.path.exists(os.path.join(build_tmp, 'setup.py')):
                    base = build_tmp
                else:
                    setups = glob.glob(
                        os.path.join(build_tmp, '*', 'setup.py'))
                    if not setups:
                        raise distutils.errors.DistutilsError(
                            "Couldn't find a setup script in %s"
                            % os.path.basename(dist.location)
                            )
                    if len(setups) > 1:
                        raise distutils.errors.DistutilsError(
                            "Multiple setup scripts in %s"
                            % os.path.basename(dist.location)
                            )
                    base = os.path.dirname(setups[0])
725

726 727 728 729 730 731 732 733 734 735 736
                setup_cfg = os.path.join(base, 'setup.cfg')
                if not os.path.exists(setup_cfg):
                    f = open(setup_cfg, 'w')
                    f.close()
                setuptools.command.setopt.edit_config(
                    setup_cfg, dict(build_ext=build_ext))

                dists = self._call_easy_install(
                    base, pkg_resources.WorkingSet(),
                    self._dest, dist)

737 738 739
                for dist in dists:
                    redo_pyc(dist.location)

740 741 742
                return [dist.location for dist in dists]
            finally:
                shutil.rmtree(build_tmp)
743

744
        finally:
745 746 747
            if tmp != self._download_cache:
                shutil.rmtree(tmp)

748 749 750 751 752 753 754 755 756 757

def normalize_versions(versions):
    """Return version dict with keys normalized to lowercase.

    PyPI is case-insensitive and not all distributions are consistent in
    their own naming.
    """
    return dict([(k.lower(), v) for (k, v) in versions.items()])


758 759 760
def default_versions(versions=None):
    old = Installer._versions
    if versions is not None:
761
        Installer._versions = normalize_versions(versions)
762
    return old
763

764 765 766
def download_cache(path=-1):
    old = Installer._download_cache
    if path != -1:
767
        if path:
768
            path = realpath(path)
769 770 771
        Installer._download_cache = path
    return old

772 773 774 775 776 777
def install_from_cache(setting=None):
    old = Installer._install_from_cache
    if setting is not None:
        Installer._install_from_cache = bool(setting)
    return old

778 779 780 781 782 783
def prefer_final(setting=None):
    old = Installer._prefer_final
    if setting is not None:
        Installer._prefer_final = bool(setting)
    return old

784 785 786 787 788 789
def use_dependency_links(setting=None):
    old = Installer._use_dependency_links
    if setting is not None:
        Installer._use_dependency_links = bool(setting)
    return old

Jim Fulton's avatar
Jim Fulton committed
790 791 792 793 794 795
def allow_picked_versions(setting=None):
    old = Installer._allow_picked_versions
    if setting is not None:
        Installer._allow_picked_versions = bool(setting)
    return old

796 797
def store_required_by(setting=None):
    old = Installer._store_required_by
798
    if setting is not None:
799
        Installer._store_required_by = bool(setting)
800 801
    return old

802 803 804 805 806
def get_picked_versions():
    picked_versions = sorted(Installer._picked_versions.items())
    required_by = Installer._required_by
    return (picked_versions, required_by)

807

808 809
def install(specs, dest,
            links=(), index=None,
Jim Fulton's avatar
Jim Fulton committed
810 811
            executable=sys.executable,
            always_unzip=None, # Backward compat :/
812
            path=None, working_set=None, newest=True, versions=None,
813 814 815 816
            use_dependency_links=None, allow_hosts=('*',),
            include_site_packages=None,
            allowed_eggs_from_site_packages=None,
            ):
Jim Fulton's avatar
Jim Fulton committed
817
    assert executable == sys.executable, (executable, sys.executable)
818 819 820
    assert include_site_packages is None
    assert allowed_eggs_from_site_packages is None

Jim Fulton's avatar
Jim Fulton committed
821 822
    installer = Installer(dest, links, index, sys.executable,
                          always_unzip, path,
823
                          newest, versions, use_dependency_links,
Tarek Ziad's avatar
Tarek Ziad committed
824
                          allow_hosts=allow_hosts)
825 826 827 828 829 830
    return installer.install(specs, working_set)


def build(spec, dest, build_ext,
          links=(), index=None,
          executable=sys.executable,
Tarek Ziad's avatar
Tarek Ziad committed
831
          path=None, newest=True, versions=None, allow_hosts=('*',)):
Jim Fulton's avatar
Jim Fulton committed
832
    assert executable == sys.executable, (executable, sys.executable)
Jim Fulton's avatar
Jim Fulton committed
833
    installer = Installer(dest, links, index, executable,
Jim Fulton's avatar
Jim Fulton committed
834
                          True, path, newest,
Tarek Ziad's avatar
Tarek Ziad committed
835
                          versions, allow_hosts=allow_hosts)
836 837
    return installer.build(spec, build_ext)

838

839

840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
def _rm(*paths):
    for path in paths:
        if os.path.isdir(path):
            shutil.rmtree(path)
        elif os.path.exists(path):
            os.remove(path)

def _copyeggs(src, dest, suffix, undo):
    result = []
    undo.append(lambda : _rm(*result))
    for name in os.listdir(src):
        if name.endswith(suffix):
            new = os.path.join(dest, name)
            _rm(new)
            os.rename(os.path.join(src, name), new)
            result.append(new)
856

857
    assert len(result) == 1, str(result)
858
    undo.pop()
859

860 861 862 863 864
    return result[0]

def develop(setup, dest,
            build_ext=None,
            executable=sys.executable):
Jim Fulton's avatar
Jim Fulton committed
865
    assert executable == sys.executable, (executable, sys.executable)
866 867 868 869 870
    if os.path.isdir(setup):
        directory = setup
        setup = os.path.join(directory, 'setup.py')
    else:
        directory = os.path.dirname(setup)
871

872 873 874 875 876 877 878 879 880 881 882 883
    undo = []
    try:
        if build_ext:
            setup_cfg = os.path.join(directory, 'setup.cfg')
            if os.path.exists(setup_cfg):
                os.rename(setup_cfg, setup_cfg+'-develop-aside')
                def restore_old_setup():
                    if os.path.exists(setup_cfg):
                        os.remove(setup_cfg)
                    os.rename(setup_cfg+'-develop-aside', setup_cfg)
                undo.append(restore_old_setup)
            else:
884 885
                f = open(setup_cfg, 'w')
                f.close()
886 887 888 889 890 891 892 893
                undo.append(lambda: os.remove(setup_cfg))
            setuptools.command.setopt.edit_config(
                setup_cfg, dict(build_ext=build_ext))

        fd, tsetup = tempfile.mkstemp()
        undo.append(lambda: os.remove(tsetup))
        undo.append(lambda: os.close(fd))

894
        os.write(fd, (runsetup_template % dict(
895
            setuptools=setuptools_loc,
896 897 898
            setupdir=directory,
            setup=setup,
            __file__ = setup,
899
            )).encode())
900 901

        tmp3 = tempfile.mkdtemp('build', dir=dest)
902
        undo.append(lambda : shutil.rmtree(tmp3))
903

Jim Fulton's avatar
Jim Fulton committed
904
        args = [executable,  tsetup, '-q', 'develop', '-mxN', '-d', tmp3]
905 906

        log_level = logger.getEffectiveLevel()
907 908
        if log_level <= 0:
            if log_level == 0:
909
                del args[2]
910
            else:
911
                args[2] == '-v'
912
        if log_level < logging.DEBUG:
913
            logger.debug("in: %r\n%s", directory, ' '.join(args))
914

915
        call_subprocess(args)
916 917 918 919 920 921

        return _copyeggs(tmp3, dest, '.egg-link', undo)

    finally:
        undo.reverse()
        [f() for f in undo]
922 923


924 925 926
def working_set(specs, executable, path=None,
                include_site_packages=None,
                allowed_eggs_from_site_packages=None):
Jim Fulton's avatar
Jim Fulton committed
927 928 929 930 931
    # Backward compat:
    if path is None:
        path = executable
    else:
        assert executable == sys.executable, (executable, sys.executable)
932 933 934
    assert include_site_packages is None
    assert allowed_eggs_from_site_packages is None

Jim Fulton's avatar
Jim Fulton committed
935
    return install(specs, None, path=path)
936

Jim Fulton's avatar
Jim Fulton committed
937
def scripts(reqs, working_set, executable, dest=None,
938 939 940
            scripts=None,
            extra_paths=(),
            arguments='',
941
            interpreter=None,
942
            initialization='',
943
            relative_paths=False,
944
            ):
Jim Fulton's avatar
Jim Fulton committed
945
    assert executable == sys.executable, (executable, sys.executable)
946

947 948
    path = [dist.location for dist in working_set]
    path.extend(extra_paths)
949 950 951 952 953
    # order preserving unique
    unique_path = []
    for p in path:
        if p not in unique_path:
            unique_path.append(p)
954
    path = list(map(realpath, unique_path))
955

956 957
    generated = []

958 959 960 961
    if isinstance(reqs, str):
        raise TypeError('Expected iterable of requirements or entry points,'
                        ' got string.')

962 963 964
    if initialization:
        initialization = '\n'+initialization+'\n'

965
    entry_points = []
966
    distutils_scripts = []
967 968 969 970
    for req in reqs:
        if isinstance(req, str):
            req = pkg_resources.Requirement.parse(req)
            dist = working_set.find(req)
971
            # regular console_scripts entry points
972
            for name in pkg_resources.get_entry_map(dist, 'console_scripts'):
973 974
                entry_point = dist.get_entry_info('console_scripts', name)
                entry_points.append(
975 976
                    (name, entry_point.module_name,
                     '.'.join(entry_point.attrs))
Jim Fulton's avatar
Jim Fulton committed
977
                    )
978 979 980
            # The metadata on "old-style" distutils scripts is not retained by
            # distutils/setuptools, except by placing the original scripts in
            # /EGG-INFO/scripts/.
981 982
            if dist.metadata_isdir('scripts'):
                for name in dist.metadata_listdir('scripts'):
Reinout van Rees's avatar
Reinout van Rees committed
983 984 985
                    if dist.metadata_isdir('scripts/' + name):
                        # Probably Python 3 __pycache__ directory.
                        continue
986
                    contents = dist.get_metadata('scripts/' + name)
987
                    distutils_scripts.append((name, contents))
988 989
        else:
            entry_points.append(req)
990

991 992 993 994 995 996 997
    for name, module_name, attrs in entry_points:
        if scripts is not None:
            sname = scripts.get(name)
            if sname is None:
                continue
        else:
            sname = name
998

999
        sname = os.path.join(dest, sname)
1000 1001
        spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)

1002
        generated.extend(
Jim Fulton's avatar
Jim Fulton committed
1003
            _script(module_name, attrs, spath, sname, arguments,
1004
                    initialization, rpsetup)
1005
            )
1006

Reinout van Rees's avatar
Reinout van Rees committed
1007
    for name, contents in distutils_scripts:
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
        if scripts is not None:
            sname = scripts.get(name)
            if sname is None:
                continue
        else:
            sname = name

        sname = os.path.join(dest, sname)
        spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)

        generated.extend(
Jim Fulton's avatar
Jim Fulton committed
1019
            _distutils_script(spath, sname, contents, initialization, rpsetup)
1020 1021
            )

1022 1023
    if interpreter:
        sname = os.path.join(dest, interpreter)
1024
        spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)
1025
        generated.extend(_pyscript(spath, sname, rpsetup, initialization))
1026 1027 1028

    return generated

1029

1030 1031
def _relative_path_and_setup(sname, path, relative_paths):
    if relative_paths:
1032 1033
        relative_paths = os.path.normcase(relative_paths)
        sname = os.path.normcase(os.path.abspath(sname))
1034
        spath = ',\n  '.join(
1035
            [_relativitize(os.path.normcase(path_item), sname, relative_paths)
1036 1037 1038
             for path_item in path]
            )
        rpsetup = relative_paths_setup
1039 1040
        for i in range(_relative_depth(relative_paths, sname)):
            rpsetup += "base = os.path.dirname(base)\n"
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
    else:
        spath = repr(path)[1:-1].replace(', ', ',\n  ')
        rpsetup = ''
    return spath, rpsetup


def _relative_depth(common, path):
    n = 0
    while 1:
        dirname = os.path.dirname(path)
        if dirname == path:
            raise AssertionError("dirname of %s is the same" % dirname)
        if dirname == common:
            break
        n += 1
        path = dirname
    return n

1059

1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
def _relative_path(common, path):
    r = []
    while 1:
        dirname, basename = os.path.split(path)
        r.append(basename)
        if dirname == common:
            break
        if dirname == path:
            raise AssertionError("dirname of %s is the same" % dirname)
        path = dirname
    r.reverse()
    return os.path.join(*r)

1073

1074 1075 1076 1077 1078 1079 1080
def _relativitize(path, script, relative_paths):
    if path == script:
        raise AssertionError("path == script")
    common = os.path.dirname(os.path.commonprefix([path, script]))
    if (common == relative_paths or
        common.startswith(os.path.join(relative_paths, ''))
        ):
1081
        return "join(base, %r)" % _relative_path(common, path)
1082 1083 1084 1085 1086 1087 1088 1089
    else:
        return repr(path)


relative_paths_setup = """
import os

join = os.path.join
Jim Fulton's avatar
Jim Fulton committed
1090
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
1091 1092
"""

Jim Fulton's avatar
Jim Fulton committed
1093
def _script(module_name, attrs, path, dest, arguments, initialization, rsetup):
1094
    if is_win32:
Jim Fulton's avatar
Jim Fulton committed
1095
        dest += '-script.py'
1096

Jim Fulton's avatar
Jim Fulton committed
1097 1098
    python = _safe_arg(sys.executable)

1099
    contents = script_template % dict(
Jim Fulton's avatar
Jim Fulton committed
1100
        python = python,
1101
        path = path,
1102 1103
        module_name = module_name,
        attrs = attrs,
1104
        arguments = arguments,
1105
        initialization = initialization,
1106
        relative_paths_setup = rsetup,
1107
        )
1108 1109 1110
    return _create_script(contents, dest)


Jim Fulton's avatar
Jim Fulton committed
1111
def _distutils_script(path, dest, script_content, initialization, rsetup):
1112 1113
    if is_win32:
        dest += '-script.py'
1114

Reinout van Rees's avatar
Reinout van Rees committed
1115
    lines = script_content.splitlines(True)
1116 1117 1118
    if not ('#!' in lines[0]) and ('python' in lines[0]):
        # The script doesn't follow distutil's rules.  Ignore it.
        return []
1119
    lines = lines[1:]  # Strip off the first hashbang line.
1120
    line_with_first_import = len(lines)
1121 1122 1123
    for line_number, line in enumerate(lines):
        if not 'import' in line:
            continue
1124
        if not (line.startswith('import') or line.startswith('from')):
1125 1126 1127 1128
            continue
        if '__future__' in line:
            continue
        line_with_first_import = line_number
1129 1130 1131 1132
        break

    before = ''.join(lines[:line_with_first_import])
    after = ''.join(lines[line_with_first_import:])
Jim Fulton's avatar
Jim Fulton committed
1133 1134 1135

    python = _safe_arg(sys.executable)

1136
    contents = distutils_script_template % dict(
Jim Fulton's avatar
Jim Fulton committed
1137
        python = python,
1138 1139 1140
        path = path,
        initialization = initialization,
        relative_paths_setup = rsetup,
1141 1142
        before = before,
        after = after
1143 1144 1145
        )
    return _create_script(contents, dest)

1146 1147 1148 1149 1150 1151 1152 1153 1154
def _file_changed(filename, old_contents, mode='r'):
    try:
        with open(filename, mode) as f:
            return f.read() != old_contents
    except EnvironmentError as e:
        if e.errno == errno.ENOENT:
            return True
        else:
            raise
1155 1156 1157 1158 1159

def _create_script(contents, dest):
    generated = []
    script = dest

1160
    changed = _file_changed(dest, contents)
1161

1162
    if is_win32:
1163
        # generate exe file and give the script a magic name:
1164
        win32_exe = os.path.splitext(dest)[0] # remove ".py"
1165
        if win32_exe.endswith('-script'):
1166 1167
            win32_exe = win32_exe[:-7] # remove "-script"
        win32_exe = win32_exe + '.exe' # add ".exe"
1168 1169 1170 1171 1172
        try:
            new_data = setuptools.command.easy_install.get_win_launcher('cli')
        except AttributeError:
            # fall back for compatibility with older Distribute versions
            new_data = pkg_resources.resource_string('setuptools', 'cli.exe')
1173 1174

        if _file_changed(win32_exe, new_data, 'rb'):
1175
            # Only write it if it's different.
1176 1177
            with open(win32_exe, 'wb') as f:
                f.write(new_data)
1178
        generated.append(win32_exe)
1179

1180
    if changed:
1181 1182
        with open(dest, 'w') as f:
            f.write(contents)
1183 1184 1185 1186
        logger.info(
            "Generated script %r.",
            # Normalize for windows
            script.endswith('-script.py') and script[:-10] or script)
1187 1188

        try:
1189
            os.chmod(dest, _execute_permission())
1190 1191
        except (AttributeError, os.error):
            pass
1192

Jim Fulton's avatar
Jim Fulton committed
1193 1194
    generated.append(dest)
    return generated
1195

1196

Georgy Berdyshev's avatar
Georgy Berdyshev committed
1197 1198 1199 1200 1201 1202
if is_jython and jython_os_name == 'linux':
    script_header = '#!/usr/bin/env %(python)s'
else:
    script_header = '#!%(python)s'


1203
script_template = script_header + '''\
1204

1205
%(relative_paths_setup)s
1206 1207
import sys
sys.path[0:0] = [
1208
  %(path)s,
1209
  ]
1210
%(initialization)s
1211 1212 1213
import %(module_name)s

if __name__ == '__main__':
1214
    sys.exit(%(module_name)s.%(attrs)s(%(arguments)s))
1215 1216
'''

1217 1218
distutils_script_template = script_header + '''
%(before)s
1219 1220 1221 1222 1223 1224 1225
%(relative_paths_setup)s
import sys
sys.path[0:0] = [
  %(path)s,
  ]
%(initialization)s

1226
%(after)s'''
1227

1228

1229
def _pyscript(path, dest, rsetup, initialization=''):
Jim Fulton's avatar
Jim Fulton committed
1230
    generated = []
1231
    script = dest
1232
    if is_win32:
1233 1234
        dest += '-script.py'

Jim Fulton's avatar
Jim Fulton committed
1235
    python = _safe_arg(sys.executable)
Reinout van Rees's avatar
Reinout van Rees committed
1236 1237
    if path:
        path += ','  # Courtesy comma at the end of the list.
Jim Fulton's avatar
Jim Fulton committed
1238

1239
    contents = py_script_template % dict(
Jim Fulton's avatar
Jim Fulton committed
1240
        python = python,
1241
        path = path,
1242
        relative_paths_setup = rsetup,
1243
        initialization=initialization,
1244
        )
1245
    changed = _file_changed(dest, contents)
1246

1247
    if is_win32:
Jim Fulton's avatar
Jim Fulton committed
1248
        # generate exe file and give the script a magic name:
1249
        exe = script + '.exe'
1250
        with open(exe, 'wb') as f:
1251
            f.write(
1252
                pkg_resources.resource_string('setuptools', 'cli.exe')
Jim Fulton's avatar
Jim Fulton committed
1253
            )
1254
        generated.append(exe)
Jim Fulton's avatar
Jim Fulton committed
1255

1256
    if changed:
1257 1258
        with open(dest, 'w') as f:
            f.write(contents)
1259
        try:
1260
            os.chmod(dest, _execute_permission())
1261 1262 1263 1264
        except (AttributeError, os.error):
            pass
        logger.info("Generated interpreter %r.", script)

Jim Fulton's avatar
Jim Fulton committed
1265 1266
    generated.append(dest)
    return generated
1267

Georgy Berdyshev's avatar
Georgy Berdyshev committed
1268 1269
py_script_template = script_header + '''\

1270
%(relative_paths_setup)s
1271
import sys
1272

1273
sys.path[0:0] = [
Reinout van Rees's avatar
Reinout van Rees committed
1274
  %(path)s
1275
  ]
1276
%(initialization)s
1277

Jim Fulton's avatar
Jim Fulton committed
1278 1279
_interactive = True
if len(sys.argv) > 1:
Jim Fulton's avatar
Jim Fulton committed
1280
    _options, _args = __import__("getopt").getopt(sys.argv[1:], 'ic:m:')
Jim Fulton's avatar
Jim Fulton committed
1281 1282 1283 1284 1285
    _interactive = False
    for (_opt, _val) in _options:
        if _opt == '-i':
            _interactive = True
        elif _opt == '-c':
1286
            exec(_val)
Jim Fulton's avatar
Jim Fulton committed
1287 1288 1289 1290 1291
        elif _opt == '-m':
            sys.argv[1:] = _args
            _args = []
            __import__("runpy").run_module(
                 _val, {}, "__main__", alter_sys=True)
1292

Jim Fulton's avatar
Jim Fulton committed
1293 1294
    if _args:
        sys.argv[:] = _args
Jim Fulton's avatar
Jim Fulton committed
1295
        __file__ = _args[0]
Jim Fulton's avatar
Jim Fulton committed
1296
        del _options, _args
1297 1298
        with open(__file__, 'U') as __file__f:
            exec(compile(__file__f.read(), __file__, "exec"))
Jim Fulton's avatar
Jim Fulton committed
1299 1300

if _interactive:
Jim Fulton's avatar
Jim Fulton committed
1301 1302
    del _interactive
    __import__("code").interact(banner="", local=globals())
1303
'''
1304

1305 1306
runsetup_template = """
import sys
1307
sys.path.insert(0, %(setupdir)r)
1308
sys.path.insert(0, %(setuptools)r)
1309

1310
import os, setuptools
Jim Fulton's avatar
Jim Fulton committed
1311

1312
__file__ = %(__file__)r
Jim Fulton's avatar
Jim Fulton committed
1313

1314 1315
os.chdir(%(setupdir)r)
sys.argv[0] = %(setup)r
1316

1317 1318
with open(%(setup)r, 'U') as f:
    exec(compile(f.read(), %(setup)r, 'exec'))
1319
"""
1320

1321

1322 1323 1324
class VersionConflict(zc.buildout.UserError):

    def __init__(self, err, ws):
1325 1326
        ws = list(ws)
        ws.sort()
1327 1328 1329
        self.err, self.ws = err, ws

    def __str__(self):
Jim Fulton's avatar
Jim Fulton committed
1330
        existing_dist, req = self.err.args
1331 1332 1333 1334 1335
        result = ["There is a version conflict.",
                  "We already have: %s" % existing_dist,
                  ]
        for dist in self.ws:
            if req in dist.requires():
1336
                result.append("but %s requires %r." % (dist, str(req)))
1337 1338
        return '\n'.join(result)

1339

1340 1341 1342
class MissingDistribution(zc.buildout.UserError):

    def __init__(self, req, ws):
1343 1344
        ws = list(ws)
        ws.sort()
1345 1346 1347 1348
        self.data = req, ws

    def __str__(self):
        req, ws = self.data
1349
        return "Couldn't find a distribution for %r." % str(req)
1350

1351
def _log_requirement(ws, req):
1352
    if (not logger.isEnabledFor(logging.DEBUG) and
1353
        not Installer._store_required_by):
1354
        # Sorting the working set and iterating over it's requirements
1355
        # is expensive, so short circuit the work if it won't even be
1356 1357 1358 1359 1360
        # logged.  When profiling a simple buildout with 10 parts with
        # identical and large working sets, this resulted in a
        # decrease of run time from 93.411 to 15.068 seconds, about a
        # 6 fold improvement.
        return
Jim Fulton's avatar
Jim Fulton committed
1361

1362 1363
    ws = list(ws)
    ws.sort()
1364
    for dist in ws:
1365 1366
        if req in dist.requires():
            logger.debug("  required by %s." % dist)
1367 1368 1369 1370
            req_ = str(req)
            if req_ not in Installer._required_by:
                Installer._required_by[req_] = set()
            Installer._required_by[req_].add(str(dist.as_requirement()))
1371

1372 1373 1374 1375 1376 1377 1378
def _fix_file_links(links):
    for link in links:
        if link.startswith('file://') and link[-1] != '/':
            if os.path.isdir(link[7:]):
                # work around excessive restriction in setuptools:
                link += '/'
        yield link
1379

1380 1381 1382 1383 1384 1385
_final_parts = '*final-', '*final'
def _final_version(parsed_version):
    for part in parsed_version:
        if (part[:1] == '*') and (part not in _final_parts):
            return False
    return True
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396

def redo_pyc(egg):
    if not os.path.isdir(egg):
        return
    for dirpath, dirnames, filenames in os.walk(egg):
        for filename in filenames:
            if not filename.endswith('.py'):
                continue
            filepath = os.path.join(dirpath, filename)
            if not (os.path.exists(filepath+'c')
                    or os.path.exists(filepath+'o')):
1397
                # If it wasn't compiled, it may not be compilable
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
                continue

            # OK, it looks like we should try to compile.

            # Remove old files.
            for suffix in 'co':
                if os.path.exists(filepath+suffix):
                    os.remove(filepath+suffix)

            # Compile under current optimization
            try:
                py_compile.compile(filepath)
            except py_compile.PyCompileError:
                logger.warning("Couldn't compile %s", filepath)
            else:
                # Recompile under other optimization. :)
1414
                args = [sys.executable]
1415 1416
                if __debug__:
                    args.append('-O')
1417
                args.extend(['-m', 'py_compile', filepath])
1418

1419
                call_subprocess(args)
1420

1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
def _constrained_requirement(constraint, requirement):
    return pkg_resources.Requirement.parse(
        "%s[%s]%s" % (
            requirement.project_name,
            ','.join(requirement.extras),
            _constrained_requirement_constraint(constraint, requirement)
            )
        )

class IncompatibleConstraintError(zc.buildout.UserError):
    """A specified version is incompatible with a given requirement.
    """

1434 1435
IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility

1436 1437 1438 1439 1440
def bad_constraint(constraint, requirement):
    logger.error("The constraint, %s, is not consistent with the "
                 "requirement, %r.", constraint, str(requirement))
    raise IncompatibleConstraintError("Bad constraint", constraint, requirement)

1441
_parse_constraint = re.compile(r'([<>]=?)\s*(\S+)').match
1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453
_comparef = {
    '>' : lambda x, y: x >  y,
    '>=': lambda x, y: x >= y,
    '<' : lambda x, y: x <  y,
    '<=': lambda x, y: x <= y,
    }
_opop = {'<': '>', '>': '<'}
_opeqop = {'<': '>=', '>': '<='}
def _constrained_requirement_constraint(constraint, requirement):

    # Simple cases:

pombredanne's avatar
pombredanne committed
1454
    # No specs to merge with:
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
    if not requirement.specs:
        if not constraint[0] in '<=>':
            constraint = '==' + constraint
        return constraint

    # Simple single-version constraint:
    if constraint[0] not in '<>':
        if constraint.startswith('='):
            assert constraint.startswith('==')
            constraint = constraint[2:]
        if constraint in requirement:
            return '=='+constraint
        bad_constraint(constraint, requirement)


    # OK, we have a complex constraint (<. <=, >=, or >) and specs.
    # In many cases, the spec needs to filter constraints.
    # In other cases, the constraints need to limit the constraint.

    specs = requirement.specs
    cop, cv = _parse_constraint(constraint).group(1, 2)
    pcv = pkg_resources.parse_version(cv)

    # Special case, all of the specs are == specs:
    if not [op for (op, v) in specs if op != '==']:
        # There aren't any non-== specs.

        # See if any of the specs satisfy the constraint:
        specs = [op+v for (op, v) in specs
                 if _comparef[cop](pkg_resources.parse_version(v), pcv)]
        if specs:
            return ','.join(specs)

        bad_constraint(constraint, requirement)

    cop0 = cop[0]

pombredanne's avatar
pombredanne committed
1492 1493
    # Normalize specs by splitting >= and <= specs. We need to do this
    # because these have really weird semantics. Also cache parsed
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
    # versions, which we'll need for comparisons:
    specs = []
    for op, v in requirement.specs:
        pv = pkg_resources.parse_version(v)
        if op == _opeqop[cop0]:
            specs.append((op[0], v, pv))
            specs.append(('==', v, pv))
        else:
            specs.append((op, v, pv))

    # Error if there are opposite specs that conflict with the constraint
    # and there are no equal specs that satisfy the constraint:
    if [v for (op, v, pv) in specs
        if op == _opop[cop0] and _comparef[_opop[cop0]](pv, pcv)
        ]:
        eqspecs = [op+v for (op, v, pv) in specs
                   if _comparef[cop](pv, pcv)]
        if eqspecs:
            # OK, we do, use these:
            return ','.join(eqspecs)

        bad_constraint(constraint, requirement)

    # We have a combination of range constraints and eq specs that
    # satisfy the requirement.

    # Return the constraint + the filtered specs
    return ','.join(
        op+v
        for (op, v) in (
            [(cop, cv)] +
            [(op, v) for (op, v, pv) in specs if _comparef[cop](pv, pcv)]
            )
        )