easy_install.py 19.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
##############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Python easy_install API

This module provides a high-level Python API for installing packages.
It doesn't install scripts.  It uses setuptools and requires it to be
installed.

$Id$
"""

23
import glob, logging, os, re, shutil, sys, tempfile, urlparse, zipimport
24 25 26 27 28
import distutils.errors
import pkg_resources
import setuptools.command.setopt
import setuptools.package_index
import setuptools.archive_util
29
import zc.buildout
30

31
default_index_url = os.environ.get('buildout-testing-index-url')
32

33 34
logger = logging.getLogger('zc.buildout.easy_install')

35 36
url_match = re.compile('[a-z0-9+.-]+://').match

37 38
# Include buildout and setuptools eggs in paths
buildout_and_setuptools_path = [
39 40 41 42
    pkg_resources.working_set.find(
        pkg_resources.Requirement.parse('setuptools')).location,
    pkg_resources.working_set.find(
        pkg_resources.Requirement.parse('zc.buildout')).location,
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
    ]

_versions = {sys.executable: '%d.%d' % sys.version_info[:2]}
def _get_version(executable):
    try:
        return _versions[executable]
    except KeyError:
        i, o = os.popen4(executable + ' -V')
        i.close()
        version = o.read().strip()
        o.close()
        pystring, version = version.split()
        assert pystring == 'Python'
        version = re.match('(\d[.]\d)[.]\d$', version).group(1)
        _versions[executable] = version
        return version

60 61 62 63 64 65 66
_indexes = {}
def _get_index(executable, index_url, find_links):
    key = executable, index_url, tuple(find_links)
    index = _indexes.get(key)
    if index is not None:
        return index

67 68 69
    if index_url is None:
        index_url = default_index_url

70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
    if index_url is None:
        index = setuptools.package_index.PackageIndex(
            python=_get_version(executable)
            )
    else:
        index = setuptools.package_index.PackageIndex(
            index_url, python=_get_version(executable)
            )
        
    if find_links:
        index.add_find_links(find_links)

    _indexes[key] = index
    return index

def _satisfied(req, env, dest, executable, index, links):
    dists = [dist for dist in env[req.project_name] if dist in req]
    if not dists:
        logger.debug('We have no distributions for %s', req.project_name)
        return None

    # Note that dists are sorted from best to worst, as promised by
    # env.__getitem__
93 94

    for dist in dists:
95 96 97
        if (dist.precedence == pkg_resources.DEVELOP_DIST):
            logger.debug('We have a develop egg for %s', req)
            return dist
98

99
    # Find an upprt limit in the specs, if there is one:
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
    specs = [(pkg_resources.parse_version(v), op) for (op, v) in req.specs]
    specs.sort()
    maxv = None
    greater = False
    lastv = None
    for v, op in specs:
        if op == '==' and not greater:
            maxv = v
        elif op in ('>', '>=', '!='):
            maxv = None
            greater == True
        elif op == '<':
            maxv = None
            greater == False
        elif op == '<=':
            maxv = v
            greater == False

        if v == lastv:
            # Repeated versions values are undefined, so
            # all bets are off
            maxv = None
            greater = True
        else:
            lastv = v

126 127 128 129
    best_we_have = dists[0] # Because dists are sorted from best to worst

    # Check if we have the upper limit
    if maxv is not None and best_we_have.version == maxv:
Jürgen kartnaller's avatar
Jürgen kartnaller committed
130
        logger.debug('We have the best distribution that satisfies\n%s',
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
                     req)
        return best_we_have

    # We have some installed distros.  There might, theoretically, be
    # newer ones.  Let's find out which ones are available and see if
    # any are newer.  We only do this if we're willing to install
    # something, which is only true if dest is not None:

    if dest is not None:
        best_available = _get_index(executable, index, links).obtain(req)
    else:
        best_available = None

    if best_available is None:
        # That's a bit odd.  There aren't any distros available.
        # We should use the best one we have that meets the requirement.
        logger.debug(
Jürgen kartnaller's avatar
Jürgen kartnaller committed
148
            'There are no distros available that meet %s. Using our best.', req)
149 150 151
        return best_we_have
    else:
        # Let's find out if we already have the best available:
152
        if best_we_have.parsed_version >= best_available.parsed_version:
153
            # Yup. Use it.
Jürgen kartnaller's avatar
Jürgen kartnaller committed
154
            logger.debug('We have the best distribution that satisfies\n%s', req)
155
            return best_we_have
156 157 158

    return None

Jim Fulton's avatar
Jim Fulton committed
159 160 161 162 163 164 165 166 167 168 169 170 171

if sys.platform == 'win32':
    # work around spawn lamosity on windows
    # XXX need safe quoting (see the subproces.list2cmdline) and test
    def _safe_arg(arg):
        return '"%s"' % arg
else:
    _safe_arg = str

_easy_install_cmd = _safe_arg(
    'from setuptools.command.easy_install import main; main()'
    )

172 173
def _call_easy_install(spec, env, ws, dest, links, index,
                       executable, always_unzip):
Jim Fulton's avatar
Jim Fulton committed
174

175 176 177
    path = _get_dist(pkg_resources.Requirement.parse('setuptools'),
                     env, ws, dest, links, index, executable, False).location
 
Jim Fulton's avatar
Jim Fulton committed
178
    args = ('-c', _easy_install_cmd, '-mUNxd', _safe_arg(dest))
179 180
    if always_unzip:
        args += ('-Z', )
181 182 183 184 185 186 187 188 189 190 191
    level = logger.getEffectiveLevel()
    if level > logging.DEBUG:
        args += ('-q', )
    elif level < logging.DEBUG:
        args += ('-v', )
    
    args += (spec, )

    if level <= logging.DEBUG:
        logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
                     executable, '" "'.join(args), path)
Jim Fulton's avatar
Jim Fulton committed
192

193
    args += (dict(os.environ, PYTHONPATH=path), )
194 195
    sys.stdout.flush() # We want any pending output first
    exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
196
    assert exit_code == 0
197 198 199


def _get_dist(requirement, env, ws,
200
              dest, links, index_url, executable, always_unzip):
201
    
202 203
    # Maybe an existing dist is already the best dist that satisfies the
    # requirement
204
    dist = _satisfied(requirement, env, dest, executable, index_url, links)
205 206 207

    if dist is None:
        if dest is not None:
208 209
            logger.info("Getting new distribution for %s", requirement)

210
            # Retrieve the dist:
211
            index = _get_index(executable, index_url, links)
212 213 214
            dist = index.obtain(requirement)
            if dist is None:
                raise zc.buildout.UserError(
215
                    "Couldn't find a distribution for %s."
216
                    % requirement)
217 218 219 220 221 222

            fname = dist.location
            if url_match(fname):
                fname = urlparse.urlparse(fname)[2]
                
            if fname.endswith('.egg'):
223 224 225 226 227 228 229 230
                # It's already an egg, just fetch it into the dest
                tmp = tempfile.mkdtemp('get_dist')
                try:
                    dist = index.fetch_distribution(requirement, tmp)
                    if dist is None:
                        raise zc.buildout.UserError(
                            "Couln't download a distribution for %s."
                            % requirement)
231

Jim Fulton's avatar
Jim Fulton committed
232 233 234 235 236 237 238
                    newloc = os.path.join(
                        dest, os.path.basename(dist.location))

                    if os.path.isdir(dist.location):
                        # we got a directory. It must have been
                        # obtained locally.  Jut copy it.
                        shutil.copytree(dist.location, newloc)
239
                    else:
Jim Fulton's avatar
Jim Fulton committed
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256

                        if always_unzip:
                            should_unzip = True
                        else:
                            metadata = pkg_resources.EggMetadata(
                                zipimport.zipimporter(dist.location)
                                )
                            should_unzip = (
                                metadata.has_metadata('not-zip-safe')
                                or not metadata.has_metadata('zip-safe')
                                )

                        if should_unzip:
                            setuptools.archive_util.unpack_archive(
                                dist.location, newloc)
                        else:
                            shutil.copyfile(dist.location, newloc)
257 258 259 260 261 262 263 264 265 266 267 268 269
                        
                finally:
                    shutil.rmtree(tmp)

            else:
                # It's some other kind of dist.  We'll download it to
                # a temporary directory and let easy_install have it's
                # way with it:
                tmp = tempfile.mkdtemp('get_dist')
                try:
                    dist = index.fetch_distribution(requirement, tmp)

                    # May need a new one.  Call easy_install
270 271 272
                    _call_easy_install(
                        dist.location, env, ws, dest, links, index_url,
                        executable, always_unzip)
273 274 275
                finally:
                    shutil.rmtree(tmp)

276

277 278
            # Because we have added a new egg, we need to rescan
            # the destination directory.
279 280 281 282 283

            # We may overwrite distributions, so clear importer
            # cache.
            sys.path_importer_cache.clear()

284
            env.scan([dest])
285 286 287 288
            dist = env.best_match(requirement, ws)
            logger.info("Got %s", dist)            
        else:
            dist = env.best_match(requirement, ws)
289

290 291 292
    if dist is None:
        raise ValueError("Couldn't find", requirement)

293 294 295 296 297 298
    # XXX Need test for this
    if dist.has_metadata('dependency_links.txt'):
        for link in dist.get_metadata_lines('dependency_links.txt'):
            link = link.strip()
            if link not in links:
                links.append(link)
299
                
300
    return dist
Jim Fulton's avatar
Jim Fulton committed
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321

def _maybe_add_setuptools(ws, dist, env, dest, links, index, executable):
    if dist.has_metadata('namespace_packages.txt'):
        for r in dist.requires():
            if r.project_name == 'setuptools':
                break
        else:
            # We have a namespace package but no requirement for setuptools
            if dist.precedence == pkg_resources.DEVELOP_DIST:
                logger.warn(
                    "Develop distribution for %s\n"
                    "uses namespace packages but the distribution "
                    "does not require setuptools.",
                    dist)
            requirement = pkg_resources.Requirement.parse('setuptools')
            if ws.find(requirement) is None:
                dist = _get_dist(requirement, env, ws,
                                 dest, links, index, executable,
                                 False)
                ws.add(dist)
    
322 323 324 325
    
def install(specs, dest,
            links=(), index=None,
            executable=sys.executable, always_unzip=False,
326
            path=None, working_set=None):
327 328 329 330

    logger.debug('Installing %r', specs)

    path = path and path[:] or []
331
    if dest is not None and dest not in path:
332 333 334 335 336 337 338 339 340 341 342 343 344
        path.insert(0, dest)

    path += buildout_and_setuptools_path

    links = list(links) # make copy, because we may need to mutate
    

    # For each spec, see if it is already installed.  We create a working
    # set to keep track of what we've collected and to make sue than the
    # distributions assembled are consistent.
    env = pkg_resources.Environment(path, python=_get_version(executable))
    requirements = [pkg_resources.Requirement.parse(spec) for spec in specs]

345 346 347 348
    if working_set is None:
        ws = pkg_resources.WorkingSet([])
    else:
        ws = working_set
349 350

    for requirement in requirements:
351
        dist = _get_dist(requirement, env, ws,
352
                         dest, links, index, executable, always_unzip)
353
        ws.add(dist)
Jim Fulton's avatar
Jim Fulton committed
354 355
        _maybe_add_setuptools(ws, dist,
                              env, dest, links, index, executable)
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371

    # OK, we have the requested distributions and they're in the working
    # set, but they may have unmet requirements.  We'll simply keep
    # trying to resolve requirements, adding missing requirements as they
    # are reported.
    #
    # Note that we don't pass in the environment, because we
    # want to look for new eggs unless what we have is the best that matches
    # the requirement.
    while 1:
        try:
            ws.resolve(requirements)
        except pkg_resources.DistributionNotFound, err:
            [requirement] = err
            if dest:
                logger.debug('Getting required %s', requirement)
Jim Fulton's avatar
Jim Fulton committed
372
            dist = _get_dist(requirement, env, ws,
373
                             dest, links, index, executable, always_unzip)
Jim Fulton's avatar
Jim Fulton committed
374 375 376
            ws.add(dist)
            _maybe_add_setuptools(ws, dist,
                                  env, dest, links, index, executable)
377 378 379 380 381
        else:
            break
            
    return ws

382 383 384 385 386
def build(spec, dest, build_ext,
          links=(), index=None,
          executable=sys.executable,
          path=None):

387 388
    index_url = index

389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
    logger.debug('Building %r', spec)

    path = path and path[:] or []
    if dest is not None:
        path.insert(0, dest)

    path += buildout_and_setuptools_path

    links = list(links) # make copy, because we may need to mutate
    
    # For each spec, see if it is already installed.  We create a working
    # set to keep track of what we've collected and to make sue than the
    # distributions assembled are consistent.
    env = pkg_resources.Environment(path, python=_get_version(executable))
    requirement = pkg_resources.Requirement.parse(spec)

405
    dist = _satisfied(requirement, env, dest, executable, index_url, links)
406 407 408 409 410
    if dist is not None:
        return dist

    # Get an editable version of the package to a temporary directory:
    tmp = tempfile.mkdtemp('editable')
411 412
    tmp2 = tempfile.mkdtemp('editable')
    try:
413
        index = _get_index(executable, index_url, links)
414 415 416
        dist = index.fetch_distribution(requirement, tmp2, False, True)
        if dist is None:
            raise zc.buildout.UserError(
417
                "Couldn't find a source distribution for %s."
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
                % requirement)
        setuptools.archive_util.unpack_archive(dist.location, tmp)

        if os.path.exists(os.path.join(tmp, 'setup.py')):
            base = tmp
        else:
            setups = glob.glob(os.path.join(tmp, '*', 'setup.py'))
            if not setups:
                raise distutils.errors.DistutilsError(
                    "Couldn't find a setup script in %s"
                    % os.path.basename(dist.location)
                    )
            if len(setups) > 1:
                raise distutils.errors.DistutilsError(
                    "Multiple setup scripts in %s"
                    % os.path.basename(dist.location)
                    )
            base = os.path.dirname(setups[0])


        setup_cfg = os.path.join(base, 'setup.cfg')
        if not os.path.exists(setup_cfg):
            f = open(setup_cfg, 'w')
            f.close()
        setuptools.command.setopt.edit_config(
            setup_cfg, dict(build_ext=build_ext))

        # Now run easy_install for real:
446 447
        _call_easy_install(base, env, pkg_resources.WorkingSet(),
                           dest, links, index_url, executable, True)
448 449 450
    finally:
        shutil.rmtree(tmp)
        shutil.rmtree(tmp2)
451

452 453 454
def working_set(specs, executable, path):
    return install(specs, None, executable=executable, path=path)

455 456 457 458
def scripts(reqs, working_set, executable, dest,
            scripts=None,
            extra_paths=(),
            arguments='',
459
            interpreter=None,
460
            initialization='',
461
            ):
462
    
463 464
    path = [dist.location for dist in working_set]
    path.extend(extra_paths)
Jim Fulton's avatar
Jim Fulton committed
465
    path = repr(path)[1:-1].replace(', ', ',\n  ')
466 467
    generated = []

468 469 470 471
    if isinstance(reqs, str):
        raise TypeError('Expected iterable of requirements or entry points,'
                        ' got string.')

472 473 474
    if initialization:
        initialization = '\n'+initialization+'\n'

475 476 477 478 479
    entry_points = []
    for req in reqs:
        if isinstance(req, str):
            req = pkg_resources.Requirement.parse(req)
            dist = working_set.find(req)
480
            for name in pkg_resources.get_entry_map(dist, 'console_scripts'):
481 482
                entry_point = dist.get_entry_info('console_scripts', name)
                entry_points.append(
483 484
                    (name, entry_point.module_name,
                     '.'.join(entry_point.attrs))
Jim Fulton's avatar
Jim Fulton committed
485
                    )
486 487 488 489 490 491 492 493 494 495
        else:
            entry_points.append(req)
                
    for name, module_name, attrs in entry_points:
        if scripts is not None:
            sname = scripts.get(name)
            if sname is None:
                continue
        else:
            sname = name
496

497 498
        sname = os.path.join(dest, sname)
        generated.extend(
499 500
            _script(module_name, attrs, path, sname, executable, arguments,
                    initialization)
501
            )
502

503 504 505
    if interpreter:
        sname = os.path.join(dest, interpreter)
        generated.extend(_pyscript(path, sname, executable))
506 507 508

    return generated

509 510
def _script(module_name, attrs, path, dest, executable, arguments,
            initialization):
Jim Fulton's avatar
Jim Fulton committed
511 512 513 514 515 516 517 518 519
    generated = []
    if sys.platform == 'win32':
        # generate exe file and give the script a magic name:
        open(dest+'.exe', 'wb').write(
            pkg_resources.resource_string('setuptools', 'cli.exe')
            )
        generated.append(dest+'.exe')
        dest += '-script.py'
        
520 521 522
    open(dest, 'w').write(script_template % dict(
        python = executable,
        path = path,
523 524
        module_name = module_name,
        attrs = attrs,
525
        arguments = arguments,
526
        initialization = initialization,
527 528 529 530 531
        ))
    try:
        os.chmod(dest, 0755)
    except (AttributeError, os.error):
        pass
Jim Fulton's avatar
Jim Fulton committed
532 533
    generated.append(dest)
    return generated
534 535 536 537 538 539

script_template = '''\
#!%(python)s

import sys
sys.path[0:0] = [
540
  %(path)s,
541
  ]
542
%(initialization)s
543 544 545
import %(module_name)s

if __name__ == '__main__':
546
    %(module_name)s.%(attrs)s(%(arguments)s)
547 548 549 550
'''


def _pyscript(path, dest, executable):
Jim Fulton's avatar
Jim Fulton committed
551 552 553 554 555 556 557 558 559
    generated = []
    if sys.platform == 'win32':
        # generate exe file and give the script a magic name:
        open(dest+'.exe', 'wb').write(
            pkg_resources.resource_string('setuptools', 'cli.exe')
            )
        generated.append(dest+'.exe')
        dest += '-script.py'

560 561 562 563 564 565 566 567
    open(dest, 'w').write(py_script_template % dict(
        python = executable,
        path = path,
        ))
    try:
        os.chmod(dest,0755)
    except (AttributeError, os.error):
        pass
Jim Fulton's avatar
Jim Fulton committed
568 569
    generated.append(dest)
    return generated
570 571

py_script_template = '''\
572
#!%(python)s
573
import sys
574
    
575
sys.path[0:0] = [
576
  %(path)s,
577
  ]
578

Jim Fulton's avatar
Jim Fulton committed
579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
_interactive = True
if len(sys.argv) > 1:
    import getopt
    _options, _args = getopt.getopt(sys.argv[1:], 'ic:')
    _interactive = False
    for (_opt, _val) in _options:
        if _opt == '-i':
            _interactive = True
        elif _opt == '-c':
            exec _val
            
    if _args:
        sys.argv[:] = _args
        execfile(sys.argv[0])

if _interactive:
    import code
    code.interact(banner="", local=globals())
597
'''
Jim Fulton's avatar
Jim Fulton committed
598 599 600