Commit fbb8c37b authored by Leonardo Rochael Almeida's avatar Leonardo Rochael Almeida Committed by Jim Fulton

Installer: Encapsulate all uses of `._env` and `._path` (#352)

* buildout.cfg: Remove redundancy

Share the eggs definition between the the online and offline tests.

* Remove redundant code

Trying to insert `self._dest` in `self._path`  in `Installer.install()`
is unnecessary since:

 * it's already done in `__init__()`

 * nothing changes either `self._path` or `self._dest` during the
   lifetime of `Installer`

Remove `Installer._load_dist(self, dist)` since it has never been
used in the whole history of Buildout.

Finally, simplify the interface to `._get_dist()` and  `_call_easy_install()`:

All callers to `._get_dist()` add the resulting dist to the WorkingSet. No
reason to do it in `._get_dist()` or to pass the `ws` into
`._call_easy_install()`.

* Be stricter when adding setuptools as requirements

Only do it for namespace package dists that actually need it. I.e. those
that use `pkg_resources.declare_namespace()` instead of
`pkg_util.extend_path()` or PEP 420.

* easy_install: `eggs-directory` may contain more than eggs

Modify the `easy_install.Installer` class to locate not only eggs, but
anything that has a `.dist-info` inside `._dest` (i.e. the Buildout
`eggs-directory`).

This makes it easier for extensions like `buildout.wheel` to add non eggs
inside it: by installing a distribution inside a subdirectory of `._dest` as if
that subdirectory was `site-packages`, you can later locate that
distribution with the Installer.

Conversely, make sure any distribution found in a direct child of `.dest` is
treated as if it was an `egg` distribution (as opposed to a develop
distribution or a site-packages distribution).

All access to `Installer._path` and all access that modifies `._env` is now
done through `Installer` methods, so that Buildout extensions can install
subclasses of `Installer` that alter their behaviour.

* Refactor `_get_dist()` and `_call_easy_install()`

Move the actual invocation of `easy_install` to a module global function
`call_easy_install()`. Simplify its signature so it's simpler to
override in Buildout extensions.

Refactor `._get_dist()` and remove all unpack/install logic into
`_move_to_eggs_dir_and_compile()`, bypassing
`Installer._call_easy_install()`.

`._get_dist()` now calls directly `call_easy_install()` instead. But
only as a fallback to a dictionary lookup of filename extensions to
distribution unpacking methods like:

 - `.egg`: `unpack_egg()`
 - `.whl`: `unpack_wheel()`

This will make it easier for extensions to add support to new dist
formats.
parent ee151de5
......@@ -25,9 +25,7 @@ eggs =
# Tests that can be run wo a network
[oltest]
recipe = zc.recipe.testrunner
eggs =
zc.buildout[test]
zc.recipe.egg
eggs = ${test:eggs}
defaults =
[
'-t',
......
......@@ -169,6 +169,51 @@ def _execute_permission():
_easy_install_cmd = 'from setuptools.command.easy_install import main; main()'
def get_namespace_package_paths(dist):
"""
Generator of the expected pathname of each __init__.py file of the
namespaces of a distribution.
"""
base = [dist.location]
init = ['__init__.py']
for namespace in dist.get_metadata_lines('namespace_packages.txt'):
yield os.path.join(*(base + namespace.split('.') + init))
def namespace_packages_need_pkg_resources(dist):
if os.path.isfile(dist.location):
# Zipped egg, with namespaces, surely needs setuptools
return True
# If they have `__init__.py` files that use pkg_resources and don't
# fallback to using `pkgutil`, then they need setuptools/pkg_resources:
for path in get_namespace_package_paths(dist):
if os.path.isfile(path):
with open(path, 'rb') as f:
source = f.read()
if (source and
b'pkg_resources' in source and
not b'pkgutil' in source):
return True
return False
def dist_needs_pkg_resources(dist):
"""
A distribution needs setuptools/pkg_resources added as requirement if:
* It has namespace packages declared with:
- `pkg_resources.declare_namespace()`
* Those namespace packages don't fall back to `pkgutil`
* It doesn't have `setuptools/pkg_resources` as requirement already
"""
return (
dist.has_metadata('namespace_packages.txt') and
# This will need to change when `pkg_resources` gets its own
# project:
'setuptools' not in {r.project_name for r in dist.requires()} and
namespace_packages_need_pkg_resources(dist)
)
class Installer:
_versions = {}
......@@ -195,7 +240,7 @@ class Installer:
check_picked=True,
):
assert executable == sys.executable, (executable, sys.executable)
self._dest = dest
self._dest = dest if dest is None else pkg_resources.normalize_path(dest)
self._allow_hosts = allow_hosts
if self._install_from_cache:
......@@ -213,13 +258,11 @@ class Installer:
self._index_url = index
path = (path and path[:] or []) + buildout_and_setuptools_path
if dest is not None and dest not in path:
path.insert(0, dest)
self._path = path
if self._dest is None:
newest = False
self._newest = newest
self._env = pkg_resources.Environment(path)
self._env = self._make_env()
self._index = _get_index(index, links, self._allow_hosts)
self._requirements_and_constraints = []
self._check_picked = check_picked
......@@ -227,6 +270,39 @@ class Installer:
if versions is not None:
self._versions = normalize_versions(versions)
def _make_env(self):
full_path = self._get_dest_dist_paths() + self._path
env = pkg_resources.Environment(full_path)
# this needs to be called whenever self._env is modified (or we could
# make an Environment subclass):
self._eggify_env_dest_dists(env, self._dest)
return env
def _env_rescan_dest(self):
self._env.scan(self._get_dest_dist_paths())
self._eggify_env_dest_dists(self._env, self._dest)
def _get_dest_dist_paths(self):
dest = self._dest
if dest is None:
return []
eggs = glob.glob(os.path.join(dest, '*.egg'))
dists = [os.path.dirname(dist_info) for dist_info in
glob.glob(os.path.join(dest, '*', '*.dist-info'))]
# sort them like pkg_resources.find_on_path() would
return pkg_resources._by_version_descending(set(eggs + dists))
@staticmethod
def _eggify_env_dest_dists(env, dest):
"""
Make sure everything found under `dest` is seen as an egg, even if it's
some other kind of dist.
"""
for project_name in env:
for dist in env[project_name]:
if os.path.dirname(dist.location) == dest:
dist.precedence = pkg_resources.EGG_DIST
def _version_conflict_information(self, name):
"""Return textual requirements/constraint information for debug purposes
......@@ -330,48 +406,17 @@ class Installer:
str(req))
return best_we_have, None
def _load_dist(self, dist):
dists = pkg_resources.Environment(dist.location)[dist.project_name]
assert len(dists) == 1
return dists[0]
def _call_easy_install(self, spec, ws, dest, dist):
def _call_easy_install(self, spec, dest, dist):
tmp = tempfile.mkdtemp(dir=dest)
try:
path = setuptools_path
args = [sys.executable, '-c',
('import sys; sys.path[0:0] = %r; ' % path) +
_easy_install_cmd, '-mZUNxd', tmp]
level = logger.getEffectiveLevel()
if level > 0:
args.append('-q')
elif level < 0:
args.append('-v')
args.append(spec)
if level <= logging.DEBUG:
logger.debug('Running easy_install:\n"%s"\npath=%s\n',
'" "'.join(args), path)
sys.stdout.flush() # We want any pending output first
exit_code = subprocess.call(list(args))
paths = call_easy_install(spec, tmp)
dists = []
env = pkg_resources.Environment([tmp])
env = pkg_resources.Environment(paths)
for project in env:
dists.extend(env[project])
if exit_code:
logger.error(
"An error occurred when trying to install %s. "
"Look above this message for any errors that "
"were output by easy_install.",
dist)
if not dists:
raise zc.buildout.UserError("Couldn't install: %s" % dist)
......@@ -397,9 +442,7 @@ class Installer:
result = []
for d in dists:
newloc = _move_to_eggs_dir_and_compile(d, dest)
[d] = pkg_resources.Environment([newloc])[d.project_name]
result.append(d)
result.append(_move_to_eggs_dir_and_compile(d, dest))
return result
......@@ -482,7 +525,7 @@ class Installer:
return dist.clone(location=new_location)
def _get_dist(self, requirement, ws, for_buildout_run=False):
def _get_dist(self, requirement, ws):
__doing__ = 'Getting distribution for %r.', str(requirement)
# Maybe an existing dist is already the best dist that satisfies the
......@@ -518,37 +561,13 @@ class Installer:
raise zc.buildout.UserError(
"Couldn't download distribution %s." % avail)
if dist.location.endswith('.whl'):
dist = wheel_to_egg(dist, tmp)
if dist.precedence == pkg_resources.EGG_DIST:
# It's already an egg, just fetch it into the dest
newloc = _move_to_eggs_dir_and_compile(dist, self._dest)
# Getting the dist from the environment causes the
# distribution meta data to be read. Cloning isn't
# good enough.
dists = pkg_resources.Environment([newloc])[
dist.project_name]
else:
# It's some other kind of dist. We'll let easy_install
# deal with it:
dists = self._call_easy_install(
dist.location, ws, self._dest, dist)
for dist in dists:
if for_buildout_run:
# ws is the global working set and we're
# installing buildout, setuptools, extensions or
# recipes. Make sure that whatever correct version
# we've just installed is the active version,
# hence the ``replace=True``.
ws.add(dist, replace=True)
dists = [_move_to_eggs_dir_and_compile(dist, self._dest)]
finally:
if tmp != self._download_cache:
shutil.rmtree(tmp)
self._env.scan([self._dest])
self._env_rescan_dest()
dist = self._env.best_match(requirement, ws)
logger.info("Got %s.", dist)
......@@ -556,23 +575,32 @@ class Installer:
dists = [dist]
if not self._install_from_cache and self._use_dependency_links:
self._add_dependency_links_from_dists(dists)
if self._check_picked:
self._check_picked_requirement_versions(requirement, dists)
return dists
def _add_dependency_links_from_dists(self, dists):
reindex = False
links = self._links
for dist in dists:
if dist.has_metadata('dependency_links.txt'):
for link in dist.get_metadata_lines('dependency_links.txt'):
link = link.strip()
if link not in self._links:
if link not in links:
logger.debug('Adding find link %r from %s',
link, dist)
self._links.append(link)
self._index = _get_index(self._index_url,
self._links,
self._allow_hosts)
links.append(link)
reindex = True
if reindex:
self._index = _get_index(self._index_url, links, self._allow_hosts)
if self._check_picked:
# Check whether we picked a version and, if we did, report it:
def _check_picked_requirement_versions(self, requirement, dists):
""" Check whether we picked a version and, if we did, report it """
for dist in dists:
if not (
dist.precedence == pkg_resources.DEVELOP_DIST
if not (dist.precedence == pkg_resources.DEVELOP_DIST
or
(len(requirement.specs) == 1
and
......@@ -588,14 +616,8 @@ class Installer:
dist.version)
)
return dists
def _maybe_add_setuptools(self, ws, dist):
if dist.has_metadata('namespace_packages.txt'):
for r in dist.requires():
if r.project_name in ('setuptools', 'setuptools'):
break
else:
if dist_needs_pkg_resources(dist):
# We have a namespace package but no requirement for setuptools
if dist.precedence == pkg_resources.DEVELOP_DIST:
logger.warn(
......@@ -610,7 +632,6 @@ class Installer:
for dist in self._get_dist(requirement, ws):
ws.add(dist)
def _constrain(self, requirement):
"""Return requirement with optional [versions] constraint added."""
constraint = self._versions.get(requirement.project_name.lower())
......@@ -632,10 +653,6 @@ class Installer:
"Base installation request: %s" % repr(specs)[1:-1])
for_buildout_run = bool(working_set)
path = self._path
dest = self._dest
if dest is not None and dest not in path:
path.insert(0, dest)
requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
for spec in specs]
......@@ -646,8 +663,7 @@ class Installer:
ws = working_set
for requirement in requirements:
for dist in self._get_dist(requirement, ws,
for_buildout_run=for_buildout_run):
for dist in self._get_dist(requirement, ws):
ws.add(dist)
self._maybe_add_setuptools(ws, dist)
......@@ -691,13 +707,12 @@ class Installer:
if not for_buildout_run:
raise VersionConflict(err, ws)
if dist is None:
if dest:
if self._dest:
logger.debug('Getting required %r', str(req))
else:
logger.debug('Adding required %r', str(req))
self._log_requirement(ws, req)
for dist in self._get_dist(req, ws,
for_buildout_run=for_buildout_run):
for dist in self._get_dist(req, ws):
ws.add(dist)
self._maybe_add_setuptools(ws, dist)
if dist not in req:
......@@ -775,9 +790,7 @@ class Installer:
setuptools.command.setopt.edit_config(
setup_cfg, dict(build_ext=build_ext))
dists = self._call_easy_install(
base, pkg_resources.WorkingSet(),
self._dest, dist)
dists = self._call_easy_install(base, self._dest, dist)
return [dist.location for dist in dists]
finally:
......@@ -1564,6 +1577,84 @@ class IncompatibleConstraintError(zc.buildout.UserError):
IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility
def call_easy_install(spec, dest):
"""
Call `easy_install` from setuptools as a subprocess to install a
distribution specified by `spec` into `dest`.
Returns all the paths inside `dest` created by the above.
"""
path = setuptools_path
args = [sys.executable, '-c',
('import sys; sys.path[0:0] = %r; ' % path) +
_easy_install_cmd, '-mZUNxd', dest]
level = logger.getEffectiveLevel()
if level > 0:
args.append('-q')
elif level < 0:
args.append('-v')
args.append(spec)
if level <= logging.DEBUG:
logger.debug('Running easy_install:\n"%s"\npath=%s\n',
'" "'.join(args), path)
sys.stdout.flush() # We want any pending output first
exit_code = subprocess.call(list(args))
if exit_code:
logger.error(
"An error occurred when trying to install %s. "
"Look above this message for any errors that "
"were output by easy_install.",
spec)
return glob.glob(os.path.join(dest, '*'))
def unpack_egg(location, dest):
# Buildout 2 no longer installs zipped eggs,
# so we always want to unpack it.
dest = os.path.join(dest, os.path.basename(location))
setuptools.archive_util.unpack_archive(location, dest)
WHEEL_TO_EGG_WARNING = """
Using unpack_wheel() shim over deprecated wheel_to_egg().
Please update your wheel extension implementation for one that installs a .whl
handler in %s.UNPACKERS
""".strip() % (__name__,)
def unpack_wheel(location, dest):
logger.warning(WHEEL_TO_EGG_WARNING)
# Deprecated backward compatibility shim. Please do not use.
basename = os.path.basename(location)
dists = setuptools.package_index.distros_for_location(location, basename)
wheel_to_egg(list(dists)[0], dest)
UNPACKERS = {
# Buildout 2 no longer installs zipped eggs, so we always want to unpack it.
'.egg': unpack_egg,
'.whl': unpack_wheel,
}
def _get_matching_dist_in_location(dist, location):
"""
Check if `locations` contain only the one intended dist.
Return the dist with metadata in the new location.
"""
# Getting the dist from the environment causes the
# distribution meta data to be read. Cloning isn't
# good enough.
env = pkg_resources.Environment([location])
dists = [ d for project_name in env for d in env[project_name] ]
dist_infos = [ (d.project_name, d.version) for d in dists ]
if dist_infos == [(dist.project_name, dist.version)]:
return dists.pop()
def _move_to_eggs_dir_and_compile(dist, dest):
"""Move distribution to the eggs destination directory.
......@@ -1575,7 +1666,7 @@ def _move_to_eggs_dir_and_compile(dist, dest):
running in parallel. So we copy to a temporary directory first.
See discussion at https://github.com/buildout/buildout/issues/307
We return the new location.
We return the new distribution with properly loaded metadata.
"""
# First make sure the destination directory exists. This could suffer from
# the same kind of race condition as the rest: if we check that it does not
......@@ -1587,23 +1678,26 @@ def _move_to_eggs_dir_and_compile(dist, dest):
if not os.path.isdir(dest):
# Unknown reason. Reraise original error.
raise
newloc = os.path.join(
dest, os.path.basename(dist.location))
tmp_dest = tempfile.mkdtemp(dir=dest)
try:
tmp_egg_dir = os.path.join(tmp_dest, os.path.basename(dist.location))
if os.path.isdir(dist.location):
# We got a directory. It must have been obtained locally.
if (os.path.isdir(dist.location) and
dist.precedence >= pkg_resources.BINARY_DIST):
# We got a pre-built directory. It must have been obtained locally.
# Just copy it.
shutil.copytree(dist.location, tmp_egg_dir)
tmp_loc = os.path.join(tmp_dest, os.path.basename(dist.location))
shutil.copytree(dist.location, tmp_loc)
else:
# It is a zipped egg. Buildout 2 no longer installs zipped eggs,
# so we always want to unpack it.
setuptools.archive_util.unpack_archive(
dist.location, tmp_egg_dir)
# We have copied the egg. Now try to rename/move it.
# It is an archive of some sort.
# Figure out how to unpack it, or fall back to easy_install.
_, ext = os.path.splitext(dist.location)
unpacker = UNPACKERS.get(ext, call_easy_install)
unpacker(dist.location, tmp_dest)
[tmp_loc] = glob.glob(os.path.join(tmp_dest, '*'))
# We have installed the dist. Now try to rename/move it.
newloc = os.path.join(dest, os.path.basename(tmp_loc))
try:
os.rename(tmp_egg_dir, newloc)
os.rename(tmp_loc, newloc)
except OSError:
# Might be for various reasons. If it is because newloc already
# exists, we can investigate.
......@@ -1611,7 +1705,8 @@ def _move_to_eggs_dir_and_compile(dist, dest):
# No, it is a different reason. Give up.
raise
# Try to use it as environment and check if our project is in it.
if not pkg_resources.Environment([newloc])[dist.project_name]:
newdist = _get_matching_dist_in_location(dist, newloc)
if newdist is None:
# Path exists, but is not our package. We could
# try something, but it seems safer to bail out
# with the original error.
......@@ -1627,7 +1722,9 @@ def _move_to_eggs_dir_and_compile(dist, dest):
# There were no problems during the rename.
# Do the compile step.
redo_pyc(newloc)
newdist = _get_matching_dist_in_location(dist, newloc)
assert newdist is not None # newloc above is missing our dist?!
finally:
# Remember that temporary directories must be removed
shutil.rmtree(tmp_dest)
return newloc
return newdist
......@@ -836,12 +836,15 @@ and we will get setuptools included in the working set.
... 'zc.buildout.easy_install', level=logging.WARNING)
>>> logging.getLogger('zc.buildout.easy_install').propagate = False
>>> [dist.project_name
>>> def get_working_set(*project_names):
... paths = [join(sample_buildout, 'eggs'),
... join(sample_buildout, 'develop-eggs')]
... return [
... dist.project_name
... for dist in zc.buildout.easy_install.working_set(
... ['foox'], sys.executable,
... [join(sample_buildout, 'eggs'),
... join(sample_buildout, 'develop-eggs'),
... ])]
... project_names, sys.executable, paths)
... ]
>>> get_working_set('foox')
['foox', 'setuptools']
>>> print_(handler)
......@@ -851,13 +854,15 @@ and we will get setuptools included in the working set.
>>> handler.clear()
On the other hand, if we have a regular egg, rather than a develop egg:
On the other hand, if we have a zipped egg, rather than a develop egg:
>>> os.remove(join('develop-eggs', 'foox.egg-link'))
>>> _ = system(join('bin', 'buildout') + ' setup foo bdist_egg -d'
... + join(sample_buildout, 'eggs'))
>>> _ = system(join('bin', 'buildout') + ' setup foo bdist_egg')
>>> foox_dist = join('foo', 'dist')
>>> import glob
>>> [foox_egg] = glob.glob(join(foox_dist, 'foox-*.egg'))
>>> _ = shutil.copy(foox_egg, join(sample_buildout, 'eggs'))
>>> ls('develop-eggs')
- zc.recipe.egg.egg-link
......@@ -870,17 +875,27 @@ On the other hand, if we have a regular egg, rather than a develop egg:
We do not get a warning, but we do get setuptools included in the working set:
>>> [dist.project_name
... for dist in zc.buildout.easy_install.working_set(
... ['foox'], sys.executable,
... [join(sample_buildout, 'eggs'),
... join(sample_buildout, 'develop-eggs'),
... ])]
>>> get_working_set('foox')
['foox', 'setuptools']
>>> print_(handler, end='')
Likewise for an unzipped egg:
>>> foox_egg_basename = os.path.basename(foox_egg)
>>> os.remove(join(sample_buildout, 'eggs', foox_egg_basename))
>>> _ = zc.buildout.easy_install.install(
... ['foox'], join(sample_buildout, 'eggs'), links=[foox_dist],
... index='file://' + foox_dist)
>>> ls('develop-eggs')
- zc.recipe.egg.egg-link
>>> get_working_set('foox')
['foox', 'setuptools']
>>> print_(handler, end='')
We get the same behavior if the it is a dependency that uses a
We get the same behavior if it is a dependency that uses a
namespace package.
......@@ -903,12 +918,7 @@ namespace package.
Develop: '/sample-buildout/foo'
Develop: '/sample-buildout/bar'
>>> [dist.project_name
... for dist in zc.buildout.easy_install.working_set(
... ['bar'], sys.executable,
... [join(sample_buildout, 'eggs'),
... join(sample_buildout, 'develop-eggs'),
... ])]
>>> get_working_set('bar')
['bar', 'foox', 'setuptools']
>>> print_(handler, end='')
......@@ -916,6 +926,32 @@ namespace package.
Develop distribution: foox 0.0.0
uses namespace packages but the distribution does not require setuptools.
On the other hand, if the distribution uses ``pkgutil.extend_path()`` to
implement its namespaces, even if just as fallback from the absence of
``pkg_resources``, then ``setuptools`` shoudn't be added as requirement to
its unzipped egg:
>>> foox_installed_egg = join(sample_buildout, 'eggs', foox_egg_basename)
>>> namespace_init = join(foox_installed_egg, 'stuff', '__init__.py')
>>> write(namespace_init,
... """try:
... __import__('pkg_resources').declare_namespace(__name__)
... except ImportError:
... __path__ = __import__('pkgutil').extend_path(__path__, __name__)
... """)
>>> os.remove(join('develop-eggs', 'foox.egg-link'))
>>> os.remove(join('develop-eggs', 'bar.egg-link'))
>>> get_working_set('foox')
['foox']
The same goes for packages using PEP420 namespaces
>>> os.remove(namespace_init)
>>> get_working_set('foox')
['foox']
Cleanup:
>>> logging.getLogger('zc.buildout.easy_install').propagate = True
>>> handler.uninstall()
......@@ -3155,8 +3191,7 @@ class UnitTests(unittest.TestCase):
def wheel_to_egg(dist, dest):
newloc = os.path.join(dest, egg_name)
shutil.copy(dist.location, newloc)
return pkg_resources.Distribution.from_location(newloc,
'demo-0.3.whl')
return pkg_resources.Distribution.from_filename(newloc)
zc.buildout.easy_install.wheel_to_egg = wheel_to_egg
egg_dir = os.path.join(self.sample_buildout, 'eggs')
self.assertFalse(egg_name in os.listdir(egg_dir))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment