Commit b08cf932 authored by Xavier Thompson's avatar Xavier Thompson

[feat] Use pip wheel + Wheel.install_as_egg

`pip install <package>` produces a `<package-name>` package folder
and a `<package-name>.dist-info` metadata folder, which is another
format than eggs. Then buildout bundles both folders into a parent
folder `<package.egg>` and tries to act as though it were an egg.

Instead, use `pip wheel` to produce a wheel - which `pip install`
does internally anyway - and `setuptools.Wheel.install_as_egg` to
produce a genuine egg.

This is much cleaner: it consistently produces genuine eggs instead
of sometimes true eggs, sometimes `.dist-info` bundles depending on
whether `pip install` is called or the package was installed from a
`.whl` or `.egg` archive directly.

The only downside it this requires setuptools >= 38.2.3.
parent 58b8ec8d
...@@ -295,7 +295,7 @@ class Installer(object): ...@@ -295,7 +295,7 @@ class Installer(object):
self._versions = normalize_versions(versions) self._versions = normalize_versions(versions)
def _make_env(self): def _make_env(self):
full_path = self._get_dest_dist_paths() + self._get_path_dist_paths() + self._path full_path = self._get_dest_dist_paths() + self._path
env = pkg_resources.Environment(full_path) env = pkg_resources.Environment(full_path)
# this needs to be called whenever self._env is modified (or we could # this needs to be called whenever self._env is modified (or we could
# make an Environment subclass): # make an Environment subclass):
...@@ -310,18 +310,9 @@ class Installer(object): ...@@ -310,18 +310,9 @@ class Installer(object):
dest = self._dest dest = self._dest
if dest is None: if dest is None:
return [] return []
return self._get_dist_paths(dest) eggs = glob.glob(os.path.join(dest, '*.egg'))
def _get_path_dist_paths(self):
dist_paths = []
for path in self._path:
dist_paths.extend(self._get_dist_paths(path))
return dist_paths
def _get_dist_paths(self, path):
eggs = glob.glob(os.path.join(path, '*.egg'))
dists = [os.path.dirname(dist_info) for dist_info in dists = [os.path.dirname(dist_info) for dist_info in
glob.glob(os.path.join(path, '*', '*.dist-info'))] glob.glob(os.path.join(dest, '*', '*.dist-info'))]
return list(set(eggs + dists)) return list(set(eggs + dists))
@staticmethod @staticmethod
...@@ -438,11 +429,11 @@ class Installer(object): ...@@ -438,11 +429,11 @@ class Installer(object):
str(req)) str(req))
return best_we_have, None return best_we_have, None
def _call_pip_install(self, spec, dest, dist): def _call_pip_wheel(self, spec, dest, dist):
tmp = tempfile.mkdtemp(dir=dest) tmp = tempfile.mkdtemp(dir=dest)
try: try:
paths = call_pip_install(spec, tmp) paths = call_pip_wheel(spec, tmp)
dists = [] dists = []
env = pkg_resources.Environment(paths) env = pkg_resources.Environment(paths)
...@@ -867,7 +858,7 @@ class Installer(object): ...@@ -867,7 +858,7 @@ class Installer(object):
setuptools.command.setopt.edit_config( setuptools.command.setopt.edit_config(
setup_cfg, dict(build_ext=build_ext)) setup_cfg, dict(build_ext=build_ext))
dists = self._call_pip_install(base, self._dest, dist) dists = self._call_pip_wheel(base, self._dest, dist)
return [dist.location for dist in dists] return [dist.location for dist in dists]
finally: finally:
...@@ -1669,13 +1660,13 @@ class IncompatibleConstraintError(zc.buildout.UserError): ...@@ -1669,13 +1660,13 @@ class IncompatibleConstraintError(zc.buildout.UserError):
IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility
def call_pip_install(spec, dest): def call_pip_wheel(spec, dest):
""" """
Call `pip install` from a subprocess to install a Call `pip wheel` from a subprocess to install a
distribution specified by `spec` into `dest`. distribution specified by `spec` into `dest`.
Returns all the paths inside `dest` created by the above. Returns all the paths inside `dest` created by the above.
""" """
args = [sys.executable, '-m', 'pip', 'install', '--no-deps', '-t', dest] args = [sys.executable, '-m', 'pip', 'wheel', '--no-deps', '-w', dest]
level = logger.getEffectiveLevel() level = logger.getEffectiveLevel()
if level >= logging.INFO: if level >= logging.INFO:
args.append('-q') args.append('-q')
...@@ -1690,8 +1681,8 @@ def call_pip_install(spec, dest): ...@@ -1690,8 +1681,8 @@ def call_pip_install(spec, dest):
except ImportError: except ImportError:
HAS_WARNING_OPTION = False HAS_WARNING_OPTION = False
if HAS_WARNING_OPTION: if HAS_WARNING_OPTION:
if not hasattr(call_pip_install, 'displayed'): if not hasattr(call_pip_wheel, 'displayed'):
call_pip_install.displayed = True call_pip_wheel.displayed = True
else: else:
args.append('--no-python-version-warning') args.append('--no-python-version-warning')
...@@ -1716,128 +1707,24 @@ def call_pip_install(spec, dest): ...@@ -1716,128 +1707,24 @@ def call_pip_install(spec, dest):
spec) spec)
sys.exit(1) sys.exit(1)
split_entries = [os.path.splitext(entry) for entry in os.listdir(dest)] entries = os.listdir(dest)
try: try:
distinfo_dir = [ assert len(entries) == 1, "Got multiple entries afer pip wheel"
base + ext for base, ext in split_entries if ext == ".dist-info" wheel = entries[0]
][0] assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
except IndexError: except AssertionError:
logger.error( logger.error(
"No .dist-info directory after successful pip install of %s", "No .whl after successful pip wheel of %s",
spec) spec)
raise raise
return make_egg_after_pip_install(dest, distinfo_dir) return make_egg_after_pip_wheel(dest, wheel)
def make_egg_after_pip_install(dest, distinfo_dir):
"""build properly named egg directory"""
# `pip install` does not build the namespace aware __init__.py files
# but they are needed in egg directories.
# Add them before moving files setup by pip
namespace_packages_file = os.path.join(
dest, distinfo_dir,
'namespace_packages.txt'
)
if os.path.isfile(namespace_packages_file):
with open(namespace_packages_file) as f:
namespace_packages = [
line.strip().replace('.', os.path.sep)
for line in f.readlines()
]
for namespace_package in namespace_packages:
namespace_package_dir = os.path.join(dest, namespace_package)
if os.path.isdir(namespace_package_dir):
init_py_file = os.path.join(
namespace_package_dir, '__init__.py')
with open(init_py_file, 'w') as f:
f.write(
"__import__('pkg_resources')."
"declare_namespace(__name__)"
)
# Remove `bin` directory if needed
# as there is no way to avoid script installation
# when running `pip install`
entry_points_file = os.path.join(dest, distinfo_dir, 'entry_points.txt')
if os.path.isfile(entry_points_file):
with open(entry_points_file) as f:
content = f.read()
if "console_scripts" in content or "gui_scripts" in content:
bin_dir = os.path.join(dest, BIN_SCRIPTS)
if os.path.exists(bin_dir):
shutil.rmtree(bin_dir)
# Make properly named new egg dir
distro = list(pkg_resources.find_distributions(dest))[0]
base = "{}-{}".format(
distro.egg_name(), pkg_resources.get_supported_platform()
)
egg_name = base + '.egg'
new_distinfo_dir = base + '.dist-info'
egg_dir = os.path.join(dest, egg_name)
os.mkdir(egg_dir)
# Move ".dist-info" dir into new egg dir
os.rename(
os.path.join(dest, distinfo_dir),
os.path.join(egg_dir, new_distinfo_dir)
)
top_level_file = os.path.join(egg_dir, new_distinfo_dir, 'top_level.txt')
if os.path.isfile(top_level_file):
with open(top_level_file) as f:
top_levels = filter(
(lambda x: len(x) != 0),
[line.strip() for line in f.readlines()]
)
else:
top_levels = ()
# Move all top_level modules or packages
for top_level in top_levels:
# as package
top_level_dir = os.path.join(dest, top_level)
if os.path.exists(top_level_dir):
shutil.move(top_level_dir, egg_dir)
continue
# as module
top_level_py = top_level_dir + '.py'
if os.path.exists(top_level_py):
shutil.move(top_level_py, egg_dir)
top_level_pyc = top_level_dir + '.pyc'
if os.path.exists(top_level_pyc):
shutil.move(top_level_pyc, egg_dir)
continue
record_file = os.path.join(egg_dir, new_distinfo_dir, 'RECORD')
if os.path.isfile(record_file):
if PY3:
with open(record_file, newline='') as f:
all_files = [row[0] for row in csv.reader(f)]
else:
with open(record_file, 'rb') as f:
all_files = [row[0] for row in csv.reader(f)]
# There might be some c extensions left over
for entry in all_files:
if entry.endswith(('.pyc', '.pyo')):
continue
dest_entry = os.path.join(dest, entry)
# work around pip install -t bug that leaves entries in RECORD
# that starts with '../../'
if not os.path.abspath(dest_entry).startswith(dest):
continue
egg_entry = os.path.join(egg_dir, entry)
if os.path.exists(dest_entry) and not os.path.exists(egg_entry):
egg_entry_dir = os.path.dirname(egg_entry)
if not os.path.exists(egg_entry_dir):
os.makedirs(egg_entry_dir)
os.rename(dest_entry, egg_entry)
return [egg_dir] def make_egg_after_pip_wheel(dest, wheel):
unpack_wheel(os.path.join(dest, wheel), dest)
assert len(os.listdir(dest)) == 2
return glob.glob(os.path.join(dest, '*.egg'))
def unpack_egg(location, dest): def unpack_egg(location, dest):
...@@ -1925,7 +1812,7 @@ def _move_to_eggs_dir_and_compile(dist, dest): ...@@ -1925,7 +1812,7 @@ def _move_to_eggs_dir_and_compile(dist, dest):
unpacker(dist.location, tmp_dest) unpacker(dist.location, tmp_dest)
[tmp_loc] = glob.glob(os.path.join(tmp_dest, '*')) [tmp_loc] = glob.glob(os.path.join(tmp_dest, '*'))
else: else:
[tmp_loc] = call_pip_install(dist.location, tmp_dest) [tmp_loc] = call_pip_wheel(dist.location, tmp_dest)
installed_with_pip = True installed_with_pip = True
# We have installed the dist. Now try to rename/move it. # We have installed the dist. Now try to rename/move it.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment