[wkrd] Use pip install --editable --user

Prior to pip 21.1, pip install --editable --target fails because it results in wrong parameters being passed to setup.py develop by pip. Prior to setuptools 45.2.0, both pip install --editable --target and pip install --editable --prefix fail because the temporary install directory used internally by pip is not added to PYTHONPATH prior to pip calling setup.py develop. In later version setuptools emits a warning instead of an error. Temporarily override PYTHONUSERBASE to point to the target directory, so as to emulate --prefix=<dir> with PYTHONUSERBASE=<dir> and --user. This is needed for Python2 because pip 21.1 and setuptools 45.2.0 are both Python3 only.

[wkrd] Use pip install --editable --user
Prior to pip 21.1, pip install --editable --target fails because it results in wrong parameters being passed to setup.py develop by pip. Prior to setuptools 45.2.0, both pip install --editable --target and pip install --editable --prefix fail because the temporary install directory used internally by pip is not added to PYTHONPATH prior to pip calling setup.py develop. In later version setuptools emits a warning instead of an error. Temporarily override PYTHONUSERBASE to point to the target directory, so as to emulate --prefix=<dir> with PYTHONUSERBASE=<dir> and --user. This is needed for Python2 because pip 21.1 and setuptools 45.2.0 are both Python3 only.
a4303921 · Xavier Thompson · c1b47cc5 · a4303921 · a4303921
Commit a4303921 authored May 29, 2024 by Xavier Thompson
Show whitespace changes
Inline Side-by-side

Showing with 54 additions and 17 deletions

src/zc/buildout/easy_install.py src/zc/buildout/easy_install.py +53 -16

src/zc/buildout/tests/test_all.py src/zc/buildout/tests/test_all.py +1 -1

No files found.
--- a/src/zc/buildout/easy_install.py
+++ b/src/zc/buildout/easy_install.py
@@ -115,6 +115,10 @@ pip_path = setuptools_path = [
  ]
 pip_pythonpath = setuptools_pythonpath = os.pathsep.join(pip_path)
+pip_version = pkg_resources.working_set.find(
+    pkg_resources.Requirement.parse('pip')
+    ).parsed_version
 python_lib = distutils.sysconfig.get_python_lib()
 FILE_SCHEME = re.compile('file://', re.I).match
@@ -1117,7 +1121,7 @@ def _rm(*paths):
 _develop_distutils_scripts = {}
-def _detect_distutils_scripts(directory):
+def _detect_distutils_scripts(egg_name, directory):
    """Record detected distutils scripts from develop eggs
    ``setup.py develop`` doesn't generate metadata on distutils scripts, in
@@ -1125,15 +1129,11 @@ def _detect_distutils_scripts(directory):
    later.
    """
-    dir_contents = os.listdir(directory)
+    if not os.path.isdir(directory):
-    egginfo_filenames = [filename for filename in dir_contents
-                         if filename.endswith('.egg-link')]
-    if not egginfo_filenames:
        return
-    egg_name = egginfo_filenames[0].replace('.egg-link', '')
    marker = 'EASY-INSTALL-DEV-SCRIPT'
    scripts_found = []
-    for filename in dir_contents:
+    for filename in os.listdir(directory):
        if filename.endswith('.exe'):
            continue
        filepath = os.path.join(directory, filename)
@@ -1192,12 +1192,11 @@ def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
        class options: pass
        options._allow_picked_versions = allow_picked_versions()
-        call_pip_editable(directory, tmp3, options, verbosity=verbosity)
+        lib_dir, scripts_dir = call_pip_editable(
+            directory, tmp3, options, verbosity=verbosity)
-        _detect_distutils_scripts(tmp3)
        egg_link, dist_info = [], []
-        for entry in os.listdir(tmp3):
+        for entry in os.listdir(lib_dir):
            if entry.endswith('.egg-link'):
              egg_link.append(entry)
            if entry.endswith('.dist-info') or entry.endswith('.egg-info'):
@@ -1206,11 +1205,14 @@ def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
        assert len(egg_link) + len(dist_info) == 1, str(egg_link + dist_info)
        entry, = (egg_link or dist_info)
-        entry_path = os.path.join(tmp3, entry)
+        entry_path = os.path.join(lib_dir, entry)
        entry_filename, ext = os.path.splitext(entry)
        project_name = entry_filename.split('-', 1)[0]
+        _detect_distutils_scripts(
+            pkg_resources.safe_name(project_name), scripts_dir)
        def move(src, dst):
            _rm(dst)
            os.rename(src, dst)
@@ -1239,7 +1241,7 @@ def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
                    "paths = {p: list(s.submodule_search_locations)"
                    " for p, s in zip(packages, specs)}; "
                    "print(json.dumps(paths))"  % {
-                        'site': tmp3,
+                        'site': lib_dir,
                        'packages': tuple(p for p in top_level if p),
                        }
                    ],
@@ -1941,10 +1943,45 @@ def call_pip_editable(path, dest, options, verbosity=-10):
    """
    Call `pip install --editable` from a subprocess to install a
    the project in `path` into `dest` in editable mode.
+    Returns the actual installation and script directories.
    """
+    if (pip_version >= pkg_resources.parse_version('21.1') and
+        setuptools_version >= pkg_resources.parse_version('45.2.0')):
        call_pip_command(
            ['install', '-t', dest], ['--editable', path],
            options, verbosity)
+        return dest, dest
+    else:
+        # BBB pip < 21.1, setuptools < 45.2.0, Python2
+        # Combining pip --editable <src> --target <dest> fails in pip<21.1.
+        # See https://github.com/pypa/pip/issues/562
+        # See https://github.com/pypa/pip/pull/9636
+        # Also --editable with --target/--prefix fails with setuptools<45.2.0.
+        # See https://github.com/pypa/pip/issues/7627
+        # See https://github.com/pypa/setuptools/pull/1941
+        # Workaround: PYTHONUSERBASE=<dest> pip --editable <src> --user
+        # The install directory will be an OS-dependent subpath of <dest>.
+        if os.name == 'posix':
+            python = 'python%d.%d' % sys.version_info[0:2]
+            lib_dir = os.path.join(dest, 'lib', python, 'site-packages')
+            script_dir = os.path.join(dest, 'bin')
+        else:
+            lib_dir = os.join(dest, 'Lib', 'site-packages')
+            script_dir = os.path.join(dest, 'Scripts')
+        for install_path in (lib_dir, script_dir):
+            os.makedirs(install_path)
+        old = {k: os.environ.get(k)
+            for k in ('PYTHONUSERBASE', 'PYTHONNOUSERSITE')}
+        try:
+            os.environ['PYTHONUSERBASE'] = dest
+            os.environ.pop('PYTHONNOUSERSITE', None)
+            call_pip_command(
+                ['install', '--user'], ['--editable', path],
+                options, verbosity)
+        finally:
+            for k, v in old.items():
+                (os.environ.pop if v is None else os.environ.__setitem__)(k, v)
+        return lib_dir, script_dir
 def call_pip_wheel(spec, dest, options):

--- a/src/zc/buildout/tests/test_all.py
+++ b/src/zc/buildout/tests/test_all.py
@@ -216,7 +216,7 @@ We should be able to deal with setup scripts that aren't setuptools based.
    Installing...
    Develop: '/sample-buildout/foo'
    Running pip install --editable /sample-buildout/foo
-    ... -m pip install -t ... --editable /sample-buildout/foo
+    ... -m pip install ... --editable /sample-buildout/foo
    ...
    """