[wkrd] Use pip install --editable --user

Prior to pip 21.1, pip install --editable --target fails because it
results in wrong parameters being passed to setup.py develop by pip.

Prior to setuptools 45.2.0, both pip install --editable --target and
pip install --editable --prefix fail because the temporary install
directory used internally by pip is not added to PYTHONPATH prior
to pip calling setup.py develop. In later version setuptools emits a
warning instead of an error.

Temporarily override PYTHONUSERBASE to point to the target directory,
so as to emulate --prefix=<dir> with PYTHONUSERBASE=<dir> and --user.

This is needed for Python2 because pip 21.1 and setuptools 45.2.0 are
both Python3 only.

src/zc/buildout/easy_install.py

@@ -115,6 +115,10 @@ pip_path = setuptools_path = [
   ]
 pip_pythonpath = setuptools_pythonpath = os.pathsep.join(pip_path)
 
+pip_version = pkg_resources.working_set.find(
+    pkg_resources.Requirement.parse('pip')
+    ).parsed_version
+
 python_lib = distutils.sysconfig.get_python_lib()
 
 FILE_SCHEME = re.compile('file://', re.I).match
@@ -1117,7 +1121,7 @@ def _rm(*paths):
 _develop_distutils_scripts = {}
 
 
-def _detect_distutils_scripts(directory):
+def _detect_distutils_scripts(egg_name, directory):
     """Record detected distutils scripts from develop eggs
 
     ``setup.py develop`` doesn't generate metadata on distutils scripts, in
@@ -1125,15 +1129,11 @@ def _detect_distutils_scripts(directory):
     later.
 
     """
-    dir_contents = os.listdir(directory)
-    egginfo_filenames = [filename for filename in dir_contents
-                         if filename.endswith('.egg-link')]
-    if not egginfo_filenames:
+    if not os.path.isdir(directory):
         return
-    egg_name = egginfo_filenames[0].replace('.egg-link', '')
     marker = 'EASY-INSTALL-DEV-SCRIPT'
     scripts_found = []
-    for filename in dir_contents:
+    for filename in os.listdir(directory):
         if filename.endswith('.exe'):
             continue
         filepath = os.path.join(directory, filename)
@@ -1192,12 +1192,11 @@ def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
         class options: pass
         options._allow_picked_versions = allow_picked_versions()
 
-        call_pip_editable(directory, tmp3, options, verbosity=verbosity)
-
-        _detect_distutils_scripts(tmp3)
+        lib_dir, scripts_dir = call_pip_editable(
+            directory, tmp3, options, verbosity=verbosity)
 
         egg_link, dist_info = [], []
-        for entry in os.listdir(tmp3):
+        for entry in os.listdir(lib_dir):
             if entry.endswith('.egg-link'):
               egg_link.append(entry)
             if entry.endswith('.dist-info') or entry.endswith('.egg-info'):
@@ -1206,11 +1205,14 @@ def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
         assert len(egg_link) + len(dist_info) == 1, str(egg_link + dist_info)
         entry, = (egg_link or dist_info)
 
-        entry_path = os.path.join(tmp3, entry)
+        entry_path = os.path.join(lib_dir, entry)
 
         entry_filename, ext = os.path.splitext(entry)
         project_name = entry_filename.split('-', 1)[0]
 
+        _detect_distutils_scripts(
+            pkg_resources.safe_name(project_name), scripts_dir)
+
         def move(src, dst):
             _rm(dst)
             os.rename(src, dst)
@@ -1246,7 +1248,7 @@ def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
                     " for p, s in zip(packages, specs) "
                     " if s.submodule_search_locations is not None}; "
                     "print(json.dumps(paths))"  % {
-                        'site': tmp3,
+                        'site': lib_dir,
                         'packages': packages,
                         }
                     ],
@@ -1948,10 +1950,50 @@ def call_pip_editable(path, dest, options, verbosity=-10):
     """
     Call `pip install --editable` from a subprocess to install a
     the project in `path` into `dest` in editable mode.
+    Returns the actual installation and script directories.
     """
+    if (pip_version >= pkg_resources.parse_version('21.1') and
+        setuptools_version >= pkg_resources.parse_version('45.2.0')):
         call_pip_command(
             ['install', '-t', dest], ['--editable', path],
             options, verbosity)
+        return dest, dest
+    else:
+        # BBB pip < 21.1, setuptools < 45.2.0, Python2
+        # Combining pip --editable <src> --target <dest> fails in pip<21.1.
+        # See https://github.com/pypa/pip/issues/562
+        # See https://github.com/pypa/pip/pull/9636
+        # Also --editable with --target/--prefix fails with setuptools<45.2.0.
+        # See https://github.com/pypa/pip/issues/7627
+        # See https://github.com/pypa/setuptools/pull/1941
+        # Workaround: PYTHONUSERBASE=<dest> pip --editable <src> --user
+        # The install directory will be an OS-dependent subpath of <dest>.
+        if os.name == 'posix':
+            python = 'python%d.%d' % sys.version_info[0:2]
+            lib_dir = os.path.join(dest, 'lib', python, 'site-packages')
+            script_dir = os.path.join(dest, 'bin')
+        else:
+            lib_dir = os.join(dest, 'Lib', 'site-packages')
+            script_dir = os.path.join(dest, 'Scripts')
+        for install_path in (lib_dir, script_dir):
+            os.makedirs(install_path)
+        old_env = {k: os.environ.get(k)
+            for k in ('PYTHONUSERBASE', 'PYTHONNOUSERSITE')}
+        old_allow_picked_versions = options._allow_picked_versions
+        try:
+            os.environ['PYTHONUSERBASE'] = dest
+            os.environ.pop('PYTHONNOUSERSITE', None)
+            # Prevent pip from using build isolation because it breaks --user.
+            # This is acceptable for pip < 21.1, setuptools < 45.2.0, Python2.
+            options._allow_picked_versions = False
+            call_pip_command(
+                ['install', '--user'], ['--editable', path],
+                options, verbosity)
+        finally:
+            options._allow_picked_versions = old_allow_picked_versions
+            for k, v in old_env.items():
+                (os.environ.pop if v is None else os.environ.__setitem__)(k, v)
+        return lib_dir, script_dir
 
 
 def call_pip_wheel(spec, dest, options):

src/zc/buildout/tests/test_all.py

@@ -216,7 +216,7 @@ We should be able to deal with setup scripts that aren't setuptools based.
     Installing...
     Develop: '/sample-buildout/foo'
     Running pip install --editable /sample-buildout/foo
-    ... -m pip install -t ... --editable /sample-buildout/foo
+    ... -m pip install ... --editable /sample-buildout/foo
     ...
     """