[feat] Add buildout:extra-paths option

This option determines what paths zc.buildout will scan for already
installed distributions. This defaults to sys.path and can be set to
an empty value to enable isolation. The special value 'legacy' yields
the previous behavior of scanning specifically the paths of the current
zc.buildout distribution and its dependencies.

doc/reference.rst

@@ -358,6 +358,16 @@ extends-cache
   substitutions, and the result is a relative path, then it will be
   interpreted relative to the buildout directory.)
 
+.. _extra-paths-buildout-option
+
+extra-paths, default: sys.path
+  Extra paths to scan for already installed distributions.
+
+  Setting this to an empty value enables isolation of buildout.
+  Setting this to 'legacy' enables the legacy behavior of
+  scanning the paths of the distributions of zc.buildout itself
+  and its dependencies, which may contain sites-packages or not.
+
 .. _find-links-option:
 
 find-links, default: ''

src/zc/buildout/buildout.py

@@ -292,6 +292,7 @@ _buildout_default_options = _annotate_section({
     'develop-eggs-directory': 'develop-eggs',
     'eggs-directory': 'eggs',
     'executable': sys.executable,
+    'extra-paths': 'sys.path',
     'find-links': '',
     'install-from-cache': 'false',
     'installed': '.installed.cfg',
@@ -533,6 +534,35 @@ class Buildout(DictMixin):
             options['installed'] = os.path.join(options['directory'],
                                                 options['installed'])
 
+        # Extra paths to scan for already installed distributions.
+        extra_paths = options['extra-paths']
+        if extra_paths == 'legacy':
+            # special case: legacy behavior
+            # this case is why this is done before setting easy_install
+            # versions and other options, to get the legacy behavior.
+            # XXX: These 'sorted' calls correspond to the original behavior,
+            # but they are quite problematic, as ther distributions for
+            # zc.buildout, pip, wheel and setuptools may take precedence
+            # over the ones currently running.
+            old_extra_paths = zc.buildout.easy_install.extra_paths(
+              sorted({d.location for d in pkg_resources.working_set}))
+            try:
+                buildout_and_setuptools_dists = list(
+                    zc.buildout.easy_install.install(['zc.buildout'], None,
+                        check_picked=False))
+            finally:
+                zc.buildout.easy_install.extra_paths(old_extra_paths)
+            extra_paths = sorted(
+                {d.location for d in buildout_and_setuptools_dists})
+            options['extra-paths'] = ' '.join(extra_paths)
+        elif extra_paths == 'sys.path':
+            # special case: sys.path
+            extra_paths = sys.path
+            options['extra-paths'] = ' '.join(extra_paths)
+        else:
+            extra_paths = extra_paths.split()
+        zc.buildout.easy_install.extra_paths(extra_paths)
+
         self._setup_logging()
         self._setup_socket_timeout()
 

src/zc/buildout/easy_install.py

@@ -91,13 +91,15 @@ if has_distribute and not has_setuptools:
     sys.exit("zc.buildout 3 needs setuptools, not distribute."
              "Did you properly install with pip in a virtualenv ?")
 
-# Include buildout and setuptools eggs in paths.  We get this
-# initially from the entire working set.  Later, we'll use the install
-# function to narrow to just the buildout and setuptools paths.
-buildout_and_setuptools_path = sorted({d.location for d in pkg_resources.working_set})
-setuptools_path = buildout_and_setuptools_path
-pip_path = buildout_and_setuptools_path
-logger.debug('before restricting versions: pip_path %r', pip_path)
+# XXX Take care to respect the sys.path order, as otherwise other
+# distributions for pip, wheel and setuptools may take precedence
+# over the ones currently running.
+pip_path = setuptools_path = [
+  dist.location
+  for dist in pkg_resources.working_set
+  if dist.project_name in ('pip', 'wheel', 'setuptools')
+  ]
+pip_pythonpath = setuptools_pythonpath = os.pathsep.join(pip_path)
 
 FILE_SCHEME = re.compile('file://', re.I).match
 DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$")
@@ -241,6 +243,7 @@ class Installer(object):
     _allow_picked_versions = True
     _store_required_by = False
     _allow_unknown_extras = False
+    _extra_paths = []
 
     def __init__(self,
                  dest=None,
@@ -275,7 +278,7 @@ class Installer(object):
             links.insert(0, self._download_cache)
 
         self._index_url = index
-        path = (path and path[:] or []) + buildout_and_setuptools_path
+        path = (path and path[:] or []) + self._extra_paths
         self._path = path
         if self._dest is None:
             newest = False
@@ -950,6 +953,12 @@ def get_picked_versions():
     required_by = Installer._required_by
     return (picked_versions, required_by)
 
+def extra_paths(setting=None):
+    old = Installer._extra_paths
+    if setting is not None:
+        Installer._extra_paths = setting
+    return old
+
 
 def install(specs, dest,
             links=(), index=None,
@@ -974,19 +983,6 @@ def install(specs, dest,
                           allow_unknown_extras=allow_unknown_extras)
     return installer.install(specs, working_set)
 
-buildout_and_setuptools_dists = list(install(['zc.buildout'], None,
-                                             check_picked=False))
-buildout_and_setuptools_path = sorted({d.location
-                                for d in buildout_and_setuptools_dists})
-
-pip_dists = [d for d in buildout_and_setuptools_dists if d.project_name != 'zc.buildout']
-pip_path = sorted({d.location for d in pip_dists})
-logger.debug('after restricting versions: pip_path %r', pip_path)
-pip_pythonpath = os.pathsep.join(pip_path)
-
-setuptools_path = pip_path
-setuptools_pythonpath = pip_pythonpath
-
 
 def build(spec, dest, build_ext,
           links=(), index=None,

src/zc/buildout/tests/buildout.txt

@@ -2917,6 +2917,7 @@ database is shown::
     directory = /sample-buildout
     eggs-directory = /sample-buildout/eggs
     executable = python
+    extra-paths = ...
     find-links =
     install-from-cache = false
     installed = /sample-buildout/.installed.cfg

src/zc/buildout/tests/test_all.py

@@ -2677,7 +2677,8 @@ honoring our version specification.
     ... eggs = foo
     ... ''' % ('\n'.join(
     ...     '%s = %s' % (d.key, d.version)
-    ...     for d in zc.buildout.easy_install.buildout_and_setuptools_dists)))
+    ...     for d in pkg_resources.working_set.resolve(
+    ...         pkg_resources.parse_requirements('zc.buildout')))))
 
     >>> print_(system(buildout), end='')
     Installing foo.