support limiting packages from site-packages

.bzrignore

+.installed.cfg
+bin
+build
+develop-eggs
+eggs
+parts
+src/zc.buildout.egg-info
+z3c.recipe.scripts_/src/z3c.recipe.scripts.egg-info
+zc.recipe.egg_/src/zc.recipe.egg.egg-info

src/zc/buildout/easy_install.py

@@ -19,6 +19,7 @@ installed.
 """
 
 import distutils.errors
+import fnmatch
 import glob
 import logging
 import os
@@ -67,6 +68,64 @@ buildout_and_setuptools_path = [
         pkg_resources.Requirement.parse('zc.buildout')).location,
     ]
 
+def _get_system_paths(executable):
+    """return lists of standard lib and site paths for executable.
+    """
+    # We want to get a list of the site packages, which is not easy.
+    # The canonical way to do this is to use
+    # distutils.sysconfig.get_python_lib(), but that only returns a
+    # single path, which does not reflect reality for many system
+    # Pythons, which have multiple additions.  Instead, we start Python
+    # with -S, which does not import site.py and set up the extra paths
+    # like site-packages or (Ubuntu/Debian) dist-packages and
+    # python-support. We then compare that sys.path with the normal one
+    # (minus user packages if this is Python 2.6, because we don't
+    # support those (yet?).  The set of the normal one minus the set of
+    # the ones in ``python -S`` is the set of packages that are
+    # effectively site-packages.
+    #
+    # The given executable might not be the current executable, so it is
+    # appropriate to do another subprocess to figure out what the
+    # additional site-package paths are. Moreover, even if this
+    # executable *is* the current executable, this code might be run in
+    # the context of code that has manipulated the sys.path--for
+    # instance, to add local zc.buildout or setuptools eggs.
+    def get_sys_path(*args, **kwargs):
+        cmd = [executable]
+        cmd.extend(args)
+        cmd.extend([
+            "-c", "import sys, os;"
+            "print repr([os.path.normpath(p) for p in sys.path if p])"])
+        # Windows needs some (as yet to be determined) part of the real env.
+        env = os.environ.copy()
+        env.update(kwargs)
+        _proc = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+        stdout, stderr = _proc.communicate();
+        if _proc.returncode:
+            raise RuntimeError(
+                'error trying to get system packages:\n%s' % (stderr,))
+        res = eval(stdout.strip())
+        try:
+            res.remove('.')
+        except ValueError:
+            pass
+        return res
+    stdlib = get_sys_path('-S') # stdlib only
+    no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')
+    site_paths = [p for p in no_user_paths if p not in stdlib]
+    return (stdlib, site_paths)
+
+def _get_version_info(executable):
+    cmd = [executable, '-Sc', 'import sys; print repr(sys.version_info)']
+    _proc = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = _proc.communicate();
+    if _proc.returncode:
+        raise RuntimeError(
+            'error trying to get system packages:\n%s' % (stderr,))
+    return eval(stdout.strip())
+
 
 class IncompatibleVersionError(zc.buildout.UserError):
     """A specified version is incompatible with a given requirement.
@@ -109,7 +168,12 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
 
 
 _indexes = {}
-def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
+def _get_index(executable, index_url, find_links, allow_hosts=('*',),
+               path=None):
+    # If path is None, the index will use sys.path.  If you provide an empty
+    # path ([]), it will complain uselessly about missing index pages for
+    # packages found in the paths that you expect to use.  Therefore, this path
+    # is always the same as the _env path in the Installer.
     key = executable, index_url, tuple(find_links)
     index = _indexes.get(key)
     if index is not None:
@@ -118,7 +182,8 @@ def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
     if index_url is None:
         index_url = default_index_url
     index = AllowHostsPackageIndex(
-        index_url, hosts=allow_hosts, python=_get_version(executable)
+        index_url, hosts=allow_hosts, search_path=path,
+        python=_get_version(executable)
         )
 
     if find_links:
@@ -192,6 +257,8 @@ class Installer:
     _use_dependency_links = True
     _allow_picked_versions = True
     _always_unzip = False
+    _include_site_packages = True
+    _allowed_eggs_from_site_packages = ('*',)
 
     def __init__(self,
                  dest=None,
@@ -203,6 +270,8 @@ class Installer:
                  newest=True,
                  versions=None,
                  use_dependency_links=None,
+                 include_site_packages=None,
+                 allowed_eggs_from_site_packages=None,
                  allow_hosts=('*',)
                  ):
         self._dest = dest
@@ -225,7 +294,28 @@ class Installer:
         self._executable = executable
         if always_unzip is not None:
             self._always_unzip = always_unzip
-        path = (path and path[:] or []) + buildout_and_setuptools_path
+        path = (path and path[:] or [])
+        if include_site_packages is not None:
+            self._include_site_packages = include_site_packages
+        if allowed_eggs_from_site_packages is not None:
+            self._allowed_eggs_from_site_packages = tuple(
+                allowed_eggs_from_site_packages)
+        stdlib, self._site_packages = _get_system_paths(executable)
+        version_info = _get_version_info(executable)
+        if version_info == sys.version_info:
+            # Maybe we can add the buildout and setuptools path.  If we
+            # are including site_packages, we only have to include the extra
+            # bits here, so we don't duplicate.  On the other hand, if we
+            # are not including site_packages, we only want to include the
+            # parts that are not in site_packages, so the code is the same.
+            path.extend(
+                set(buildout_and_setuptools_path).difference(
+                    self._site_packages))
+        if self._include_site_packages:
+            path.extend(self._site_packages)
+        # else we could try to still include the buildout_and_setuptools_path
+        # if the elements are not in site_packages, but we're not bothering
+        # with this optimization for now, in the name of code simplicity.
         if dest is not None and dest not in path:
             path.insert(0, dest)
         self._path = path
@@ -234,13 +324,42 @@ class Installer:
         self._newest = newest
         self._env = pkg_resources.Environment(path,
                                               python=_get_version(executable))
-        self._index = _get_index(executable, index, links, self._allow_hosts)
+        self._index = _get_index(executable, index, links, self._allow_hosts,
+                                 self._path)
 
         if versions is not None:
             self._versions = versions
 
+    _allowed_eggs_from_site_packages_regex = None
+    def allow_site_package_egg(self, name):
+        if (not self._include_site_packages or
+            not self._allowed_eggs_from_site_packages):
+            # If the answer is a blanket "no," perform a shortcut.
+            return False
+        if self._allowed_eggs_from_site_packages_regex is None:
+            pattern = '(%s)' % (
+                '|'.join(
+                    fnmatch.translate(name)
+                    for name in self._allowed_eggs_from_site_packages),
+                )
+            self._allowed_eggs_from_site_packages_regex = re.compile(pattern)
+        return bool(self._allowed_eggs_from_site_packages_regex.match(name))
+
     def _satisfied(self, req, source=None):
-        dists = [dist for dist in self._env[req.project_name] if dist in req]
+        # We get all distributions that match the given requirement.  If we are
+        # not supposed to include site-packages for the given egg, we also
+        # filter those out. Even if include_site_packages is False and so we
+        # have excluded site packages from the _env's paths (see
+        # Installer.__init__), we need to do the filtering here because an
+        # .egg-link, such as one for setuptools or zc.buildout installed by
+        # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
+        # path in our _site_packages.
+        dists = [dist for dist in self._env[req.project_name] if (
+                    dist in req and (
+                        dist.location not in self._site_packages or
+                        self.allow_site_package_egg(dist.project_name))
+                    )
+                ]
         if not dists:
             logger.debug('We have no distributions for %s that satisfies %r.',
                          req.project_name, str(req))
@@ -441,13 +560,21 @@ class Installer:
             # Nothing is available.
             return None
 
-        # Filter the available dists for the requirement and source flag
-        dists = [dist for dist in index[requirement.project_name]
-                 if ((dist in requirement)
-                     and
-                     ((not source) or
-                      (dist.precedence == pkg_resources.SOURCE_DIST)
-                      )
+        # Filter the available dists for the requirement and source flag.  If
+        # we are not supposed to include site-packages for the given egg, we
+        # also filter those out. Even if include_site_packages is False and so
+        # we have excluded site packages from the _env's paths (see
+        # Installer.__init__), we need to do the filtering here because an
+        # .egg-link, such as one for setuptools or zc.buildout installed by
+        # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
+        # path in our _site_packages.
+        dists = [dist for dist in index[requirement.project_name] if (
+                    dist in requirement and (
+                        dist.location not in self._site_packages or
+                        self.allow_site_package_egg(dist.project_name))
+                    and (
+                        (not source) or
+                        (dist.precedence == pkg_resources.SOURCE_DIST))
                     )
                  ]
 
@@ -608,7 +735,7 @@ class Installer:
                         self._links.append(link)
                         self._index = _get_index(self._executable,
                                                  self._index_url, self._links,
-                                                 self._allow_hosts)
+                                                 self._allow_hosts, self._path)
 
         for dist in dists:
             # Check whether we picked a version and, if we did, report it:
@@ -689,35 +816,52 @@ class Installer:
                 self._maybe_add_setuptools(ws, dist)
 
         # OK, we have the requested distributions and they're in the working
-        # set, but they may have unmet requirements.  We'll simply keep
-        # trying to resolve requirements, adding missing requirements as they
-        # are reported.
-        #
-        # Note that we don't pass in the environment, because we want
+        # set, but they may have unmet requirements.  We'll resolve these
+        # requirements. This is code modified from
+        # pkg_resources.WorkingSet.resolve.  We can't reuse that code directly
+        # because we have to constrain our requirements (see
+        # versions_section_ignored_for_dependency_in_favor_of_site_packages in
+        # zc.buildout.tests).
+        requirements.reverse() # Set up the stack.
+        processed = {}  # This is a set of processed requirements.
+        best = {}  # This is a mapping of key -> dist.
+        # Note that we don't use the existing environment, because we want
         # to look for new eggs unless what we have is the best that
         # matches the requirement.
-        while 1:
+        env = pkg_resources.Environment(ws.entries)
+        while requirements:
+            # Process dependencies breadth-first.
+            req = self._constrain(requirements.pop(0))
+            if req in processed:
+                # Ignore cyclic or redundant dependencies.
+                continue
+            dist = best.get(req.key)
+            if dist is None:
+                # Find the best distribution and add it to the map.
+                dist = ws.by_key.get(req.key)
+                if dist is None:
                     try:
-                ws.resolve(requirements)
-            except pkg_resources.DistributionNotFound, err:
-                [requirement] = err
-                requirement = self._constrain(requirement)
+                        dist = best[req.key] = env.best_match(req, ws)
+                    except pkg_resources.VersionConflict, err:
+                        raise VersionConflict(err, ws)
+                    if dist is None:
                         if destination:
-                    logger.debug('Getting required %r', str(requirement))
+                            logger.debug('Getting required %r', str(req))
                         else:
-                    logger.debug('Adding required %r', str(requirement))
-                _log_requirement(ws, requirement)
-
-                for dist in self._get_dist(requirement, ws, self._always_unzip
-                                           ):
-
+                            logger.debug('Adding required %r', str(req))
+                        _log_requirement(ws, req)
+                        for dist in self._get_dist(req,
+                                                   ws, self._always_unzip):
                             ws.add(dist)
                             self._maybe_add_setuptools(ws, dist)
-            except pkg_resources.VersionConflict, err:
-                raise VersionConflict(err, ws)
-            else:
-                break
-
+            if dist not in req:
+                # Oops, the "best" so far conflicts with a dependency.
+                raise VersionConflict(
+                    pkg_resources.VersionConflict(dist, req), ws)
+            requirements.extend(dist.requires(req.extras)[::-1])
+            processed[req] = True
+            if dist.location in self._site_packages:
+                logger.debug('Egg from site-packages: %s', dist)
         return ws
 
     def build(self, spec, build_ext):
@@ -812,6 +956,18 @@ def prefer_final(setting=None):
         Installer._prefer_final = bool(setting)
     return old
 
+def include_site_packages(setting=None):
+    old = Installer._include_site_packages
+    if setting is not None:
+        Installer._include_site_packages = bool(setting)
+    return old
+
+def allowed_eggs_from_site_packages(setting=None):
+    old = Installer._allowed_eggs_from_site_packages
+    if setting is not None:
+        Installer._allowed_eggs_from_site_packages = tuple(setting)
+    return old
+
 def use_dependency_links(setting=None):
     old = Installer._use_dependency_links
     if setting is not None:
@@ -834,9 +990,13 @@ def install(specs, dest,
             links=(), index=None,
             executable=sys.executable, always_unzip=None,
             path=None, working_set=None, newest=True, versions=None,
-            use_dependency_links=None, allow_hosts=('*',)):
+            use_dependency_links=None, include_site_packages=None,
+            allowed_eggs_from_site_packages=None, allow_hosts=('*',)):
     installer = Installer(dest, links, index, executable, always_unzip, path,
                           newest, versions, use_dependency_links,
+                          include_site_packages=include_site_packages,
+                          allowed_eggs_from_site_packages=
+                            allowed_eggs_from_site_packages,
                           allow_hosts=allow_hosts)
     return installer.install(specs, working_set)
 
@@ -844,9 +1004,14 @@ def install(specs, dest,
 def build(spec, dest, build_ext,
           links=(), index=None,
           executable=sys.executable,
-          path=None, newest=True, versions=None, allow_hosts=('*',)):
+          path=None, newest=True, versions=None, include_site_packages=None,
+          allowed_eggs_from_site_packages=None, allow_hosts=('*',)):
     installer = Installer(dest, links, index, executable, True, path, newest,
-                          versions, allow_hosts=allow_hosts)
+                          versions,
+                          include_site_packages=include_site_packages,
+                          allowed_eggs_from_site_packages=
+                            allowed_eggs_from_site_packages,
+                          allow_hosts=allow_hosts)
     return installer.build(spec, build_ext)
 
 
@@ -941,9 +1106,12 @@ def develop(setup, dest,
         undo.reverse()
         [f() for f in undo]
 
-
-def working_set(specs, executable, path):
-    return install(specs, None, executable=executable, path=path)
+def working_set(specs, executable, path, include_site_packages=None,
+                allowed_eggs_from_site_packages=None):
+    return install(
+        specs, None, executable=executable, path=path,
+        include_site_packages=include_site_packages,
+        allowed_eggs_from_site_packages=allowed_eggs_from_site_packages)
 
 ############################################################################
 # Script generation functions
@@ -1276,54 +1444,6 @@ if _interactive:
 
 # These are used only by the newer ``generate_scripts`` function.
 
-def _get_system_paths(executable):
-    """return lists of standard lib and site paths for executable.
-    """
-    # We want to get a list of the site packages, which is not easy.
-    # The canonical way to do this is to use
-    # distutils.sysconfig.get_python_lib(), but that only returns a
-    # single path, which does not reflect reality for many system
-    # Pythons, which have multiple additions.  Instead, we start Python
-    # with -S, which does not import site.py and set up the extra paths
-    # like site-packages or (Ubuntu/Debian) dist-packages and
-    # python-support. We then compare that sys.path with the normal one
-    # (minus user packages if this is Python 2.6, because we don't
-    # support those (yet?).  The set of the normal one minus the set of
-    # the ones in ``python -S`` is the set of packages that are
-    # effectively site-packages.
-    #
-    # The given executable might not be the current executable, so it is
-    # appropriate to do another subprocess to figure out what the
-    # additional site-package paths are. Moreover, even if this
-    # executable *is* the current executable, this code might be run in
-    # the context of code that has manipulated the sys.path--for
-    # instance, to add local zc.buildout or setuptools eggs.
-    def get_sys_path(*args, **kwargs):
-        cmd = [executable]
-        cmd.extend(args)
-        cmd.extend([
-            "-c", "import sys, os;"
-            "print repr([os.path.normpath(p) for p in sys.path if p])"])
-        # Windows needs some (as yet to be determined) part of the real env.
-        env = os.environ.copy()
-        env.update(kwargs)
-        _proc = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
-        stdout, stderr = _proc.communicate();
-        if _proc.returncode:
-            raise RuntimeError(
-                'error trying to get system packages:\n%s' % (stderr,))
-        res = eval(stdout.strip())
-        try:
-            res.remove('.')
-        except ValueError:
-            pass
-        return res
-    stdlib = get_sys_path('-S') # stdlib only
-    no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')
-    site_paths = [p for p in no_user_paths if p not in stdlib]
-    return (stdlib, site_paths)
-
 def _get_module_file(executable, name):
     """Return a module's file path.
 

src/zc/buildout/easy_install.txt

@@ -89,6 +89,14 @@ use_dependency_links
    for using dependency_links in preference to other
    locations. Defaults to true.
 
+include_site_packages
+    A flag indicating whether Python's non-standard-library packages should
+    be available for finding dependencies.  Defaults to true.
+
+    Paths outside of Python's standard library--or more precisely, those that
+    are not included when Python is started with the -S argument--are loosely
+    referred to as "site-packages" here.
+
 relative_paths
    Adjust egg paths so they are relative to the script path.  This
    allows scripts to work when scripts and eggs are moved, as long as
@@ -399,6 +407,68 @@ dictionary:
     >>> [d.version for d in ws]
     ['0.3', '1.1']
 
+Dependencies in Site Packages
+-----------------------------
+
+Paths outside of Python's standard library--or more precisely, those that are
+not included when Python is started with the -S argument--are loosely referred
+to as "site-packages" here.  These site-packages are searched by default for
+distributions.  This can be disabled, so that, for instance, a system Python
+can be used with buildout, cleaned of any packages installed by a user or
+system package manager.
+
+The default behavior can be controlled and introspected using
+zc.buildout.easy_install.include_site_packages.
+
+    >>> zc.buildout.easy_install.include_site_packages()
+    True
+
+Here's an example of using a Python executable that includes our dependencies.
+
+Our "py_path" will have the "demoneeded," and "demo" packages available.
+ We'll simply be asking for "demoneeded" here, but without any external
+ index or links.
+
+    >>> from zc.buildout.tests import create_sample_sys_install
+    >>> py_path, site_packages_path = make_py()
+    >>> create_sample_sys_install(site_packages_path)
+
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None)
+    >>> [dist.project_name for dist in workingset]
+    ['demoneeded']
+
+That worked fine.  Let's try again with site packages not allowed.  We'll
+change the policy by changing the default.  Notice that the function for
+changing the default value returns the previous value.
+
+    >>> zc.buildout.easy_install.include_site_packages(False)
+    True
+
+    >>> zc.buildout.easy_install.include_site_packages()
+    False
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+    >>> rmdir(example_dest)
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None)
+    Traceback (most recent call last):
+        ...
+    MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+    >>> zc.buildout.easy_install.clear_index_cache()
+
+Now we'll reset the default.
+
+    >>> zc.buildout.easy_install.include_site_packages(True)
+    False
+
+    >>> zc.buildout.easy_install.include_site_packages()
+    True
+
 Dependency links
 ----------------
 
@@ -1259,6 +1329,7 @@ call to another text fixture to create.
     >>> namespace_eggs = tmpdir('namespace_eggs')
     >>> create_sample_namespace_eggs(namespace_eggs)
 
+    >>> reset_interpreter()
     >>> ws = zc.buildout.easy_install.install(
     ...     ['demo', 'tellmy.fortune'], join(interpreter_dir, 'eggs'),
     ...     links=[link_server, namespace_eggs], index=link_server+'index/')
@@ -1319,6 +1390,7 @@ The most complex that this function gets is if you use namespace packages,
 include site-packages, and use relative paths.  For completeness, we'll look
 at that result.
 
+    >>> reset_interpreter()
     >>> generated = zc.buildout.easy_install.generate_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py', add_site_packages=True,

src/zc/buildout/testing.py

@@ -222,11 +222,37 @@ def wait_until(label, func, *args, **kw):
         time.sleep(0.01)
     raise ValueError('Timed out waiting for: '+label)
 
+def get_installer_values():
+    """Get the current values for the easy_install module.
+
+    This is necessary because instantiating a Buildout will force the
+    Buildout's values on the installer.
+
+    Returns a dict of names-values suitable for set_installer_values."""
+    names = ('default_versions', 'download_cache', 'install_from_cache',
+             'prefer_final', 'include_site_packages',
+             'allowed_eggs_from_site_packages', 'use_dependency_links',
+             'allow_picked_versions', 'always_unzip'
+            )
+    values = {}
+    for name in names:
+        values[name] = getattr(zc.buildout.easy_install, name)()
+    return values
+
+def set_installer_values(values):
+    """Set the given values on the installer."""
+    for name, value in values.items():
+        getattr(zc.buildout.easy_install, name)(value)
+
 def make_buildout():
-    # Create a basic buildout.cfg to avoid a warning from buildout:
+    """Make a buildout that uses this version of zc.buildout."""
+    # Create a basic buildout.cfg to avoid a warning from buildout.
     open('buildout.cfg', 'w').write(
         "[buildout]\nparts =\n"
         )
+    # Get state of installer defaults so we can reinstate them (instantiating
+    # a Buildout will force the Buildout's defaults on the installer).
+    installer_values = get_installer_values()
     # Use the buildout bootstrap command to create a buildout
     zc.buildout.buildout.Buildout(
         'buildout.cfg',
@@ -234,20 +260,23 @@ def make_buildout():
          # trick bootstrap into putting the buildout develop egg
          # in the eggs dir.
          ('buildout', 'develop-eggs-directory', 'eggs'),
-         ]
+         ],
+        user_defaults=False,
         ).bootstrap([])
     # Create the develop-eggs dir, which didn't get created the usual
     # way due to the trick above:
     os.mkdir('develop-eggs')
+    # Reinstate the default values of the installer.
+    set_installer_values(installer_values)
 
 def buildoutSetUp(test):
 
     test.globs['__tear_downs'] = __tear_downs = []
     test.globs['register_teardown'] = register_teardown = __tear_downs.append
 
-    prefer_final = zc.buildout.easy_install.prefer_final()
+    installer_values = get_installer_values()
     register_teardown(
-        lambda: zc.buildout.easy_install.prefer_final(prefer_final)
+        lambda: set_installer_values(installer_values)
         )
 
     here = os.getcwd()
@@ -367,8 +396,6 @@ def buildoutSetUp(test):
         make_py = make_py
         ))
 
-    zc.buildout.easy_install.prefer_final(prefer_final)
-
 def buildoutTearDown(test):
     for f in test.globs['__tear_downs']:
         f()

src/zc/buildout/tests.py

@@ -385,6 +385,64 @@ buildout will tell us who's asking for something that we can't find.
     Error: Couldn't find a distribution for 'demoneeded'.
     """
 
+def show_eggs_from_site_packages():
+    """
+Sometimes you want to know what eggs are coming from site-packages.  This
+might be for a diagnostic, or so that you can get a starting value for the
+allowed-eggs-from-site-packages option.  The -v flag will also include this
+information.
+
+Our "py_path" has the "demoneeded," "demo"
+packages available.  We'll ask for "bigdemo," which will get both of them.
+
+Here's our set up.
+
+    >>> py_path, site_packages_path = make_py()
+    >>> create_sample_sys_install(site_packages_path)
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = eggs
+    ... prefer-final = true
+    ... find-links = %(link_server)s
+    ...
+    ... [primed_python]
+    ... executable = %(py_path)s
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... python = primed_python
+    ... eggs = bigdemo
+    ... ''' % globals())
+
+Now here is the output.  The lines that begin with "Egg from site-packages:"
+indicate the eggs from site-packages that have been selected.  You'll see
+we have two: demo 0.3 and demoneeded 1.1.
+
+    >>> print system(py_path+" "+buildout+" -v")
+    Installing 'zc.buildout', 'setuptools'.
+    We have a develop egg: zc.buildout V
+    We have the best distribution that satisfies 'setuptools'.
+    Picked: setuptools = V
+    Installing 'zc.recipe.egg'.
+    We have a develop egg: zc.recipe.egg V
+    Installing eggs.
+    Installing 'bigdemo'.
+    We have no distributions for bigdemo that satisfies 'bigdemo'.
+    Getting distribution for 'bigdemo'.
+    Got bigdemo 0.1.
+    Picked: bigdemo = 0.1
+    Getting required 'demo'
+      required by bigdemo 0.1.
+    We have a develop egg: demo V
+    Egg from site-packages: demo 0.3
+    Getting required 'demoneeded'
+      required by demo 0.3.
+    We have a develop egg: demoneeded V
+    Egg from site-packages: demoneeded 1.1
+    <BLANKLINE>
+    """
 
 def test_comparing_saved_options_with_funny_characters():
     """
@@ -2003,6 +2061,197 @@ Now the install works correctly, as seen here.
 
     """
 
+def isolated_include_site_packages():
+    """
+
+This is an isolated test of the include_site_packages functionality, passing
+the argument directly to install, overriding a default.
+
+Our "py_path" has the "demoneeded" and "demo" packages available.  We'll
+simply be asking for "demoneeded" here.
+
+    >>> py_path, site_packages_path = make_py()
+    >>> create_sample_sys_install(site_packages_path)
+    >>> zc.buildout.easy_install.include_site_packages(False)
+    True
+
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None, include_site_packages=True)
+    >>> [dist.project_name for dist in workingset]
+    ['demoneeded']
+
+That worked fine.  Let's try again with site packages not allowed (and
+reversing the default).
+
+    >>> zc.buildout.easy_install.include_site_packages(True)
+    False
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+    >>> rmdir(example_dest)
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None, include_site_packages=False)
+    Traceback (most recent call last):
+        ...
+    MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+That's a failure, as expected.
+
+Now we explore an important edge case.
+
+Some system Pythons include setuptools (and other Python packages) in their
+site-packages (or equivalent) using a .egg-info directory.  The pkg_resources
+module (from setuptools) considers a package installed using .egg-info to be a
+develop egg.
+
+zc.buildout.buildout.Buildout.bootstrap will make setuptools and zc.buildout
+available to the buildout via the eggs directory, for normal eggs; or the
+develop-eggs directory, for develop-eggs.
+
+If setuptools or zc.buildout is found in site-packages and considered by
+pkg_resources to be a develop egg, then the bootstrap code will use a .egg-link
+in the local develop-eggs, pointing to site-packages, in its entirety.  Because
+develop-eggs must always be available for searching for distributions, this
+indirectly brings site-packages back into the search path for distributions.
+
+Because of this, we have to take special care that we still exclude
+site-packages even in this case.  See the comments about site packages in the
+Installer._satisfied and Installer._obtain methods for the implementation
+(as of this writing).
+
+In this demonstration, we insert a link to the "demoneeded" distribution
+in our develop-eggs, which would bring the package back in, except for
+the special care we have taken to exclude it.
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+    >>> rmdir(example_dest)
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> mkdir(example_dest, 'develop-eggs')
+    >>> write(example_dest, 'develop-eggs', 'demoneeded.egg-link',
+    ...       site_packages_path)
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[],
+    ...     path=[join(example_dest, 'develop-eggs')],
+    ...     executable=py_path,
+    ...     index=None, include_site_packages=False)
+    Traceback (most recent call last):
+        ...
+    MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+The MissingDistribution error shows that buildout correctly excluded the
+"site-packages" source even though it was indirectly included in the path
+via a .egg-link file.
+
+    """
+
+def allowed_eggs_from_site_packages():
+    """
+Sometimes you need or want to control what eggs from site-packages are used.
+The allowed-eggs-from-site-packages option allows you to specify a whitelist of
+project names that may be included from site-packages.  You can use globs to
+specify the value.  It defaults to a single value of '*', indicating that any
+package may come from site-packages.
+
+This option interacts with include-site-packages in the following ways.
+
+If include-site-packages is true, then allowed-eggs-from-site-packages filters
+what eggs from site-packages may be chosen.  If allowed-eggs-from-site-packages
+is an empty list, then no eggs from site-packages are chosen, but site-packages
+will still be included at the end of path lists.
+
+If include-site-packages is false, allowed-eggs-from-site-packages is
+irrelevant.
+
+This test shows the interaction with the zc.buildout.easy_install API.  Another
+test below (allow_site_package_eggs_option) shows using it with a buildout.cfg.
+
+Our "py_path" has the "demoneeded" and "demo" packages available.  We'll
+simply be asking for "demoneeded" here.
+
+    >>> py_path, site_packages_path = make_py()
+    >>> create_sample_sys_install(site_packages_path)
+
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None,
+    ...     allowed_eggs_from_site_packages=['demoneeded', 'other'])
+    >>> [dist.project_name for dist in workingset]
+    ['demoneeded']
+
+That worked fine.  It would work fine for a glob too.
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+    >>> rmdir(example_dest)
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None,
+    ...     allowed_eggs_from_site_packages=['?emon*', 'other'])
+    >>> [dist.project_name for dist in workingset]
+    ['demoneeded']
+
+But now let's try again with 'demoneeded' not allowed.
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+    >>> rmdir(example_dest)
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None,
+    ...     allowed_eggs_from_site_packages=['demo'])
+    Traceback (most recent call last):
+        ...
+    MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+Here's the same, but with an empty list.
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+    >>> rmdir(example_dest)
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None,
+    ...     allowed_eggs_from_site_packages=[])
+    Traceback (most recent call last):
+        ...
+    MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+Of course, this doesn't stop us from getting a package from elsewhere.  Here,
+we add a link server.
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+    >>> rmdir(example_dest)
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, executable=py_path,
+    ...     links=[link_server], index=link_server+'index/',
+    ...     allowed_eggs_from_site_packages=['other'])
+    >>> [dist.project_name for dist in workingset]
+    ['demoneeded']
+    >>> [dist.location for dist in workingset]
+    ['/site-packages-example-install/demoneeded-1.1-py2.6.egg']
+
+Finally, here's an example of an interaction: we say that it is OK to
+allow the "demoneeded" egg to come from site-packages, but we don't
+include-site-packages.
+
+    >>> zc.buildout.easy_install.clear_index_cache()
+    >>> rmdir(example_dest)
+    >>> example_dest = tmpdir('site-packages-example-install')
+    >>> workingset = zc.buildout.easy_install.install(
+    ...     ['demoneeded'], example_dest, links=[], executable=py_path,
+    ...     index=None, include_site_packages=False,
+    ...     allowed_eggs_from_site_packages=['demoneeded'])
+    Traceback (most recent call last):
+        ...
+    MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+    """
+
 if sys.version_info > (2, 4):
     def test_exit_codes():
         """
@@ -2930,44 +3179,29 @@ def create_sample_namespace_eggs(dest, site_packages_path=None):
         finally:
             shutil.rmtree(tmp)
 
-def create_sample_eggs(test, executable=sys.executable):
-    write = test.globs['write']
-    dest = test.globs['sample_eggs']
-    tmp = tempfile.mkdtemp()
-    try:
+def _write_eggrecipedemoneeded(tmp, minor_version, suffix=''):
+    from zc.buildout.testing import write
     write(tmp, 'README.txt', '')
-
-        for i in (0, 1, 2):
-            write(tmp, 'eggrecipedemoneeded.py', 'y=%s\ndef f():\n  pass' % i)
-            c1 = i==2 and 'c1' or ''
+    write(tmp, 'eggrecipedemoneeded.py',
+          'y=%s\ndef f():\n  pass' % minor_version)
     write(
         tmp, 'setup.py',
         "from setuptools import setup\n"
         "setup(name='demoneeded', py_modules=['eggrecipedemoneeded'],"
         " zip_safe=True, version='1.%s%s', author='bob', url='bob', "
         "author_email='bob')\n"
-                % (i, c1)
+        % (minor_version, suffix)
         )
-            zc.buildout.testing.sdist(tmp, dest)
 
-        write(
-            tmp, 'setup.py',
-            "from setuptools import setup\n"
-            "setup(name='other', zip_safe=False, version='1.0', "
-            "py_modules=['eggrecipedemoneeded'])\n"
-            )
-        zc.buildout.testing.bdist_egg(tmp, executable, dest)
-
-        os.remove(os.path.join(tmp, 'eggrecipedemoneeded.py'))
-
-        for i in (1, 2, 3, 4):
+def _write_eggrecipedemo(tmp, minor_version, suffix=''):
+    from zc.buildout.testing import write
+    write(tmp, 'README.txt', '')
     write(
         tmp, 'eggrecipedemo.py',
         'import eggrecipedemoneeded\n'
         'x=%s\n'
         'def main(): print x, eggrecipedemoneeded.y\n'
-                % i)
-            c1 = i==4 and 'c1' or ''
+        % minor_version)
     write(
         tmp, 'setup.py',
         "from setuptools import setup\n"
@@ -2975,10 +3209,46 @@ def create_sample_eggs(test, executable=sys.executable):
         " install_requires = 'demoneeded',"
         " entry_points={'console_scripts': "
              "['demo = eggrecipedemo:main']},"
-                " zip_safe=True, version='0.%s%s')\n" % (i, c1)
+        " zip_safe=True, version='0.%s%s')\n" % (minor_version, suffix)
+        )
+
+def create_sample_sys_install(site_packages_path):
+    for creator, minor_version in (
+        (_write_eggrecipedemoneeded, 1),
+        (_write_eggrecipedemo, 3)):
+        # Write the files and install in site_packages_path.
+        tmp = tempfile.mkdtemp()
+        try:
+            creator(tmp, minor_version)
+            zc.buildout.testing.sys_install(tmp, site_packages_path)
+        finally:
+            shutil.rmtree(tmp)
+
+def create_sample_eggs(test, executable=sys.executable):
+    from zc.buildout.testing import write
+    dest = test.globs['sample_eggs']
+    tmp = tempfile.mkdtemp()
+    try:
+        for i in (0, 1, 2):
+            suffix = i==2 and 'c1' or ''
+            _write_eggrecipedemoneeded(tmp, i, suffix)
+            zc.buildout.testing.sdist(tmp, dest)
+
+        write(
+            tmp, 'setup.py',
+            "from setuptools import setup\n"
+            "setup(name='other', zip_safe=False, version='1.0', "
+            "py_modules=['eggrecipedemoneeded'])\n"
             )
         zc.buildout.testing.bdist_egg(tmp, executable, dest)
 
+        os.remove(os.path.join(tmp, 'eggrecipedemoneeded.py'))
+
+        for i in (1, 2, 3, 4):
+            suffix = i==4 and 'c1' or ''
+            _write_eggrecipedemo(tmp, i, suffix)
+            zc.buildout.testing.bdist_egg(tmp, executable, dest)
+
         write(tmp, 'eggrecipebigdemo.py', 'import eggrecipedemo')
         write(
             tmp, 'setup.py',