Merge pull request #5 from buildout/nosite

Nosite

bootstrap/bootstrap.py

@@ -23,66 +23,135 @@ from optparse import OptionParser
 
 tmpeggs = tempfile.mkdtemp()
 
-# parsing arguments
-parser = OptionParser(
-    'This is a custom version of the zc.buildout %prog script.  It is '
-    'intended to meet a temporary need if you encounter problems with '
-    'the zc.buildout 1.5 release.')
-parser.add_option("-v", "--version", dest="version", default='1.4.4',
-                          help='Use a specific zc.buildout version.  *This '
-                          'bootstrap script defaults to '
-                          '1.4.4, unlike usual buildpout bootstrap scripts.*')
+usage = '''\
+[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
+
+Bootstraps a buildout-based project.
+
+Simply run this script in a directory containing a buildout.cfg, using the
+Python that you want bin/buildout to use.
+
+Note that by using --setup-source and --download-base to point to
+local resources, you can keep this script from going over the network.
+'''
+
+parser = OptionParser(usage=usage)
+parser.add_option("-v", "--version", help="use a specific zc.buildout version")
+
+parser.add_option("-t", "--accept-buildout-test-releases",
+                  dest='accept_buildout_test_releases',
+                  action="store_true", default=False,
+                  help=("Normally, if you do not specify a --version, the "
+                        "bootstrap script and buildout gets the newest "
+                        "*final* versions of zc.buildout and its recipes and "
+                        "extensions for you.  If you use this flag, "
+                        "bootstrap and buildout will get the newest releases "
+                        "even if they are alphas or betas."))
 parser.add_option("-c", None, action="store", dest="config_file",
                    help=("Specify the path to the buildout configuration "
                          "file to be used."))
 
 options, args = parser.parse_args()
 
-# if -c was provided, we push it back into args for buildout' main function
-if options.config_file is not None:
-    args += ['-c', options.config_file]
+######################################################################
+# handle -S
 
-if options.version is not None:
-    VERSION = '==%s' % options.version
+def normpath(p):
+    return p[:-1] if p.endswith(os.path.sep) else p
+
+nosite = 'site' not in sys.modules
+if nosite:
+    # They've asked not to import site.  Cool, but distribute is going to
+    # import it anyway, so we're going to have to clean up. :(
+    initial_paths = set(map(normpath, sys.path))
+    import site
+    to_remove = set(map(normpath, sys.path)) - initial_paths
 else:
-    VERSION = ''
+    to_remove = ()
 
-args = args + ['bootstrap']
+######################################################################
+# load/install distribute
 
 to_reload = False
 try:
-    import pkg_resources
+    import pkg_resources, setuptools
     if not hasattr(pkg_resources, '_distribute'):
         to_reload = True
         raise ImportError
 except ImportError:
     ez = {}
-    exec urllib2.urlopen('http://python-distribute.org/distribute_setup.py'
+    exec urllib2.urlopen(
+        'http://python-distribute.org/distribute_setup.py'
         ).read() in ez
-    ez['use_setuptools'](to_dir=tmpeggs, download_delay=0, no_fake=True)
+    setup_args = dict(to_dir=tmpeggs, download_delay=0, no_fake=True)
+    ez['use_setuptools'](**setup_args)
 
     if to_reload:
         reload(pkg_resources)
-    else:
     import pkg_resources
+    # This does not (always?) update the default working set.  We will
+    # do it.
+    for path in sys.path:
+        if path not in pkg_resources.working_set.entries:
+            pkg_resources.working_set.add_entry(path)
+
+# Clean up
+if nosite and 'site' in sys.modules:
+    del sys.modules['site']
+    sys.path[:] = [p for p in sys.path[:]
+        if normpath(p) not in to_remove
+        ]
+
+######################################################################
+# Install buildout
 
 ws  = pkg_resources.working_set
 
-requirement = 'distribute'
-
-env = dict(os.environ,
-           PYTHONPATH=
-           ws.find(pkg_resources.Requirement.parse(requirement)).location
-           )
-
 cmd = [sys.executable, '-c',
        'from setuptools.command.easy_install import main; main()',
        '-mZqNxd', tmpeggs]
 
-if 'bootstrap-testing-find-links' in os.environ:
-    cmd.extend(['-f', os.environ['bootstrap-testing-find-links']])
-
-cmd.append('zc.buildout' + VERSION)
+find_links = os.environ.get('bootstrap-testing-find-links')
+if find_links:
+    cmd.extend(['-f', find_links])
+
+distribute_path = ws.find(
+    pkg_resources.Requirement.parse('distribute')).location
+env = dict(os.environ, PYTHONPATH=distribute_path)
+
+requirement = 'zc.buildout'
+version = options.version
+if version is None and not options.accept_buildout_test_releases:
+    # Figure out the most recent final version of zc.buildout.
+    import setuptools.package_index
+    _final_parts = '*final-', '*final'
+    def _final_version(parsed_version):
+        for part in parsed_version:
+            if (part[:1] == '*') and (part not in _final_parts):
+                return False
+        return True
+    index = setuptools.package_index.PackageIndex(
+        search_path=[distribute_path])
+    if find_links:
+        index.add_find_links((find_links,))
+    req = pkg_resources.Requirement.parse(requirement)
+    if index.obtain(req) is not None:
+        best = []
+        bestv = None
+        for dist in index[req.project_name]:
+            distv = dist.parsed_version
+            if _final_version(distv):
+                if bestv is None or distv > bestv:
+                    best = [dist]
+                    bestv = distv
+                elif distv == bestv:
+                    best.append(dist)
+        if best:
+            best.sort()
+            version = best[-1].version
+if version:
+    requirement = '=='.join((requirement, version))
+cmd.append(requirement)
 
 import subprocess
 if subprocess.call(cmd, env=env) != 0:
@@ -90,8 +159,23 @@ if subprocess.call(cmd, env=env) != 0:
         "Failed to execute command:\n%s",
         repr(cmd)[1:-1])
 
+######################################################################
+# Import and run buildout
+
 ws.add_entry(tmpeggs)
-ws.require('zc.buildout' + VERSION)
+ws.require(requirement)
 import zc.buildout.buildout
+
+if not args:
+    # Note that if there are args, they may be for another command, say, init.
+    args = ['bootstrap']
+
+if options.accept_buildout_test_releases:
+    args.append('buildout:accept-buildout-test-releases=true')
+
+# if -c was provided, we push it back into args for buildout' main function
+if options.config_file is not None:
+    args[0:0] = ['-c', options.config_file]
+
 zc.buildout.buildout.main(args)
 shutil.rmtree(tmpeggs)

dev.py

@@ -19,30 +19,65 @@ buildout egg itself is installed as a develop egg.
 
 import os, shutil, sys, subprocess, urllib2
 
-is_jython = sys.platform.startswith('java')
-
-for d in 'eggs', 'develop-eggs', 'bin':
+for d in 'eggs', 'develop-eggs', 'bin', 'parts':
     if not os.path.exists(d):
         os.mkdir(d)
 
 if os.path.isdir('build'):
     shutil.rmtree('build')
 
+######################################################################
+# handle -S
+
+def normpath(p):
+    return p[:-1] if p.endswith(os.path.sep) else p
+
+nosite = 'site' not in sys.modules
+if nosite:
+    # They've asked not to import site.  Cool, but distribute is going to
+    # import it anyway, so we're going to have to clean up. :(
+    initial_paths = set(map(normpath, sys.path))
+    import site
+    to_remove = set(map(normpath, sys.path)) - initial_paths
+else:
+    to_remove = ()
+
+######################################################################
+# Make sure we have a relatively clean environment
 try:
-    import pkg_resources
+    import pkg_resources, setuptools
 except ImportError:
-    ez = {}
-    exec urllib2.urlopen(
-        'http://python-distribute.org/distribute_setup.py'
-        ).read() in ez
-    ez['use_setuptools'](to_dir='eggs', download_delay=0)
+    pass
+else:
+    raise SystemError(
+        "Buildout development with a pre-installed setuptools or "
+        "distribute is not supported.%s"
+        % ('' if nosite else ' Try running with -S option to Python.'))
+
+######################################################################
+# Install distribute
+ez = {}
+exec urllib2.urlopen(
+    'http://python-distribute.org/distribute_setup.py').read() in ez
+ez['use_setuptools'](to_dir='eggs', download_delay=0)
+
+import pkg_resources
+
+# Clean up
+if nosite and 'site' in sys.modules:
+    del sys.modules['site']
+    sys.path[:] = [p for p in sys.path[:]
+        if normpath(p) not in to_remove
+        ]
 
-    import pkg_resources
+######################################################################
+# Install buildout
 
-subprocess.Popen(
+if subprocess.call(
     [sys.executable] +
     ['setup.py', '-q', 'develop', '-m', '-x', '-d', 'develop-eggs'],
-    env = {'PYTHONPATH': os.path.dirname(pkg_resources.__file__)}).wait()
+    env = {'PYTHONPATH': os.path.dirname(pkg_resources.__file__)}):
+    raise RuntimeError("buildout build failed.")
 
 pkg_resources.working_set.add_entry('src')
 
@@ -52,7 +87,7 @@ zc.buildout.easy_install.scripts(
 
 bin_buildout = os.path.join('bin', 'buildout')
 
-if is_jython:
+if sys.platform.startswith('java'):
     # Jython needs the script to be called twice via sys.executable
     assert subprocess.Popen([sys.executable] + [bin_buildout]).wait() == 0
 

src/zc/buildout/buildout.py

@@ -14,12 +14,12 @@
 """Buildout main script
 """
 
+# handle -S
+import zc.buildout.easy_install
+no_site = zc.buildout.easy_install.no_site
+
 from rmtree import rmtree
-try:
-    from hashlib import md5
-except ImportError:
-    # Python 2.4 and older
-    from md5 import md5
+from hashlib import md5
 
 import ConfigParser
 import copy
@@ -37,7 +37,6 @@ import tempfile
 import UserDict
 import zc.buildout
 import zc.buildout.download
-import zc.buildout.easy_install
 
 
 realpath = zc.buildout.easy_install.realpath
@@ -906,6 +905,8 @@ class Buildout(UserDict.DictMixin):
             args.insert(1, '-W')
             if not __debug__:
                 args.insert(0, '-O')
+            if no_site:
+                args.insert(0, '-S')
             args.insert(0, zc.buildout.easy_install._safe_arg (sys.executable))
             os.execv(sys.executable, args)
 
@@ -929,6 +930,8 @@ class Buildout(UserDict.DictMixin):
         args = sys.argv[:]
         if not __debug__:
             args.insert(0, '-O')
+        if no_site:
+            args.insert(0, '-S')
         args.insert(0, sys.executable)
         sys.exit(subprocess.call(args))
 
@@ -989,8 +992,10 @@ class Buildout(UserDict.DictMixin):
                 setup=setup,
                 __file__ = setup,
                 ))
-            zc.buildout.easy_install.call_subprocess(
-                [sys.executable, tsetup] + args)
+            args = [sys.executable, tsetup] + args
+            if no_site:
+                args.insert(1, '-S')
+            zc.buildout.easy_install.call_subprocess(args)
         finally:
             os.close(fd)
             os.remove(tsetup)

src/zc/buildout/easy_install.py

@@ -18,10 +18,26 @@ It doesn't install scripts.  It uses distribute and requires it to be
 installed.
 """
 
+import os
+import sys
+
+######################################################################
+# handle -S
+
+def normpath(p):
+    return p[:-1] if p.endswith(os.path.sep) else p
+
+no_site = 'site' not in sys.modules
+if no_site:
+    initial_paths = set(map(normpath, sys.path))
+    import site
+    sys.path[:] = [p for p in sys.path if normpath(p) in initial_paths]
+#
+######################################################################
+
 import distutils.errors
 import glob
 import logging
-import os
 import pkg_resources
 import py_compile
 import re
@@ -30,7 +46,6 @@ import setuptools.command.setopt
 import setuptools.package_index
 import shutil
 import subprocess
-import sys
 import tempfile
 import zc.buildout
 import zipimport
@@ -273,6 +288,8 @@ class Installer:
             path = distribute_loc
 
             args = [sys.executable, '-c', _easy_install_cmd, '-mZUNxd', tmp]
+            if no_site:
+                args.insert(1, '-S')
             level = logger.getEffectiveLevel()
             if level > 0:
                 args.append('-q')
@@ -736,7 +753,7 @@ def build(spec, dest, build_ext,
           executable=sys.executable,
           path=None, newest=True, versions=None, allow_hosts=('*',)):
     assert executable == sys.executable, (executable, sys.executable)
-    installer = Installer(dest, links, index, sys.executable,
+    installer = Installer(dest, links, index, executable,
                           True, path, newest,
                           versions, allow_hosts=allow_hosts)
     return installer.build(spec, build_ext)
@@ -806,7 +823,7 @@ def develop(setup, dest,
         tmp3 = tempfile.mkdtemp('build', dir=dest)
         undo.append(lambda : shutil.rmtree(tmp3))
 
-        args = [sys.executable,  tsetup, '-q', 'develop', '-mxN', '-d', tmp3]
+        args = [executable,  tsetup, '-q', 'develop', '-mxN', '-d', tmp3]
 
         log_level = logger.getEffectiveLevel()
         if log_level <= 0:
@@ -817,6 +834,9 @@ def develop(setup, dest,
         if log_level < logging.DEBUG:
             logger.debug("in: %r\n%s", directory, ' '.join(args))
 
+        if no_site:
+            args.insert(1, '-S')
+
         call_subprocess(args)
 
         return _copyeggs(tmp3, dest, '.egg-link', undo)
@@ -913,8 +933,7 @@ def scripts(reqs, working_set, executable, dest=None,
         spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)
 
         generated.extend(
-            _distutils_script(spath, sname, contents,
-                              executable, initialization, rpsetup)
+            _distutils_script(spath, sname, contents, initialization, rpsetup)
             )
 
     if interpreter:
@@ -994,8 +1013,12 @@ def _script(module_name, attrs, path, dest, arguments, initialization, rsetup):
     if is_win32:
         dest += '-script.py'
 
+    python = _safe_arg(sys.executable)
+    if no_site:
+        python += ' -S'
+
     contents = script_template % dict(
-        python = _safe_arg(sys.executable),
+        python = python,
         path = path,
         module_name = module_name,
         attrs = attrs,
@@ -1006,16 +1029,20 @@ def _script(module_name, attrs, path, dest, arguments, initialization, rsetup):
     return _create_script(contents, dest)
 
 
-def _distutils_script(path, dest, script_content, executable,
-                      initialization, rsetup):
+def _distutils_script(path, dest, script_content, initialization, rsetup):
 
     lines = script_content.splitlines(True)
     if not ('#!' in lines[0]) and ('python' in lines[0]):
         # The script doesn't follow distutil's rules.  Ignore it.
         return []
     original_content = ''.join(lines[1:])
+
+    python = _safe_arg(sys.executable)
+    if no_site:
+        python += ' -S'
+
     contents = distutils_script_template % dict(
-        python = _safe_arg(executable),
+        python = python,
         path = path,
         initialization = initialization,
         relative_paths_setup = rsetup,
@@ -1093,8 +1120,12 @@ def _pyscript(path, dest, rsetup):
     if is_win32:
         dest += '-script.py'
 
+    python = _safe_arg(sys.executable)
+    if no_site:
+        python += ' -S'
+
     contents = py_script_template % dict(
-        python = _safe_arg(sys.executable),
+        python = python,
         path = path,
         relative_paths_setup = rsetup,
         )

src/zc/buildout/isolation.txt

+Isolation
+=========
+
+When developing applications, we recommended that you work with a
+minimal Python environment and let buildout install needed
+dependencies.  If you share a Python environment with other
+applications [#simple]_, you're likely to have unexpected
+interactions with the other applications, through extra modules or
+module version clashes.
+
+There are a number of ways to achive isolation from other
+applications:
+
+1. Use a "clean" Python installation that doesn't contain any
+   non-standard modules.
+
+2. Use a 'virtual environment
+   <http://pypi.python.org/pypi/virtualenv>'_ that wraps a Python
+   installation in a way that hides 3rd-party modules.  `This is buildt
+   into Python 3.3 <http://docs.python.org/dev/library/venv.html>`_
+
+3. Use the -S Python option when bootstrapping buildout.  This will
+   bypass the normal execution of the Python site.py file, which is
+   what usually causes directories containing third-party modules to be
+   added to the Python path on most platforms [#macos]_.
+
+   If the -S option is used when bootstrapping buildout, then it will
+   be remembered by buildout and any scripts generated by buildout
+   will also use the -S option.
+
+The first option works best in our opinion because, in addition to
+isolating you from extra third-party libraries, it protects you from
+other changes that a preexisting build might have, such as missing
+standard libraries or non-standard Python build options.
+
+This document tests the 3rd option.
+
+    >>> sample_buildout = tmpdir('sample')
+    >>> cd(sample_buildout)
+    >>> import sys
+    >>> print system("%s -S %s init demo" % (sys.executable, bootstrap_py)),
+    ... # doctest: +ELLIPSIS
+    Downloading http://pypi.python.org/packages/source/d/distribute/...
+    Creating '/sample/buildout.cfg'.
+    Creating directory '/sample/bin'.
+    Creating directory '/sample/parts'.
+    Creating directory '/sample/eggs'.
+    Creating directory '/sample/develop-eggs'.
+    Generated script '/sample/bin/buildout'...
+    Getting distribution for 'zc.recipe.egg'.
+    Got zc.recipe.egg 1.3.3dev.
+    Installing py.
+    Getting distribution for 'demo'.
+    Got demo 0.4c1.
+    Getting distribution for 'demoneeded'.
+    Got demoneeded 1.2c1.
+    Generated script '/sample/bin/demo'.
+    Generated interpreter '/sample/bin/py'.
+
+If we look at the scripts generated, se see that the -S option is
+provided to the Python executable:
+
+    >>> cat('bin', 'buildout') # doctest: +ELLIPSIS
+    #!/usr/local/bin/python2.7 -S
+    ...
+
+    >>> cat('bin', 'demo') # doctest: +ELLIPSIS
+    #!/usr/local/bin/python2.7 -S
+    ...
+
+    >>> cat('bin', 'py') # doctest: +ELLIPSIS
+    #!/usr/local/bin/python2.7 -S
+    ...
+
+The -S option is also used when invoking setup scripts.
+
+    >>> mkdir('proj')
+    >>> write('proj', 'setup.py', """
+    ... from distutils.core import setup
+    ... import sys
+    ... print 'site:', 'site' in sys.modules
+    ... setup(name='hassite')
+    ... """)
+    >>> print system(join('bin', 'buildout')+' setup proj sdist')
+    ... # doctest: +ELLIPSIS
+    Running setup script 'proj/setup.py'.
+    site: True
+    ...
+
+    >>> write('buildout.cfg', """
+    ... [buildout]
+    ... parts = egg
+    ... find-links = %s
+    ... [egg]
+    ... recipe = zc.recipe.egg
+    ... eggs = hassite
+    ... """ % join('proj', 'dist'))
+    >>> print system(join('bin', 'buildout'))
+    ... # doctest: +ELLIPSIS
+    Uninstalling py.
+    Installing egg.
+    Getting distribution for 'hassite'.
+    site: True
+    ...
+
+
+
+    >>> write('buildout.cfg', """
+    ... [buildout]
+    ... parts =
+    ... develop = %s
+    ... """ % join('proj'))
+    >>> print system(join('bin', 'buildout'))
+    ... # doctest: +ELLIPSIS
+    Develop: '/sample/proj'
+    site: True
+    ...
+
+
+.. [#simple] It's worth noting that system Python builds can make
+   simple applications easier to build, as hard-to-build extension
+   modules are already installed or can be easily installed using
+   system packaging tools.
+
+.. [#macos] On Mac OS X, there are some directories containing
+   3rd-party libraries that are added to the Python path regardless of
+   whether site.py is loaded.  Still, even on Mac OS X, use of the -S
+   option prevents some 3rd-party modules from being added to the
+   Python path.
+

src/zc/buildout/tests.py

@@ -2832,13 +2832,20 @@ def updateSetup(test):
     makeNewRelease('distribute', ws, new_releases)
     os.mkdir(os.path.join(new_releases, 'distribute'))
 
+bootstrap_py = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.dirname(
+        os.path.dirname(__file__)))),
+    'bootstrap', 'bootstrap.py')
+
 def bootstrapSetup(test):
-    easy_install_SetUp(test)
+    buildout_txt_setup(test)
+    test.globs['link_server'] = test.globs['start_server'](
+        test.globs['sample_eggs'])
     sample_eggs = test.globs['sample_eggs']
     ws = getWorkingSetWithBuildoutEgg(test)
-    makeNewRelease('zc.buildout', ws, sample_eggs, '1.4.4')
+    makeNewRelease('zc.buildout', ws, sample_eggs, '2.0.0')
     os.environ['bootstrap-testing-find-links'] = test.globs['link_server']
-
+    test.globs['bootstrap_py'] = bootstrap_py
 
 normalize_bang = (
     re.compile(re.escape('#!'+
@@ -2846,6 +2853,11 @@ normalize_bang = (
     '#!/usr/local/bin/python2.7',
     )
 
+normalize_S = (
+    re.compile(r'#!/usr/local/bin/python2.7 -S'),
+    '#!/usr/local/bin/python2.7',
+    )
+
 def test_suite():
     test_suite = [
         doctest.DocFileSuite(
@@ -2937,8 +2949,10 @@ def test_suite():
                 zc.buildout.testing.normalize_script,
                 zc.buildout.testing.normalize_egg_py,
                 normalize_bang,
+                normalize_S,
                 (re.compile('99[.]99'), 'NINETYNINE.NINETYNINE'),
-               (re.compile(r'(zc.buildout|distribute)( version)? \d+[.]\d+\S*'),
+                (re.compile(
+                    r'(zc.buildout|distribute)( version)? \d+[.]\d+\S*'),
                  '\\1 V.V'),
                 (re.compile('[-d]  distribute'), '-  distribute'),
                ])
@@ -2955,6 +2969,7 @@ def test_suite():
                 zc.buildout.testing.normalize_script,
                 zc.buildout.testing.normalize_egg_py,
                 normalize_bang,
+                normalize_S,
                 (re.compile('extdemo[.]pyd'), 'extdemo.so'),
                 (re.compile('[-d]  distribute-\S+[.]egg'), 'distribute.egg'),
                 (re.compile(r'\\[\\]?'), '/'),
@@ -3005,7 +3020,8 @@ def test_suite():
                    '-q develop -mxN -d /sample-buildout/develop-eggs'
                 ),
                (re.compile(r'^[*]...'), '...'),
-               # for bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section
+               # for
+               # bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section
                (re.compile(r"Unused options for buildout: 'eggs' 'scripts'\."),
                 "Unused options for buildout: 'scripts' 'eggs'."),
                ]),
@@ -3043,21 +3059,11 @@ def test_suite():
             'testing_bugfix.txt'),
     ]
 
-    # adding bootstrap.txt doctest to the suite
+    # adding bootstrap.txt and isolation.txt doctest to the suite
     # only if bootstrap.py is present
-    bootstrap_py = os.path.join(
-       os.path.dirname(
-          os.path.dirname(
-             os.path.dirname(
-                os.path.dirname(zc.buildout.__file__)
-                )
-             )
-          ),
-       'bootstrap', 'bootstrap.py')
-
     if os.path.exists(bootstrap_py):
         test_suite.append(doctest.DocFileSuite(
-            'bootstrap.txt',
+            'bootstrap.txt', 'isolation.txt',
             setUp=bootstrapSetup,
             tearDown=zc.buildout.testing.buildoutTearDown,
             checker=renormalizing.RENormalizing([

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -47,6 +47,7 @@ def test_suite():
                zc.buildout.testing.normalize_script,
                zc.buildout.testing.normalize_egg_py,
                zc.buildout.tests.normalize_bang,
+               zc.buildout.tests.normalize_S,
                (re.compile('[d-]  zc.buildout(-\S+)?[.]egg(-link)?'),
                 'zc.buildout.egg'),
                (re.compile('[d-]  distribute-[^-]+-'), 'distribute-X-'),