fixes for bootstrap and a system Python; changes based on learning what would...

fixes for bootstrap and a system Python; changes based on learning what would be necessary to be able to develop buildout with a system Python (zc.recipe.testing would also need to use sitepackage_safe_scripts)

bootstrap/bootstrap.py

@@ -23,6 +23,35 @@ $Id$
 import os, shutil, sys, tempfile, textwrap, urllib, urllib2
 from optparse import OptionParser
 
+if sys.platform == 'win32':
+    def quote(c):
+        if ' ' in c:
+            return '"%s"' % c # work around spawn lamosity on windows
+        else:
+            return c
+else:
+    quote = str
+
+# In order to be more robust in the face of system Pythons, we want to run
+# with site-packages loaded.  This is somewhat tricky, in particular because
+# Python 2.6's distutils imports site, so starting with the -S flag is not
+# sufficient.
+if 'site' in sys.modules:
+    # We will restart with python -S.
+    args = sys.argv[:]
+    args[0:0] = [sys.executable, '-S']
+    args = map(quote, args)
+    os.execv(sys.executable, args)
+clean_path = sys.path[:]
+import site
+sys.path[:] = clean_path
+for k, v in sys.modules.items():
+    if (hasattr(v, '__path__') and
+        len(v.__path__)==1 and
+        not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))):
+        # This is a namespace package.  Remove it.
+        sys.modules.pop(k)
+
 is_jython = sys.platform.startswith('java')
 
 setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'
@@ -128,16 +157,6 @@ except ImportError:
         if path not in pkg_resources.working_set.entries:
             pkg_resources.working_set.add_entry(path)
 
-if sys.platform == 'win32':
-    def quote(c):
-        if ' ' in c:
-            return '"%s"' % c # work around spawn lamosity on windows
-        else:
-            return c
-else:
-    def quote (c):
-        return c
-
 cmd = [quote(sys.executable),
        '-c',
        quote('from setuptools.command.easy_install import main; main()'),

buildout.cfg

@@ -3,7 +3,7 @@ develop = zc.recipe.egg_ z3c.recipe.scripts_ .
 parts = test oltest py
 
 [py]
-recipe = zc.recipe.egg
+recipe = z3c.recipe.scripts
 eggs = zc.buildout
        zope.testing
 interpreter = py

dev.py

@@ -21,6 +21,35 @@ $Id$
 
 import os, shutil, sys, subprocess, urllib2
 
+if sys.platform == 'win32':
+    def quote(c):
+        if ' ' in c:
+            return '"%s"' % c # work around spawn lamosity on windows
+        else:
+            return c
+else:
+    quote = str
+
+# In order to be more robust in the face of system Pythons, we want to run
+# with site-packages loaded.  This is somewhat tricky, in particular because
+# Python 2.6's distutils imports site, so starting with the -S flag is not
+# sufficient.
+if 'site' in sys.modules:
+    # We will restart with python -S.
+    args = sys.argv[:]
+    args[0:0] = [sys.executable, '-S']
+    args = map(quote, args)
+    os.execv(sys.executable, args)
+clean_path = sys.path[:]
+import site
+sys.path[:] = clean_path
+for k, v in sys.modules.items():
+    if (hasattr(v, '__path__') and
+        len(v.__path__)==1 and
+        not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))):
+        # This is a namespace package.  Remove it.
+        sys.modules.pop(k)
+
 is_jython = sys.platform.startswith('java')
 
 for d in 'eggs', 'develop-eggs', 'bin':
@@ -49,14 +78,20 @@ env = os.environ.copy() # Windows needs yet-to-be-determined values from this.
 env['PYTHONPATH'] = os.path.dirname(pkg_resources.__file__)
 subprocess.Popen(
     [sys.executable] +
-    ['setup.py', '-q', 'develop', '-m', '-x', '-d', 'develop-eggs'],
+    ['-S', 'setup.py', '-q', 'develop', '-m', '-x', '-d', 'develop-eggs'],
     env=env).wait()
 
 pkg_resources.working_set.add_entry('src')
 
 import zc.buildout.easy_install
-zc.buildout.easy_install.scripts(
-    ['zc.buildout'], pkg_resources.working_set , sys.executable, 'bin')
+if not os.path.exists('parts'):
+    os.mkdir('parts')
+partsdir = os.path.join('parts', 'buildout')
+if not os.path.exists(partsdir):
+    os.mkdir(partsdir)
+zc.buildout.easy_install.sitepackage_safe_scripts(
+    'bin', pkg_resources.working_set, sys.executable, partsdir,
+    reqs=['zc.buildout'])
 
 bin_buildout = os.path.join('bin', 'buildout')
 
@@ -64,4 +99,5 @@ if is_jython:
     # Jython needs the script to be called twice via sys.executable
     assert subprocess.Popen([sys.executable] + [bin_buildout]).wait() == 0
 
+
 sys.exit(subprocess.Popen(bin_buildout).wait())

src/zc/buildout/buildout.py

@@ -828,16 +828,19 @@ class Buildout(UserDict.DictMixin):
         if not self.newest:
             return
 
+        options = self['buildout']
+
         ws = zc.buildout.easy_install.install(
             [
-            (spec + ' ' + self['buildout'].get(spec+'-version', '')).strip()
+            (spec + ' ' + options.get(spec+'-version', '')).strip()
             for spec in ('zc.buildout', 'setuptools')
             ],
-            self['buildout']['eggs-directory'],
-            links = self['buildout'].get('find-links', '').split(),
-            index = self['buildout'].get('index'),
-            path = [self['buildout']['develop-eggs-directory']],
-            allow_hosts = self._allow_hosts
+            options['eggs-directory'],
+            links = options.get('find-links', '').split(),
+            index = options.get('index'),
+            path = [options['develop-eggs-directory']],
+            allow_hosts = self._allow_hosts,
+            include_site_packages=False
             )
 
         upgraded = []
@@ -853,7 +856,7 @@ class Buildout(UserDict.DictMixin):
         __doing__ = 'Upgrading.'
 
         should_run = realpath(
-            os.path.join(os.path.abspath(self['buildout']['bin-directory']),
+            os.path.join(os.path.abspath(options['bin-directory']),
                          'buildout')
             )
         if sys.platform == 'win32':
@@ -885,21 +888,26 @@ class Buildout(UserDict.DictMixin):
 
         # the new dist is different, so we've upgraded.
         # Update the scripts and return True
-        zc.buildout.easy_install.scripts(
-            ['zc.buildout'], ws, sys.executable,
-            self['buildout']['bin-directory'],
-            )
+        partsdir = os.path.join(options['parts-directory'], 'buildout')
+        if not os.path.exists(partsdir):
+            os.mkdir(partsdir)
+        zc.buildout.easy_install.sitepackage_safe_scripts(
+            options['bin-directory'], ws, sys.executable, partsdir,
+            reqs=['zc.buildout'])
 
         # Restart
         args = map(zc.buildout.easy_install._safe_arg, sys.argv)
         if not __debug__:
             args.insert(0, '-O')
-        args.insert(0, zc.buildout.easy_install._safe_arg (sys.executable))
-
+        args.insert(0, zc.buildout.easy_install._safe_arg(sys.executable))
+        env = os.environ.copy()
+        env['PYTHONPATH'] = partsdir
         if is_jython:
-            sys.exit(subprocess.Popen([sys.executable] + list(args)).wait())
+            sys.exit(
+                subprocess.Popen(
+                    [sys.executable] + list(args), env=env).wait())
         else:
-            sys.exit(os.spawnv(os.P_WAIT, sys.executable, args))
+            sys.exit(os.spawnve(os.P_WAIT, sys.executable, args, env))
 
     def _load_extensions(self):
         __doing__ = 'Loading extensions.'
@@ -920,7 +928,8 @@ class Buildout(UserDict.DictMixin):
                 working_set=pkg_resources.working_set,
                 links = self['buildout'].get('find-links', '').split(),
                 index = self['buildout'].get('index'),
-                newest=self.newest, allow_hosts=self._allow_hosts)
+                newest=self.newest, allow_hosts=self._allow_hosts,
+                include_site_packages=False)
 
             # Clear cache because extensions might now let us read pages we
             # couldn't read before.

src/zc/buildout/easy_install.py

@@ -1486,8 +1486,10 @@ def _get_module_file(executable, name):
            "fp, path, desc = imp.find_module(%r); "
            "fp.close; "
            "print path" % (name,)]
+    env = os.environ.copy()
+    env.pop('PYTHONPATH', None)
     _proc = subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
     stdout, stderr = _proc.communicate();
     if _proc.returncode:
         logger.info(
@@ -1583,7 +1585,9 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
         site.close()
         real_site.close()
     if not successful_rewrite:
-        raise RuntimeError('Buildout did not successfully rewrite site.py')
+        raise RuntimeError(
+            'Buildout did not successfully rewrite %s to %s' %
+            (real_site_path, site_path))
     return site_path
 
 namespace_include_site_packages_setup = '''

src/zc/buildout/update.txt

@@ -78,22 +78,26 @@ new versions found in new releases:
     zc.buildout 99.99
     setuptools 99.99
 
-Our buildout script has been updated to use the new eggs:
+Our buildout script's site.py has been updated to use the new eggs:
 
-    >>> cat(sample_buildout, 'bin', 'buildout')
-    ... # doctest: +NORMALIZE_WHITESPACE
-    #!/usr/local/bin/python2.4
+    >>> cat(sample_buildout, 'parts', 'buildout', 'site.py')
+    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+    "...
+    def addsitepackages(known_paths):
+        """Add site packages, as determined by zc.buildout.
     <BLANKLINE>
-    import sys
-    sys.path[0:0] = [
-      '/sample-buildout/eggs/zc.buildout-99.99-py2.4.egg',
-      '/sample-buildout/eggs/setuptools-99.99-py2.4.egg',
+        See original_addsitepackages, below, for the original version."""
+        buildout_paths = [
+            '/sample-buildout/eggs/zc.buildout-99.99-pyN.N.egg',
+            '/sample-buildout/eggs/setuptools-99.99-pyN.N.egg'
             ]
-    <BLANKLINE>
-    import zc.buildout.buildout
-    <BLANKLINE>
-    if __name__ == '__main__':
-        zc.buildout.buildout.main()
+        for path in buildout_paths:
+            sitedir, sitedircase = makepath(path)
+            if not sitedircase in known_paths and os.path.exists(sitedir):
+                sys.path.append(sitedir)
+                known_paths.add(sitedircase)
+        return known_paths
+    ...
 
 Now, let's recreate the sample buildout. If we specify constraints on
 the versions of zc.buildout and setuptools to use, running the
@@ -120,7 +124,6 @@ Now we can see that we actually "upgrade" to an earlier version.
       zc.buildout version 1.0.0,
       setuptools version 0.6;
     restarting.
-    Generated script '/sample-buildout/bin/buildout'.
     Develop: '/sample-buildout/showversions'
     Updating show-versions.
     zc.buildout 1.0.0