Merge pull request #248 from buildout/reinout-bootstrap-dev-eggs

Bootstrap fixes, including develop-eggs removal

CHANGES.rst

@@ -4,6 +4,10 @@ Change History
 2.4.0 (unreleased)
 ==================
 
+- Buildout no longer breaks on packages that contain a file with a non-ascii
+  filename. Fixes #89 and #148.
+  [reinout]
+
 - Undo breakage on Windows machines where ``sys.prefix`` can also be a
   ``site-packages`` directory:  don't remove it from ``sys.path``.  See
   https://github.com/buildout/buildout/issues/217 .
@@ -22,6 +26,11 @@ Change History
   https://github.com/buildout/buildout/pull/222 .
   [lrowe]
 
+- Updated buildout's `travis-ci <https://travis-ci.org/buildout/buildout>`_
+  configuration so that tests run much quicker so that buildout is easier and
+  quicker to develop.
+  [reinout]
+
 - Note: zc.recipe.egg has also been updated to 2.0.2 together with this
   zc.buildout release. Fixed: In ``zc.recipe.egg#custom`` recipe's ``rpath``
   support, don't assume path elements are buildout-relative if they start with
@@ -29,21 +38,6 @@ Change History
   https://github.com/buildout/buildout/issues/225.
   [tseaver]
 
-- Bootstrap script now accepts ``--to-dir``. Setuptools is installed there. If
-  already available there, it is reused. This can be used to bootstrap
-  buildout without internet access. Similarly, a local ``ez_setup.py`` is used
-  when available instead of it being downloaded. You need setuptools 14.0 or
-  higher for this functionality.
-  [lrowe]
-
-- Buildout no longer breaks on packages that contain a file with a non-ascii
-  filename. Fixes #89 and #148.
-  [reinout]
-
-- Updated buildout's `travis-ci <https://travis-ci.org/buildout/buildout>`_
-  configuration so that tests run much quicker so that buildout is easier and
-  quicker to develop.
-
 - ``download-cache``, ``eggs-directory`` and ``extends-cache`` are now
   automatically created if their parent directory exists. Also they can be
   relative directories (relative to the location of the buildout config file
@@ -52,6 +46,30 @@ Change History
   running buildout.
   [lelit]
 
+- A new boostrap.py file is released (version 2015-07-01).
+
+- When bootstrapping, the ``develop-eggs/`` directory is first removed. This
+  prevents old left-over ``.egg-link`` files from breaking buildout's careful
+  package collection mechanism.
+  [reinout]
+
+- The bootstrap script now accepts ``--to-dir``. Setuptools is installed
+  there. If already available there, it is reused. This can be used to
+  bootstrap buildout without internet access. Similarly, a local
+  ``ez_setup.py`` is used when available instead of it being downloaded. You
+  need setuptools 14.0 or higher for this functionality.
+  [lrowe]
+
+- The bootstrap script now uses ``--buildout-version`` instead of
+  ``--version`` to pick a specific buildout version.
+  [reinout]
+
+- The bootstrap script now accepts ``--version`` which prints the bootstrap
+  version. This version is the date the bootstrap.py was last changed. A date
+  is handier or less confusing than either tracking zc.buildout's version or
+  having a separate bootstrap version number.
+  [reinout]
+
 2.3.1 (2014-12-16)
 ==================
 

DEVELOPERS.txt

@@ -29,3 +29,32 @@ with them do::
 
 The actual Python compilation is only done once and then re-used. So on
 subsequent builds, only the development buildout itself needs to be redone.
+
+
+Releases: zc.buildout, zc.recipe.egg and bootstrap.py
+-----------------------------------------------------
+
+Buildout consists of two python packages that are released separately:
+zc.buildout and zc.recipe.egg. zc.recipe.egg is changed much less often than
+zc.buildout.
+
+zc.buildout's setup.py and changelog is in the same directory as this
+``DEVELOPERS.txt`` and the code is in ``src/zc/buildout``.
+
+zc.recipe.egg, including setup.py and a separate changelog, is in the
+``zc.recipe.egg_`` subdirectory.
+
+A third item is the bootstrap.py file in ``bootstrap/bootstrap.py``. The
+canonical location is at https://bootstrap.pypa.io/bootstrap-buildout.py,
+(though it is unfortunate that it isn't named just ``bootstrap.py``). This
+file is pulled automatically every 15 minutes from the bootstrap-release
+branch. When a new buildout release is made, **the releaser** should update
+the bootstrap-release branch, too.
+
+The http://downloads.buildout.org/2/bootstrap.py location doesn't need to be
+updated manually: it is a redirect now, to
+https://bootstrap.pypa.io/bootstrap-buildout.py .
+
+If there are changes to bootstrap.py, be sure to update the date in the
+``__version__`` attribute and to record the bootstrap change (including the
+new date/version) in zc.buildout's changelog.

bootstrap/bootstrap.py

@@ -25,7 +25,10 @@ import tempfile
 
 from optparse import OptionParser
 
-tmpeggs = tempfile.mkdtemp()
+__version__ = '2015-07-01'
+# See zc.buildout's changelog if this version is up to date.
+
+tmpeggs = tempfile.mkdtemp(prefix='bootstrap-')
 
 usage = '''\
 [DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
@@ -40,8 +43,9 @@ this script from going over the network.
 '''
 
 parser = OptionParser(usage=usage)
-parser.add_option("-v", "--version", help="use a specific zc.buildout version")
-
+parser.add_option("--version",
+                  action="store_true", default=False,
+                  help=("Return bootstrap.py version."))
 parser.add_option("-t", "--accept-buildout-test-releases",
                   dest='accept_buildout_test_releases',
                   action="store_true", default=False,
@@ -59,21 +63,24 @@ parser.add_option("-f", "--find-links",
 parser.add_option("--allow-site-packages",
                   action="store_true", default=False,
                   help=("Let bootstrap.py use existing site packages"))
+parser.add_option("--buildout-version",
+                  help="Use a specific zc.buildout version")
 parser.add_option("--setuptools-version",
-                  help="use a specific setuptools version")
+                  help="Use a specific setuptools version")
 parser.add_option("--setuptools-to-dir",
-                  help=("allow for re-use of existing directory of "
+                  help=("Allow for re-use of existing directory of "
                         "setuptools versions"))
 
 options, args = parser.parse_args()
+if options.version:
+    print("bootstrap.py version %s" % __version__)
+    sys.exit(0)
+
 
 ######################################################################
 # load/install setuptools
 
 try:
-    if options.allow_site_packages:
-        import setuptools
-        import pkg_resources
     from urllib.request import urlopen
 except ImportError:
     from urllib2 import urlopen
@@ -97,7 +104,8 @@ if not options.allow_site_packages:
             # are not sys.prefix; this is because on Windows
             # sys.prefix is a site-package directory.
             if sitepackage_path != sys.prefix:
-                sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
+                sys.path[:] = [x for x in sys.path
+                               if sitepackage_path not in x]
 
 setup_args = dict(to_dir=tmpeggs, download_delay=0)
 
@@ -140,7 +148,7 @@ if find_links:
     cmd.extend(['-f', find_links])
 
 requirement = 'zc.buildout'
-version = options.version
+version = options.buildout_version
 if version is None and not options.accept_buildout_test_releases:
     # Figure out the most recent final version of zc.buildout.
     import setuptools.package_index

src/zc/buildout/bootstrap.txt

@@ -59,14 +59,14 @@ By default it gets the latest version:
       '/sample/eggs/zc.buildout-22.0.0...egg',
       ]...
 
-Now trying the `--version` option, that let you define a version for
+Now trying the `--buildout-version` option, that let you define a version for
 `zc.buildout`.
 
 Let's try with an unknown version::
 
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
-    ...     'bootstrap.py --version UNKNOWN')); print_('X') # doctest: +ELLIPSIS
+    ...     'bootstrap.py --buildout-version UNKNOWN')); print_('X') # doctest: +ELLIPSIS
     ...
     X
     ...
@@ -77,7 +77,7 @@ Now let's try with `2.0.0`, which happens to exist::
 
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
-    ...     'bootstrap.py --version 2.0.0')); print_('X')
+    ...     'bootstrap.py --buildout-version 2.0.0')); print_('X')
     ... # doctest: +ELLIPSIS
     X...Generated script '/sample/bin/buildout'...X
 
@@ -115,7 +115,7 @@ which happens to exist::
 
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
-    ...     'bootstrap.py --setuptools-version 8.0 --version 2.0.0')); print_('X')
+    ...     'bootstrap.py --setuptools-version 8.0 --buildout-version 2.0.0')); print_('X')
     ... # doctest: +ELLIPSIS
     X...Generated script '/sample/bin/buildout'...X
 
@@ -145,7 +145,7 @@ specify the setuptools version, and to reuse the setuptools zipfile.
 
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
-    ...     'bootstrap.py --setuptools-version 14.3 --version 2.0.0 '+
+    ...     'bootstrap.py --setuptools-version 14.3 --buildout-version 2.0.0 '+
     ...     '--setuptools-to-dir .')); print_('X')
     ... # doctest: +ELLIPSIS
     X...Using local ez_setup.py...Generated script '/sample/bin/buildout'...X
@@ -159,3 +159,16 @@ specify the setuptools version, and to reuse the setuptools zipfile.
       '/sample/eggs/setuptools-14.3...egg',
       '/sample/eggs/zc.buildout-2.0.0...egg',
       ]...
+
+You can ask bootstrap.py for its version. This is really the day the last
+change was made. A date leads to less confusion than a version number separate
+from buildout's own version number. Similarly, tracking buildout's version
+number leads to bootstraps with a new version number but without changes or to
+bootstraps with version number from an already-older buildout. So: a date is
+best.
+
+    >>> print_('X'); print_(system(
+    ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
+    ...     'bootstrap.py --version')); print_('X')
+    ... # doctest: +ELLIPSIS
+    X...2015...X

src/zc/buildout/buildout.py

@@ -434,6 +434,12 @@ class Buildout(DictMixin):
     def bootstrap(self, args):
         __doing__ = 'Bootstrapping.'
 
+        if os.path.exists(self['buildout']['develop-eggs-directory']):
+            if os.path.isdir(self['buildout']['develop-eggs-directory']):
+                rmtree(self['buildout']['develop-eggs-directory'])
+                self._logger.debug(
+                    "Removed existing develop-eggs directory")
+
         self._setup_directories()
 
         # Now copy buildout and setuptools eggs, and record destination eggs:

src/zc/buildout/buildout.txt

@@ -2860,6 +2860,7 @@ or paths to use:
     >>> remove('setup.cfg')
     >>> print_(system(buildout + ' -csetup.cfg init demo other ./src'), end='')
     Creating '/sample-bootstrapped/setup.cfg'.
+    Creating directory '/sample-bootstrapped/develop-eggs'.
     Getting distribution for 'zc.recipe.egg>=2.0.0a3'.
     Got zc.recipe.egg
     Installing py.
@@ -2918,6 +2919,7 @@ for us:
     >>> remove('setup.cfg')
     >>> print_(system(buildout + ' -csetup.cfg init demo other ./src'), end='')
     Creating '/sample-bootstrapped/setup.cfg'.
+    Creating directory '/sample-bootstrapped/develop-eggs'.
     Installing py.
     Generated script '/sample-bootstrapped/bin/demo'.
     Generated script '/sample-bootstrapped/bin/distutilsscript'.

src/zc/buildout/easy_install.py

@@ -72,7 +72,7 @@ if has_distribute and not has_setuptools:
     sys.exit("zc.buildout 2 needs setuptools, not distribute."
              "  Are you using an outdated bootstrap.py?  Make sure"
              " you have the latest version downloaded from"
-             " http://downloads.buildout.org/2/bootstrap.py")
+             " https://bootstrap.pypa.io/bootstrap-buildout.py")
 
 setuptools_loc = pkg_resources.working_set.find(
     pkg_resources.Requirement.parse('setuptools')