- If a distribution defines namespace packages but fails to declare

  setuptools as one of its dependencies, we now treat setuptools as an 
  implicit dependency.  We generate a warning if the distribution
  is a develop egg.

- Remove old develop egg links. This requires storing the old link
  paths in .installed.cfg.

CHANGES.txt

@@ -33,6 +33,17 @@ Feature Changes
   update is called.  For backward compatibility, recipes that don't
   define update methiods are still supported.
 
+- If a distribution defines namespace packages but fails to declare 
+  setuptools as one of its dependencies, we now treat setuptools as an 
+  implicit dependency.  We generate a warning if the distribution
+  is a develop egg.
+
+Bugs Fixed
+----------
+
+- Egg links weren't removed when corresponding entries were removed
+  from develop sections.
+
 1.0.0b9 (2006-10-02)
 ====================
 

src/zc/buildout/buildout.py

@@ -251,12 +251,18 @@ class Buildout(dict):
         # Check for updates. This could cause the process to be rstarted
         self._maybe_upgrade()
 
-        # Build develop eggs
-        self._develop()
-
         # load installed data
         installed_part_options = self._read_installed_part_options()
 
+        # Remove old develop eggs
+        self._uninstall(
+            installed_part_options['buildout'].get(
+                'installed_develop_eggs', '')
+            )
+
+        # Build develop eggs
+        installed_develop_eggs = self._develop()
+
         # get configured and installed part lists
         conf_parts = self['buildout']['parts']
         conf_parts = conf_parts and conf_parts.split() or []
@@ -380,6 +386,9 @@ class Buildout(dict):
                 +
                 [p for p in installed_parts if p not in conf_parts] 
             )
+            installed_part_options['buildout']['installed_develop_eggs'
+                                               ] = installed_develop_eggs
+            
             self._save_installed_options(installed_part_options)
 
     def _setup_directories(self):
@@ -395,9 +404,15 @@ class Buildout(dict):
         """Install sources by running setup.py develop on them
         """
         develop = self['buildout'].get('develop')
-        if develop:
-            env = dict(os.environ, PYTHONPATH=pkg_resources_loc)
-            here = os.getcwd()
+        if not develop:
+            return ''
+
+        dest = self['buildout']['develop-eggs-directory']
+        old_files = os.listdir(dest)
+
+        env = dict(os.environ, PYTHONPATH=pkg_resources_loc)
+        here = os.getcwd()
+        try:
             try:
                 for setup in develop.split():
                     setup = self._buildout_path(setup)
@@ -413,9 +428,7 @@ class Buildout(dict):
                         '-f', zc.buildout.easy_install._safe_arg(
                             ' '.join(self._links)
                                   ),
-                        '-d', zc.buildout.easy_install._safe_arg(
-                                  self['buildout']['develop-eggs-directory']
-                                  ),
+                        '-d', zc.buildout.easy_install._safe_arg(dest),
                         ]
 
                     if self._log_level <= logging.DEBUG:
@@ -425,11 +438,39 @@ class Buildout(dict):
                             args[1] == '-v'
                         self._logger.debug("in: %s\n%r",
                                            os.path.dirname(setup), args)
-                        
+
                     args.append(env)
-                    os.spawnle(os.P_WAIT, sys.executable, sys.executable, *args)
-            finally:
-                os.chdir(here)
+                    assert os.spawnle(
+                        os.P_WAIT, sys.executable, sys.executable, *args
+                        ) == 0
+            except:
+                # if we had an error, we need to roll back changes, by
+                # removing any files we created.
+                self._sanity_check_develop_eggs_files(dest, old_files)
+                self._uninstall('\n'.join(
+                    [os.path.join(dest, f)
+                     for f in os.listdir(dest)
+                     if f not in old_files
+                     ]))
+            else:
+                self._sanity_check_develop_eggs_files(dest, old_files)
+                return '\n'.join([os.path.join(dest, f)
+                                  for f in os.listdir(dest)
+                                  if f not in old_files
+                                  ])
+
+        finally:
+            os.chdir(here)
+
+
+    def _sanity_check_develop_eggs_files(self, dest, old_files):
+        for f in os.listdir(dest):
+            if f in old_files:
+                continue
+            if not (os.path.isfile(os.path.join(dest, f))
+                    and f.endswith('.egg-link')):
+                self._logger.warning(
+                    "Unexpected entry, %s, in develop-eggs directory", f)
 
     def _load_recipes(self, parts):
         recipes = {}

src/zc/buildout/buildout.txt

@@ -307,6 +307,7 @@ about the part we installed:
 
     >>> cat(sample_buildout, '.installed.cfg')
     [buildout]
+    installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
     parts = data-dir
     <BLANKLINE>
     [data-dir]
@@ -904,6 +905,7 @@ the buildout in the usual way:
 
     >>> cat(sample_buildout, '.installed.cfg')
     [buildout]
+    installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
     parts = debug d1 d2 d3
     <BLANKLINE>
     [debug]
@@ -988,6 +990,7 @@ The .installed.cfg is only updated for the recipes that ran:
 
     >>> cat(sample_buildout, '.installed.cfg')
     [buildout]
+    installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
     parts = debug d2 d3 d4 d1
     <BLANKLINE>
     [debug]

src/zc/buildout/easy_install.py

@@ -294,7 +294,7 @@ def _get_dist(requirement, env, ws,
             link = link.strip()
             if link not in links:
                 links.append(link)
-
+                
     return dist
     
 def install(specs, dest,
@@ -325,9 +325,28 @@ def install(specs, dest,
         ws = working_set
 
     for requirement in requirements:
-        ws.add(_get_dist(requirement, env, ws,
+        dist = _get_dist(requirement, env, ws,
                          dest, links, index, executable, always_unzip)
-               )
+        ws.add(dist)
+        if dist.has_metadata('namespace_packages.txt'):
+            for r in dist.requires():
+                if r.project_name == 'setuptools':
+                    break
+            else:
+                # We have a namespace package but no requirement for setuptools
+                if dist.precedence == pkg_resources.DEVELOP_DIST:
+                    logger.warn(
+                        "Develop distribution for %s\n"
+                        "uses namespace packages but the distribution "
+                        "does not require setuptools.",
+                        dist)
+                requirement = pkg_resources.Requirement.parse('setuptools')
+                if ws.find(requirement) is None:
+                    dist = _get_dist(requirement, env, ws,
+                                     dest, links, index, executable,
+                                     False)
+                    ws.add(dist)
+                    
 
     # OK, we have the requested distributions and they're in the working
     # set, but they may have unmet requirements.  We'll simply keep

src/zc/buildout/tests.py

@@ -382,7 +382,179 @@ bootstrapping.
     buildout: Creating directory /sample-bootstrap/develop-eggs
     """
 
+def removing_eggs_from_develop_section_causes_egg_link_to_be_removed():
+    '''
+    >>> cd(sample_buildout)
 
+Create a develop egg:
+
+    >>> mkdir('foo')
+    >>> write('foo', 'setup.py',
+    ... """
+    ... from setuptools import setup
+    ... setup(name='foox')
+    ... """)
+    >>> write('buildout.cfg',
+    ... """
+    ... [buildout]
+    ... develop = foo
+    ... parts =
+    ... """)
+
+    >>> print system(join('bin', 'buildout')),
+    buildout: Develop: /sample-buildout/foo/setup.py
+
+    >>> ls('develop-eggs')
+    -  foox.egg-link
+
+Create another:
+
+    >>> mkdir('bar')
+    >>> write('bar', 'setup.py',
+    ... """
+    ... from setuptools import setup
+    ... setup(name='fooy')
+    ... """)
+    >>> write('buildout.cfg',
+    ... """
+    ... [buildout]
+    ... develop = foo bar
+    ... parts =
+    ... """)
+
+    >>> print system(join('bin', 'buildout')),
+    buildout: Develop: /sample-buildout/foo/setup.py
+    buildout: Develop: /sample-buildout/bar/setup.py
+
+    >>> ls('develop-eggs')
+    -  foox.egg-link
+    -  fooy.egg-link
+
+Remove one:
+
+    >>> write('buildout.cfg',
+    ... """
+    ... [buildout]
+    ... develop = bar
+    ... parts =
+    ... """)
+    >>> print system(join('bin', 'buildout')),
+    buildout: Develop: /sample-buildout/bar/setup.py
+
+It is gone
+
+    >>> ls('develop-eggs')
+    -  fooy.egg-link
+
+Remove the other:
+
+    >>> write('buildout.cfg',
+    ... """
+    ... [buildout]
+    ... parts =
+    ... """)
+    >>> print system(join('bin', 'buildout')),
+
+All gone
+
+    >>> ls('develop-eggs')
+    '''
+
+
+def add_setuptools_to_dependencies_when_namespace_packages():
+    '''    
+Often, a package depends on setuptools soley by virtue of using
+namespace packages. In this situation, package authors often forget to
+declare setuptools as a dependency. This is a mistake, but,
+unfortunately, a common one that we need to work around.  If an egg
+uses namespace packages and does not include setuptools as a depenency,
+we willll still include setuptools in the working set.  If we see this for
+a devlop egg, we will also generate a warning.
+
+    >>> cd(sample_buildout)
+
+    >>> mkdir('foo')
+    >>> mkdir('foo', 'src')
+    >>> mkdir('foo', 'src', 'stuff')
+    >>> write('foo', 'src', 'stuff', '__init__.py',
+    ... """__import__('pkg_resources').declare_namespace(__name__)
+    ... """)
+    >>> mkdir('foo', 'src', 'stuff', 'foox')
+    >>> write('foo', 'src', 'stuff', 'foox', '__init__.py', '')
+    >>> write('foo', 'setup.py',
+    ... """
+    ... from setuptools import setup
+    ... setup(name='foox',
+    ...       namespace_packages = ['stuff'],
+    ...       package_dir = {'': 'src'},
+    ...       packages = ['stuff', 'stuff.foox'],
+    ...       )
+    ... """)
+    >>> write('foo', 'README.txt', '')
+    
+    >>> write('buildout.cfg',
+    ... """
+    ... [buildout]
+    ... develop = foo
+    ... parts = 
+    ... """)
+
+    >>> print system(join('bin', 'buildout')),
+    buildout: Develop: /sample-buildout/foo/setup.py
+
+Now, if we generate a working set using the egg link, we will get a warning
+and we will get setuptools included in the working set.
+
+    >>> import logging, zope.testing.loggingsupport
+    >>> handler = zope.testing.loggingsupport.InstalledHandler(
+    ...        'zc.buildout', level=logging.WARNING)
+    >>> logging.getLogger('zc').propagate = False
+    
+    >>> [dist.project_name
+    ...  for dist in zc.buildout.easy_install.working_set(
+    ...    ['foox'], sys.executable,
+    ...    [join(sample_buildout, 'eggs'),
+    ...     join(sample_buildout, 'develop-eggs'),
+    ...     ])]
+    ['foox', 'setuptools']
+
+    >>> print handler
+    zc.buildout.easy_install WARNING
+      Develop distribution for foox 0.0.0
+    uses namespace packages but the distribution does not require setuptools.
+
+    >>> handler.clear()
+
+On the other hand, if we have a regular egg, rather than a develop egg:
+
+    >>> os.remove(join('develop-eggs', 'foox.egg-link'))
+
+    >>> _ = system(join('bin', 'buildout') + ' setup foo bdist_egg -d'
+    ...            + join(sample_buildout, 'eggs'))
+
+    >>> ls('develop-eggs')
+    
+    >>> ls('eggs') # doctest: +ELLIPSIS
+    -  foox-0.0.0-py2.4.egg
+    ...
+    
+We do not get a warning, but we do get setuptools included in the working set:
+
+    >>> [dist.project_name
+    ...  for dist in zc.buildout.easy_install.working_set(
+    ...    ['foox'], sys.executable,
+    ...    [join(sample_buildout, 'eggs'),
+    ...     join(sample_buildout, 'develop-eggs'),
+    ...     ])]
+    ['foox', 'setuptools']
+
+
+    >>> print handler,
+
+    >>> logging.getLogger('zc').propagate = True
+    >>> handler.uninstall()
+    '''
+    
 def create_sample_eggs(test, executable=sys.executable):
     write = test.globs['write']
     dest = test.globs['sample_eggs']
@@ -603,7 +775,6 @@ def test_suite():
         doctest.DocTestSuite(
             setUp=zc.buildout.testing.buildoutSetUp,
             tearDown=zc.buildout.testing.buildoutTearDown,
-
             checker=renormalizing.RENormalizing([
                zc.buildout.testing.normalize_path,
                zc.buildout.testing.normalize_script,

zc.recipe.egg_/src/zc/recipe/egg/api.txt

@@ -96,6 +96,7 @@ computed by the egg recipe by looking at .installed.cfg:
 
     >>> cat(sample_buildout, '.installed.cfg')
     [buildout]
+    installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link
     parts = sample-part
     <BLANKLINE>
     [sample-part]
@@ -104,9 +105,9 @@ computed by the egg recipe by looking at .installed.cfg:
             zc.recipe.egg-cAsnudgkduAa/Fd+WJIM6Q==
             setuptools-0.6-py2.4.egg
             zc.buildout-+rYeCcmFuD1K/aB77XTj5A==
-    _b = /tmp/tmpb7kP9bsample-buildout/bin
-    _d = /tmp/tmpb7kP9bsample-buildout/develop-eggs
-    _e = /tmp/tmpb7kP9bsample-buildout/eggs
+    _b = /sample-buildout/bin
+    _d = /sample-buildout/develop-eggs
+    _e = /sample-buildout/eggs
     eggs = demo<0.3
     executable = /usr/local/bin/python2.3
     extras = other

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -56,8 +56,7 @@ def test_suite():
             'api.txt',
             setUp=setUp, tearDown=zc.buildout.testing.buildoutTearDown,
             checker=renormalizing.RENormalizing([
-               (re.compile('_b = \S+sample-buildout.bin'),
-                '_b = sample-buildout/bin'),
+               zc.buildout.testing.normalize_path,
                (re.compile('__buildout_signature__ = '
                            'sample-\S+\s+'
                            'zc.recipe.egg-\S+\s+'
@@ -65,10 +64,6 @@ def test_suite():
                            'zc.buildout-\S+\s*'
                            ),
                 '__buildout_signature__ = sample- zc.recipe.egg-'),
-               (re.compile('_d = \S+sample-buildout.develop-eggs'),
-                '_d = sample-buildout/develop-eggs'),
-               (re.compile('_e = \S+sample-buildout.eggs'),
-                '_e = sample-buildout/eggs'),
                (re.compile('executable = \S+python\S*'),
                 'executable = python'),
                (re.compile('index = \S+python\S+'),

zc.recipe.testrunner/src/zc/recipe/testrunner/README.txt

@@ -205,6 +205,7 @@ extra-paths option to specify them:
     sys.path[0:0] = [
       '/sample-buildout/demo',
       '/sample-buildout/eggs/zope.testing-3.0-py2.3.egg',
+      '/sample-buildout/eggs/setuptools-0.6-py1.3.egg',
       '/usr/local/zope/lib/python',
       ]
     <BLANKLINE>
@@ -248,6 +249,7 @@ using the -v option.
     sys.path[0:0] = [
       '/sample-buildout/demo',
       '/sample-buildout/eggs/zope.testing-3.0-py2.4.egg',
+      '/sample-buildout/eggs/setuptools-0.6-py1.3.egg',
       '/usr/local/zope/lib/python',
       ]
     <BLANKLINE>

zc.recipe.testrunner/src/zc/recipe/testrunner/tests.py

@@ -45,6 +45,7 @@ def test_suite():
                (re.compile('#!\S+python\S*'), '#!python'),
                (re.compile('\d[.]\d+ seconds'), '0.001 seconds'),
                (re.compile('zope.testing-[^-]+-'), 'zope.testing-X-'),
+               (re.compile('setuptools-[^-]+-'), 'setuptools-X-'),
                ])
             ),