Refactored the way recipes are run and how they should be written.

If a recipe uses any data from other sections, the recipe needs to
update it's data when the recipe is constructed.

Need more discussion of this in the docs.


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@68634 62d5b8a3-27da-0310-9561-8e5933582275

eggrecipe/src/zc/recipe/egg/egg.py

@@ -16,6 +16,7 @@
 $Id$
 """
 
+import os
 import zc.buildout.egglinker
 import zc.buildout.easy_install
 
@@ -25,26 +26,27 @@ class Egg:
         self.buildout = buildout
         self.name = name
         self.options = options
-
-    def install(self):
-        distribution = self.options.get('distribution', self.name)
-        links = self.options.get(
-            'find-links',
-            self.buildout['buildout'].get('find-links'),
-            )
+        links = options.get('find-links',
+                            buildout['buildout'].get('find-links'))
         if links:
-            links = links.split()
+            buildout_directory = buildout['buildout']['directory']
+            links = [os.path.join(buildout_directory, link)
+                     for link in links.split()]
+            options['find-links'] = '\n'.join(links)
         else:
             links = ()
+        self.links = links
 
-        buildout = self.buildout
+        options['_b'] = buildout['buildout']['bin-directory']
+        options['_e'] = buildout['buildout']['eggs-directory']
+
+    def install(self):
+        options = self.options
+        distribution = options.get('distribution', self.name)
         zc.buildout.easy_install.install(
-            distribution,
-            buildout.eggs,
-            [buildout.buildout_path(link) for link in links],
-            )
+            distribution, options['_e'], self.links)
 
-        scripts = self.options.get('scripts')
+        scripts = options.get('scripts')
         if scripts or scripts is None:
             if scripts is not None:
                 scripts = scripts.split()
@@ -53,6 +55,6 @@ class Egg:
                     for s in scripts
                     ])
             return zc.buildout.egglinker.scripts(
-                [distribution], buildout.bin, [buildout.eggs],
-                scripts=scripts)
+                [distribution],
+                options['_b'], [options['_e']], scripts=scripts)
             

src/zc/buildout/buildout.py

@@ -46,6 +46,12 @@ class Options(dict):
         except KeyError:
             raise MissingOption("Missing option", self.section, option)
 
+    # XXX need test
+    def __setitem__(self, option, value):
+        if not isinstance(value, str):
+            raise TypeError('Option values must be strings', value)
+        super(Options, self).__setitem__(option, value)
+
     def copy(self):
         return Options(self.buildout, self.section, self)
 
@@ -63,6 +69,8 @@ class Buildout(dict):
                               'bin-directory': 'bin',
                               'parts-directory': 'parts',
                               'installed': '.installed.cfg',
+                              'python': 'buildout',
+                              'executable': sys.executable,
                               },
                        )
 
@@ -106,8 +114,8 @@ class Buildout(dict):
 
         self._buildout_dir = options['directory']
         for name in ('bin', 'parts', 'eggs'):
-            d = self.buildout_path(options[name+'-directory'])
-            setattr(self, name, d)
+            d = self._buildout_path(options[name+'-directory'])
+            options[name+'-directory'] = d
             if not os.path.exists(d):
                 os.mkdir(d)
 
@@ -148,45 +156,75 @@ class Buildout(dict):
 
         return ''.join([''.join(v) for v in zip(value[::2], subs)])
 
-    def buildout_path(self, *names):
+    def _buildout_path(self, *names):
         return os.path.join(self._buildout_dir, *names)
 
     def install(self, install_parts):
         self._develop()
-        new_part_options = self._gather_part_info()
+
+        # load installed data
         installed_part_options = self._read_installed_part_options()
-        old_parts = installed_part_options['buildout']['parts'].split()
-        old_parts.reverse()
-
-        new_old_parts = []
-        for part in old_parts:
-            if install_parts and (part not in install_parts):
-                # We were asked to install specific parts and this
-                # wasn't one of them.  Leave it alone.
-                new_old_parts.append(part)
-                continue
-                
-            installed_options = installed_part_options[part].copy()
-            installed = installed_options.pop('__buildout_installed__')
-            if installed_options != new_part_options.get(part):
-                self._uninstall(installed)
-                del installed_part_options[part]
-            else:
-                new_old_parts.append(part)
-        new_old_parts.reverse()
-
-        new_parts = []
+
+        # get configured and installed part lists
+        conf_parts = self['buildout']['parts']
+        conf_parts = conf_parts and conf_parts.split() or []
+        installed_parts = installed_part_options['buildout']['parts']
+        installed_parts = installed_parts and installed_parts.split() or []
+
+
+        # If install_parts is given, then they must be listed in parts
+        # and we don't uninstall anything. Otherwise, we install
+        # the configured parts and uninstall anything else.
+        if install_parts:
+            extra = [p for p in install_parts if p not in conf_parts]
+            if extra:
+                error('Invalid install parts:', *extra)
+            uninstall_missing = False
+        else:
+            install_parts = conf_parts
+            uninstall_missing = True
+
+        # load recipes
+        recipes = self._load_recipes(install_parts)
+
+        # compute new part recipe signatures
+        self._compute_part_signatures(install_parts)
+
         try:
-            for part in new_part_options['buildout']['parts'].split():
-                if (not install_parts) or (part in install_parts):
-                    installed = self._install(part)
-                    new_part_options[part]['__buildout_installed__'] = installed
-                    installed_part_options[part] = new_part_options[part]
-                new_parts.append(part)
-                new_old_parts = [p for p in new_old_parts if p != part]
+            # uninstall parts that are no-longer used or who's configs
+            # have changed
+            for part in reversed(installed_parts):
+                if part in install_parts:
+                    old_options = installed_part_options[part].copy()
+                    old_options.pop('__buildout_installed__')
+                    if old_options == self.get(part):
+                        continue
+                elif not uninstall_missing:
+                    continue
+
+                # ununstall part
+                self._uninstall(
+                    installed_part_options[part]['__buildout_installed__'])
+                installed_parts = [p for p in installed_parts if p != part]
+
+            # install new parts
+            for part in install_parts:
+                installed_part_options[part] = self[part].copy()
+                del self[part]['__buildout_signature__']
+                installed_files = recipes[part].install() or ()
+                if isinstance(installed_files, str):
+                    installed_files = [installed_files]
+                installed_part_options[part]['__buildout_installed__'] = (
+                    '\n'.join(installed_files)
+                    )
+                if part not in installed_parts:
+                    installed_parts.append(part)
         finally:
-            new_parts.extend(new_old_parts)
-            installed_part_options['buildout']['parts'] = ' '.join(new_parts)
+            installed_part_options['buildout']['parts'] = ' '.join(
+                [p for p in conf_parts if p in installed_parts]
+                +
+                [p for p in installed_parts if p not in conf_parts] 
+            )
             self._save_installed_options(installed_part_options)
 
     def _develop(self):
@@ -197,7 +235,7 @@ class Buildout(dict):
             here = os.getcwd()
             try:
                 for setup in develop.split():
-                    setup = self.buildout_path(setup)
+                    setup = self._buildout_path(setup)
                     if os.path.isdir(setup):
                         setup = os.path.join(setup, 'setup.py')
 
@@ -206,46 +244,53 @@ class Buildout(dict):
                         os.P_WAIT, sys.executable, sys.executable,
                         setup, '-q', 'develop', '-m', '-x',
                         '-f', ' '.join(self._links),
-                        '-d', self.eggs,
+                        '-d', self['buildout']['eggs-directory'],
                         {'PYTHONPATH':
                          os.path.dirname(pkg_resources.__file__)},
                         )
             finally:
                 os.chdir(os.path.dirname(here))
 
-    def _gather_part_info(self):
-        """Get current part info, including part options and recipe info
-        """
-        parts = self['buildout']['parts']
-        part_info = {'buildout': {'parts': parts}}
+    def _load_recipes(self, parts):
+        recipes = {}
         recipes_requirements = []
-        pkg_resources.working_set.add_entry(self.eggs)
+        pkg_resources.working_set.add_entry(self['buildout']['eggs-directory'])
 
-        parts = parts and parts.split() or []
+        # Install the recipe distros
         for part in parts:
             options = self.get(part)
             if options is None:
                 options = self[part] = {}
-            options = options.copy()
             recipe, entry = self._recipe(part, options)
             zc.buildout.easy_install.install(
-                recipe, self.eggs, self._links)
+                recipe, self['buildout']['eggs-directory'], self._links)
             recipes_requirements.append(recipe)
-            part_info[part] = options
 
-        # Load up the recipe distros
+        # Add the distros to the working set
         pkg_resources.require(recipes_requirements)
 
-        base = self.eggs + os.path.sep
+        # instantiate the recipes
         for part in parts:
-            options = part_info[part]
+            options = self[part]
+            recipe, entry = self._recipe(part, options)
+            recipe_class = pkg_resources.load_entry_point(
+                recipe, 'zc.buildout', entry)
+            recipes[part] = recipe_class(self, part, options)
+        
+        return recipes
+
+    def _compute_part_signatures(self, parts):
+        # Compute recipe signature and add to options
+        base = self['buildout']['eggs-directory'] + os.path.sep
+        for part in parts:
+            options = self.get(part)
+            if options is None:
+                options = self[part] = {}
             recipe, entry = self._recipe(part, options)
             req = pkg_resources.Requirement.parse(recipe)
             sig = _dists_sig(pkg_resources.working_set.resolve([req]), base)
             options['__buildout_signature__'] = ' '.join(sig)
 
-        return part_info
-
     def _recipe(self, part, options):
         recipe = options.get('recipe', part)
         if ':' in recipe:
@@ -260,17 +305,18 @@ class Buildout(dict):
         if os.path.isfile(old):
             parser = ConfigParser.SafeConfigParser()
             parser.read(old)
-            return dict([(section, dict(parser.items(section)))
-                         for section in parser.sections()])
+            return dict([
+                (section, Options(self, section, parser.items(section)))
+                for section in parser.sections()])
         else:
-            return {'buildout': {'parts': ''}}
+            return {'buildout': Options(self, 'buildout', {'parts': ''})}
 
     def _installed_path(self):        
-        return self.buildout_path(self['buildout']['installed'])
+        return self._buildout_path(self['buildout']['installed'])
 
     def _uninstall(self, installed):
         for f in installed.split():
-            f = self.buildout_path(f)
+            f = self._buildout_path(f)
             if os.path.isdir(f):
                 shutil.rmtree(f)
             elif os.path.isfile(f):
@@ -286,7 +332,7 @@ class Buildout(dict):
             installed = []
         elif isinstance(installed, basestring):
             installed = [installed]
-        base = self.buildout_path('')
+        base = self._buildout_path('')
         installed = [d.startswith(base) and d[len(base):] or d
                      for d in installed]
         return ' '.join(installed)
@@ -412,3 +458,9 @@ def main(args=None):
         command = 'install'
 
     getattr(buildout, command)(args)
+
+if sys.version_info[:2] < (2, 4):
+    def reversed(iterable):
+        result = list(iterable);
+        result.reverse()
+        return result

src/zc/buildout/buildout.txt

@@ -92,9 +92,13 @@ and then we'll create a source file for our mkdir recipe:
     ...         self.buildout = buildout
     ...         self.name = name
     ...         self.options = options
+    ...         options['path'] = os.path.join(
+    ...                               buildout['buildout']['directory'],
+    ...                               options['path'],
+    ...                               )
     ...
     ...     def install(self):
-    ...         path = self.buildout.buildout_path(self.options['path'])
+    ...         path = self.options['path']
     ...         if not os.path.isdir(path):
     ...             print 'Creating directory', os.path.basename(path)
     ...             os.mkdir(path)
@@ -104,14 +108,21 @@ and then we'll create a source file for our mkdir recipe:
 The recipe defines a constructor that takes a buildout object, a part
 name, and an options dictionary. It saves them in instance attributes.
 
+If the path is relative, we'll interpret it as relative to the
+buildout directory.  The buildout object passed in is a mapping from
+section name to a mapping of options for that section. The buildout
+directory is available as the directory option of the buildout
+section.  We normalize the path and save it back into the options
+directory.  
+
+**IMPORTANT**: Any time we use data from another section, it is important
+to reflect that data in the recipe options, as this data is used to
+decide if a part configuration has changed and a part needs to be
+reinstalled.
+
 The install method is responsible for creating the part.  In this
 case, we need the path of the directory to create.  We'll use a
-buildout option from our options dictionary.  If the path is relative,
-we'll interpret it relative to the buildout directory.  The buildout
-buildout_path method gives us a path relative to the buildout.  It
-uses os.path.join, so if we pass it an absolute path, we'll get the
-absolute path back. (If no arguments are passed to base_path, then the
-buildout directory is returned.)
+path option from our options dictionary.
 
 We made the method chatty so that we can observe what it's doing.
 
@@ -134,14 +145,13 @@ installed as an egg.  We need to define a setup script for this:
     ...     )
     ... """)
 
-Here we've defined a package containing just our module.  We've also
-defined an entry point.  Entry points provide a way for an egg to
-define the services it provides.  Here we've said that we define a
-zc.buildout entry point named default.  Recipe classes must be exposed
-as entry points in the zc.buildout group.  we give entry points names
-within the group.  The name "default" is somewhat special because it
-allows a recipe to be referenced using a package name without naming
-an entry point.
+Here we've defined a package with an entry_point. Entry points provide
+a way for an egg to define the services it provides.  Here we've said
+that we define a zc.buildout entry point named default.  Recipe
+classes must be exposed as entry points in the zc.buildout group.  we
+give entry points names within the group.  The name "default" is
+somewhat special because it allows a recipe to be referenced using a
+package name without naming an entry point.
 
 We also need a README.txt for our recipes to avoid a warning:
 
@@ -219,12 +229,14 @@ installed:
     parts = data_dir
     <BLANKLINE>
     [data_dir]
-    __buildout_installed__ = mystuff
-    __buildout_signature__ = recipes-O3ypTgKOkHMqMwKvMfvHnA==
-    path = mystuff
+    __buildout_installed__ = /tmp/sample-buildout/mystuff
+    __buildout_signature__ = recipes-c7vHV6ekIDUPy/7fjAaYjg==
+    path = /tmp/sample-buildout/mystuff
     recipe = recipes:mkdir
 
 Note that the directory we installed is included in .installed.cfg.
+In addition, the path option includes the actual destination
+directory. 
 
 If we change the name of the directory in the configuration file,
 we'll see that the directory gets removed and recreated:
@@ -639,27 +651,27 @@ configuration and run the buildout in the usual way:
     <BLANKLINE>
     [debug]
     __buildout_installed__ = 
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
     op1 = 1
     op7 = 7
     recipe = recipes:debug
     <BLANKLINE>
     [d1]
-    __buildout_installed__ = d1
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
-    path = d1
+    __buildout_installed__ = /tmp/sample-buildout/d1
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    path = /tmp/sample-buildout/d1
     recipe = recipes:mkdir
     <BLANKLINE>
     [d2]
-    __buildout_installed__ = d2
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
-    path = d2
+    __buildout_installed__ = /tmp/sample-buildout/d2
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    path = /tmp/sample-buildout/d2
     recipe = recipes:mkdir
     <BLANKLINE>
     [d3]
-    __buildout_installed__ = d3
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
-    path = d3
+    __buildout_installed__ = /tmp/sample-buildout/d3
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    path = /tmp/sample-buildout/d3
     recipe = recipes:mkdir
 
 Now we'll update our configuration file:
@@ -721,33 +733,33 @@ The .installed.cfg is only updated for the recipes that ran:
     <BLANKLINE>
     [debug]
     __buildout_installed__ = 
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
     op1 = 1
     op7 = 7
     recipe = recipes:debug
     <BLANKLINE>
     [d2]
-    __buildout_installed__ = d2
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
-    path = d2
+    __buildout_installed__ = /tmp/sample-buildout/d2
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    path = /tmp/sample-buildout/d2
     recipe = recipes:mkdir
     <BLANKLINE>
     [d3]
-    __buildout_installed__ = data3
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
-    path = data3
+    __buildout_installed__ = /tmp/sample-buildout/data3
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    path = /tmp/sample-buildout/data3
     recipe = recipes:mkdir
     <BLANKLINE>
     [d4]
-    __buildout_installed__ = data4
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
-    path = data4
+    __buildout_installed__ = /tmp/sample-buildout/data4
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    path = /tmp/sample-buildout/data4
     recipe = recipes:mkdir
     <BLANKLINE>
     [d1]
-    __buildout_installed__ = d1
-    __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
-    path = d1
+    __buildout_installed__ = /tmp/sample-buildout/d1
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    path = /tmp/sample-buildout/d1
     recipe = recipes:mkdir
 
 Note that the installed data for debug, d1, and d2 haven't changed,

src/zc/buildout/testing.py

@@ -53,7 +53,7 @@ def system(command, input=''):
     return o.read()
 
 def buildoutSetUp(test):
-    sample = tempfile.mkdtemp('buildout-tests')
+    sample = tempfile.mkdtemp('sample-buildout')
     for name in ('bin', 'eggs', 'parts'):
         os.mkdir(os.path.join(sample, name))
 
@@ -118,7 +118,7 @@ def runsetup(d):
         os.chdir(here)
 
 def create_sample_eggs(test):
-    sample = tempfile.mkdtemp('eggtest')
+    sample = tempfile.mkdtemp('sample-eggs')
     test.globs['_sample_eggs_container'] = sample
     test.globs['sample_eggs'] = os.path.join(sample, 'dist')
     write(sample, 'README.txt', '')

src/zc/buildout/tests.py

@@ -94,7 +94,9 @@ def test_suite():
             checker=renormalizing.RENormalizing([
                (re.compile('__buildout_signature__ = recipes-\S+'),
                 '__buildout_signature__ = recipes-SSSSSSSSSSS'),
-               ])
+               (re.compile('\S+sample-(\w+)%s(\S+)' % os.path.sep),
+                r'/sample-\1/\2'),
+                ])
             ),
         doctest.DocFileSuite(
             'egglinker.txt', 'easy_install.txt', 

testrunnerrecipe/src/zc/recipe/testrunner/__init__.py

@@ -25,6 +25,11 @@ class TestRunner:
         self.buildout = buildout
         self.name = name
         self.options = options
+        options['script'] = os.path.join(buildout['buildout']['bin-directory'],
+                                         options.get('script', self.name),
+                                         )
+        options['_e'] = buildout['buildout']['eggs-directory']
+
 
     def install(self):
         distributions = [
@@ -34,13 +39,12 @@ class TestRunner:
             ]
         path = zc.buildout.egglinker.path(
             distributions+['zope.testing'],
-            [self.buildout.eggs],
+            [self.options['_e']],
             )
         locations = [zc.buildout.egglinker.location(distribution,
-                                                    [self.buildout.eggs])
+                                                    [self.options['_e']])
                      for distribution in distributions]
-        script = self.options.get('script', self.name)
-        script = self.buildout.buildout_path('bin', script)
+        script = self.options['script']
         open(script, 'w').write(tests_template % dict(
             PYTHON=sys.executable,
             PATH="',\n  '".join(path),