Author: Thomas Lotze <tl@gocept.com>

Date:   Wed Feb 16 15:47:25 2011 +0000
made sure to download extended configuration files only once per buildout run (based on patch by Rafael Monnerat)

Cherry-picked from svn trunk

CHANGES.rst

@@ -25,6 +25,9 @@ New features:
 
 - Remove duplicate path from script's sys.path setup.
 
+- Make sure to download extended configuration files only once per buildout
+  run even if they are referenced multiple times (patch by Rafael Monnerat).
+
 Bugs fixed:
 
 - In the download module, fixed the handling of directories that are pointed

src/zc/buildout/buildout.py

@@ -195,12 +195,12 @@ class Buildout(UserDict.DictMixin):
                                        '.buildout', 'default.cfg')
             if os.path.exists(user_config):
                 _update(data, _open(os.path.dirname(user_config), user_config,
-                                    [], data['buildout'].copy(), override))
+                                    [], data['buildout'].copy(), override, set()))
 
         # load configuration files
         if config_file:
             _update(data, _open(os.path.dirname(config_file), config_file, [],
-                                data['buildout'].copy(), override))
+                                data['buildout'].copy(), override, set()))
 
         # apply command-line options
         _update(data, cloptions)
@@ -1286,17 +1286,26 @@ def _save_options(section, options, f):
     for option, value in items:
         _save_option(option, value, f)
 
-def _open(base, filename, seen, dl_options, override):
+def _convert_bool(name, value):
+    if value not in ('true', 'false'):
+        raise zc.buildout.UserError(
+            'Invalid value for %s option: %s' % (name, value))
+    else:
+        return value == 'true'
+
+def _open(base, filename, seen, dl_options, override, downloaded):
     """Open a configuration file and return the result as a dictionary,
 
     Recursively open other files based on buildout options found.
     """
     _update_section(dl_options, override)
     _dl_options = _unannotate_section(dl_options.copy())
-    is_temp = False
+    newest = _convert_bool('newest', _dl_options.get('newest', 'false'))
+    fallback = newest and not (filename in downloaded)
     download = zc.buildout.download.Download(
-        _dl_options, cache=_dl_options.get('extends-cache'), fallback=True,
-        hash_name=True)
+        _dl_options, cache=_dl_options.get('extends-cache'),
+        fallback=fallback, hash_name=True)
+    is_temp = False
     if _isurl(filename):
         path, is_temp = download(filename)
         fp = open(path)
@@ -1314,6 +1323,7 @@ def _open(base, filename, seen, dl_options, override):
         filename = os.path.join(base, filename)
         fp = open(filename)
         base = os.path.dirname(filename)
+    downloaded.add(filename)
 
     if filename in seen:
         if is_temp:
@@ -1348,9 +1358,11 @@ def _open(base, filename, seen, dl_options, override):
 
     if extends:
         extends = extends.split()
-        eresult = _open(base, extends.pop(0), seen, dl_options, override)
+        eresult = _open(base, extends.pop(0), seen, dl_options, override,
+                        downloaded)
         for fname in extends:
-            _update(eresult, _open(base, fname, seen, dl_options, override))
+            _update(eresult, _open(base, fname, seen, dl_options, override,
+                    downloaded))
         result = _update(eresult, result)
 
     if extended_by:
@@ -1358,9 +1370,9 @@ def _open(base, filename, seen, dl_options, override):
             "The extendedBy option is deprecated.  Stop using it."
             )
         for fname in extended_by.split():
-            result = _update(result,
-                             _open(base, fname, seen, dl_options, override))
-
+            result = _update(
+                result,
+                _open(base, fname, seen, dl_options, override, downloaded))
     seen.pop()
     return result
 

src/zc/buildout/extends-cache.txt

@@ -386,6 +386,105 @@ While:
 An internal error occured ...
 ValueError: install_from_cache set to true with no download cache
 
+>>> rmdir('home', '.buildout')
+
+Newest and non-newest behaviour for extends cache
+-------------------------------------------------
+
+While offline mode forbids network access completely, 'newest' mode determines
+whether to look for updated versions of a resource even if some version of it
+is already present locally. If we run buildout in newest mode
+(``newest = true``), the configuration files are updated with each run:
+
+>>> mkdir("cache")
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... """)
+>>> write('buildout.cfg', """\
+... [buildout]
+... extends-cache = cache
+... extends = %sbase.cfg
+... """ % server_url)
+>>> print system(buildout)
+>>> ls('cache')
+-  5aedc98d7e769290a29d654a591a3a45
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+
+A change to ``base.cfg`` is picked up on the next buildout run:
+
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... foo = bar
+... """)
+>>> print system(buildout + " -n")
+Unused options for buildout: 'foo'.
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+foo = bar
+
+In contrast, when not using ``newest`` mode (``newest = false``), the files
+already present in the extends cache will not be updated:
+
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... """)
+>>> print system(buildout + " -N")
+Unused options for buildout: 'foo'.
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+foo = bar
+
+Even when updating base configuration files with a buildout run, any given
+configuration file will be downloaded only once during that particular run. If
+some base configuration file is extended more than once, its cached copy is
+used:
+
+>>> write(server_data, 'baseA.cfg', """\
+... [buildout]
+... extends = %sbase.cfg
+... foo = bar
+... """ % server_url)
+>>> write(server_data, 'baseB.cfg', """\
+... [buildout]
+... extends-cache = cache
+... extends = %sbase.cfg
+... bar = foo
+... """ % server_url)
+>>> write('buildout.cfg', """\
+... [buildout]
+... extends-cache = cache
+... newest = true
+... extends = %sbaseA.cfg %sbaseB.cfg
+... """ % (server_url, server_url))
+>>> print system(buildout + " -n")
+Unused options for buildout: 'bar' 'foo'.
+
+(XXX We patch download utility's API to produce readable output for the test;
+a better solution would utilise the logging already done by the utility.)
+
+>>> import zc.buildout
+>>> old_download = zc.buildout.download.Download.download
+>>> def wrapper_download(self, url, md5sum=None, path=None):
+...   print "The URL %s was downloaded." % url
+...   return old_download(url, md5sum, path)
+>>> zc.buildout.download.Download.download = wrapper_download
+
+>>> zc.buildout.buildout.main([])
+The URL http://localhost/baseA.cfg was downloaded.
+The URL http://localhost/base.cfg was downloaded.
+The URL http://localhost/baseB.cfg was downloaded.
+Not upgrading because not running a local buildout command.
+Unused options for buildout: 'bar' 'foo'.
+
+>>> zc.buildout.download.Download.download = old_download
+
 
 Clean up
 --------