[feat] Reimplement the extends algorithm

The new algorithm avoids fetching the same extended file more than once
and correctly handles overriding values and += and -=:

The new algorithm starts as if there was a buildout file containing

```
[buildout]
extends =
  user/defaults.cfg # if it exists
  buildout.cfg # if it exists
  command_line_extends.cfg # if passed on the command line
```

The files are then fetched in depth-first-search postorder and fetching
child nodes in the order given by the extends directive, ignoring files
that have already been fetched.

The buildout dicts are then collected in order, and this linearisation
is then merged at the end, overriding the first configs collected with
the later ones. The first dict in the linearisation is not from a file,
but the dict of buildout's (hardcoded) defaults. This is equivalent to
acting as though every file that does not extend anything extends these
defaults.

The first time a file must be downloaded from a url, the linearisation
is merged with the configs already collected, and the resulting options
are then used to determine the download options for this download, and
every subsequent download.

This is a break with buildout's current logic for download options.

By analogy with classes in Python, we are computing a linearisation of
the class hierarchy to determine the method resolution order (MRO).
This algorithm is not the same as Python's MRO since Python 2.3 (C3).

It could be good to switch to a C3 linearisation like Python.

src/zc/buildout/buildout.py

@@ -29,6 +29,11 @@ try:
 except ImportError:
     from UserDict import DictMixin
 
+try:
+    from urllib.parse import urljoin
+except ImportError:
+    from urlparse import urljoin
+
 import zc.buildout.configparser
 import copy
 import datetime
@@ -261,11 +266,14 @@ def _print_annotate(data, verbose, chosen_sections, basedir):
 
 
 def _unannotate_section(section):
-    return {key: entry.value for key, entry in section.items()}
-
+    for key in section:
+        section[key] = section[key].value
+    return section
 
 def _unannotate(data):
-    return {key: _unannotate_section(section) for key, section in data.items()}
+    for key in data:
+        _unannotate_section(data[key])
+    return data
 
 
 def _format_picked_versions(picked_versions, required_by):
@@ -355,54 +363,24 @@ class Buildout(DictMixin):
                 data['buildout']['directory'] = SectionKey(
                     os.path.dirname(config_file), 'COMPUTED_VALUE')
 
-        cloptions = dict(
-            (section, dict((option, SectionKey(value, 'COMMAND_LINE_VALUE'))
-                           for (_, option, value) in v))
-            for (section, v) in itertools.groupby(sorted(cloptions),
-                                                  lambda v: v[0])
-            )
-        override = copy.deepcopy(cloptions.get('buildout', {}))
+        result = {}
+        for section, option, value in cloptions:
+          result.setdefault(section, {})[option] = value
+
+        options = result.setdefault('buildout', {})
 
-        # load user defaults, which override defaults
+        extends = []
         user_config = _get_user_config()
         if use_user_defaults and os.path.exists(user_config):
-            download_options = data['buildout']
-            user_defaults, _ = _open(
-                os.path.dirname(user_config),
-                user_config, [], download_options,
-                override, set(), {}
-            )
-            for_download_options = _update(data, user_defaults)
-        else:
-            user_defaults = {}
-            for_download_options = copy.deepcopy(data)
-
-        # load configuration files
+            extends.append(user_config)
         if config_file:
-            download_options = for_download_options['buildout']
-            cfg_data, _ = _open(
-                os.path.dirname(config_file),
-                config_file, [], download_options,
-                override, set(), user_defaults
-            )
-            data = _update(data, cfg_data)
-
-        # extends from command-line
-        if 'buildout' in cloptions:
-            cl_extends = cloptions['buildout'].pop('extends', None)
-            if cl_extends:
-                for extends in cl_extends.value.split():
-                    download_options = for_download_options['buildout']
-                    cfg_data, _ = _open(
-                        os.path.dirname(extends),
-                        os.path.basename(extends),
-                        [], download_options,
-                        override, set(), user_defaults
-                    )
-                    data = _update(data, cfg_data)
+            extends.append(config_file)
+        clextends = options.get('extends')
+        if clextends:
+            extends.append(clextends)
+        options['extends'] = '\n'.join(extends)
 
-        # apply command-line options
-        data = _update(data, cloptions)
+        data = _extends(data, result, os.getcwd(), 'COMMAND_LINE_VALUE')
 
         # Set up versions section, if necessary
         if 'versions' not in data['buildout']:
@@ -1539,10 +1517,10 @@ class Options(DictMixin):
                     raise zc.buildout.UserError("No section named %r" % iname)
                 result.update(self._do_extend_raw(iname, raw, doing))
 
-            result = _annotate_section(result, "")
+            _annotate_section(result, "")
             data = _annotate_section(copy.deepcopy(data), "")
-            result = _update_section(result, data)
-            result = _unannotate_section(result)
+            _update_section(result, data)
+            _unannotate_section(result)
             result.pop('<', None)
             return result
         finally:
@@ -1831,101 +1809,107 @@ def _default_globals():
 
 variable_template_split = re.compile('([$]{[^}]*})').split
 
-def _open(
-        base, filename, seen, download_options,
-        override, downloaded, user_defaults
-        ):
-    """Open a configuration file and return the result as a dictionary,
 
-    Recursively open other files based on buildout options found.
-    """
-    download_options = _update_section(download_options, override)
-    raw_download_options = _unannotate_section(download_options)
-    newest = bool_option(raw_download_options, 'newest', 'false')
-    fallback = newest and not (filename in downloaded)
-    extends_cache = raw_download_options.get('extends-cache')
+class _extends(object):
+
+    def __new__(cls, defaults, *args):
+        self = super(_extends, cls).__new__(cls)
+        self.seen = set()
+        self.processing = []
+        self.extends = [defaults]
+        self._download_options = []
+        self.collect(*args)
+        return self.merge()
+
+    def merge(self):
+        result = {}
+        for d in self.extends:
+          _update(result, d)
+        return result
+
+    def __getattr__(self, attr):
+        if attr == 'download_options':
+            # Compute processed options
+            result_so_far = self.merge()
+            self.extends[:] = [result_so_far]
+            value = copy.deepcopy(result_so_far.get('buildout')) or {}
+            # Update with currently-being-processed options
+            for options in reversed(self._download_options):
+                _update_section(value, options)
+            value = _unannotate_section(value)
+            setattr(self, attr, value)
+            return value
+        return self.__getattribute__(attr)
+
+    def collect(self, result, base, filename):
+        options = result.get('buildout', {})
+        extends = options.pop('extends', '')
+
+        # Sanitize buildout options
+        if 'extended-by' in options:
+            raise zc.buildout.UserError(
+                'No-longer supported "extended-by" option found in %s.' %
+                filename)
+
+        _annotate(result, filename)
+
+        # Collect extends and unprocessed download options
+        self.processing.append(filename)
+        self._download_options.append(options)
+        for fextends in extends.split():
+            self.open(base, fextends)
+        self.extends.append(result)
+        del self.processing[-1], self._download_options[-1]
+
+    def open(self, base, filename):
+        # Determine file location  
+        if _isurl(filename):
+            download = True
+        elif _isurl(base):
+            download = True
+            filename = urljoin(base + '/', filename)
+        else:
+            download = False
+            filename = os.path.realpath(
+                os.path.join(base, os.path.expanduser(filename)))
+
+        # Detect repetitions and loops
+        if filename in self.seen:
+            if filename in self.processing:
+                raise zc.buildout.UserError("circular extends: %s" % filename)
+            return
+        self.seen.add(filename)
+
+        # Fetch file
+        is_temp = False
+        try:
+            if download:
+                download_options = self.download_options
+                extends_cache = download_options.get('extends-cache')
                 if extends_cache and variable_template_split(extends_cache)[1::2]:
                     raise ValueError(
                         "extends-cache '%s' may not contain ${section:variable} to expand."
                         % extends_cache
                     )
-    download = zc.buildout.download.Download(
-        raw_download_options, cache=extends_cache,
-        fallback=fallback, hash_name=True)
-    is_temp = False
-    downloaded_filename = None
-    if _isurl(filename):
-        downloaded_filename, is_temp = download(filename)
-        fp = open(downloaded_filename)
-        base = filename[:filename.rfind('/')]
-    elif _isurl(base):
-        if os.path.isabs(filename):
-            fp = open(filename)
-            base = os.path.dirname(filename)
-        else:
-            filename = base + '/' + filename
-            downloaded_filename, is_temp = download(filename)
-            fp = open(downloaded_filename)
+                downloaded_filename, is_temp = zc.buildout.download.Download(
+                    download_options, cache=extends_cache,
+                    fallback=bool_option(download_options, 'newest'),
+                    hash_name=True)(filename)
+                filename_for_logging = '%s (downloaded as %s)' % (
+                    filename, downloaded_filename)
                 base = filename[:filename.rfind('/')]
             else:
-        filename = os.path.join(base, filename)
-        fp = open(filename)
+                downloaded_filename = filename_for_logging = filename
                 base = os.path.dirname(filename)
-    downloaded.add(filename)
 
-    if filename in seen:
-        if is_temp:
-            fp.close()
-            os.remove(downloaded_filename)
-        raise zc.buildout.UserError("Recursive file include", seen, filename)
-
-    root_config_file = not seen
-    seen.append(filename)
-
-    filename_for_logging = filename
-    if downloaded_filename:
-        filename_for_logging = '%s (downloaded as %s)' % (
-            filename, downloaded_filename)
+            with open(downloaded_filename) as fp:
                 result = zc.buildout.configparser.parse(
                     fp, filename_for_logging, _default_globals)
-
-    fp.close()
+        finally:
             if is_temp:
                 os.remove(downloaded_filename)
 
-    options = result.get('buildout', {})
-    extends = options.pop('extends', None)
-    if 'extended-by' in options:
-        raise zc.buildout.UserError(
-            'No-longer supported "extended-by" option found in %s.' %
-            filename)
-
-    result = _annotate(result, filename)
-
-    if root_config_file and 'buildout' in result:
-        download_options = _update_section(
-            download_options, result['buildout']
-        )
-
-    if extends:
-        extends = extends.split()
-        eresult, user_defaults = _open(
-            base, extends.pop(0), seen, download_options, override,
-            downloaded, user_defaults
-        )
-        for fname in extends:
-            next_extend, user_defaults = _open(
-                base, fname, seen, download_options, override,
-                downloaded, user_defaults
-            )
-            eresult = _update(eresult, next_extend)
-        result = _update(eresult, result)
-    else:
-        if user_defaults:
-            result = _update(user_defaults, result)
-            user_defaults = {}
-    seen.pop()
-    return result, user_defaults
+        return self.collect(result, base, filename)
 
 
 ignore_directories = '.svn', 'CVS', '__pycache__', '.git'
@@ -1980,8 +1964,7 @@ def _dists_sig(dists):
             result.append(os.path.basename(location))
     return result
 
-def _update_section(in1, s2):
-    s1 = copy.deepcopy(in1)
+def _update_section(s1, s2):
     # Base section 2 on section 1; section 1 is copied, with key-value pairs
     # in section 2 overriding those in section 1. If there are += or -=
     # operators in section 2, process these to add or subtract items (delimited
@@ -2013,13 +1996,12 @@ def _update_section(in1, s2):
             s1[key] = section_key
     return s1
 
-def _update(in1, d2):
-    d1 = copy.deepcopy(in1)
+def _update(d1, d2):
     for section in d2:
         if section in d1:
-            d1[section] = _update_section(d1[section], d2[section])
+            _update_section(d1[section], d2[section])
         else:
-            d1[section] = copy.deepcopy(d2[section])
+            d1[section] = d2[section]
     return d1
 
 def _recipe(options):