Commit 7fded038 authored by Xavier Thompson's avatar Xavier Thompson

[feat] Reimplement the extends algorithm

The new algorithm avoids fetching the same extended file more than once
and correctly handles overriding values and += and -=:

The new algorithm starts as if there was a buildout file containing

```
[buildout]
extends =
  user/defaults.cfg # if it exists
  buildout.cfg # if it exists
  command_line_extends.cfg # if passed on the command line
```

The files are then fetched in depth-first-search postorder and fetching
child nodes in the order given by the extends directive, ignoring files
that have already been fetched.

The buildout dicts are then collected in order, and this linearisation
is then merged at the end, overriding the first configs collected with
the later ones. The first dict in the linearisation is not from a file,
but the dict of buildout's (hardcoded) defaults. This is equivalent to
acting as though every file that does not extend anything extends these
defaults.

The first time a file must be downloaded from a url, the linearisation
is merged with the configs already collected, and the resulting options
are then used to determine the download options for this download, and
every subsequent download.

This is a break with buildout's current logic for download options.

By analogy with classes in Python, we are computing a linearisation of
the class hierarchy to determine the method resolution order (MRO).
This algorithm is not the same as Python's MRO since Python 2.3 (C3).

It could be good to switch to a C3 linearisation like Python.
parent a1b8a4cf
...@@ -29,6 +29,11 @@ try: ...@@ -29,6 +29,11 @@ try:
except ImportError: except ImportError:
from UserDict import DictMixin from UserDict import DictMixin
try:
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin
import zc.buildout.configparser import zc.buildout.configparser
import copy import copy
import datetime import datetime
...@@ -261,11 +266,14 @@ def _print_annotate(data, verbose, chosen_sections, basedir): ...@@ -261,11 +266,14 @@ def _print_annotate(data, verbose, chosen_sections, basedir):
def _unannotate_section(section): def _unannotate_section(section):
return {key: entry.value for key, entry in section.items()} for key in section:
section[key] = section[key].value
return section
def _unannotate(data): def _unannotate(data):
return {key: _unannotate_section(section) for key, section in data.items()} for key in data:
_unannotate_section(data[key])
return data
def _format_picked_versions(picked_versions, required_by): def _format_picked_versions(picked_versions, required_by):
...@@ -355,54 +363,24 @@ class Buildout(DictMixin): ...@@ -355,54 +363,24 @@ class Buildout(DictMixin):
data['buildout']['directory'] = SectionKey( data['buildout']['directory'] = SectionKey(
os.path.dirname(config_file), 'COMPUTED_VALUE') os.path.dirname(config_file), 'COMPUTED_VALUE')
cloptions = dict( result = {}
(section, dict((option, SectionKey(value, 'COMMAND_LINE_VALUE')) for section, option, value in cloptions:
for (_, option, value) in v)) result.setdefault(section, {})[option] = value
for (section, v) in itertools.groupby(sorted(cloptions),
lambda v: v[0])
)
override = copy.deepcopy(cloptions.get('buildout', {}))
# load user defaults, which override defaults options = result.setdefault('buildout', {})
extends = []
user_config = _get_user_config() user_config = _get_user_config()
if use_user_defaults and os.path.exists(user_config): if use_user_defaults and os.path.exists(user_config):
download_options = data['buildout'] extends.append(user_config)
user_defaults, _ = _open(
os.path.dirname(user_config),
user_config, [], download_options,
override, set(), {}
)
for_download_options = _update(data, user_defaults)
else:
user_defaults = {}
for_download_options = copy.deepcopy(data)
# load configuration files
if config_file: if config_file:
download_options = for_download_options['buildout'] extends.append(config_file)
cfg_data, _ = _open( clextends = options.get('extends')
os.path.dirname(config_file), if clextends:
config_file, [], download_options, extends.append(clextends)
override, set(), user_defaults options['extends'] = '\n'.join(extends)
)
data = _update(data, cfg_data)
# extends from command-line
if 'buildout' in cloptions:
cl_extends = cloptions['buildout'].pop('extends', None)
if cl_extends:
for extends in cl_extends.value.split():
download_options = for_download_options['buildout']
cfg_data, _ = _open(
os.path.dirname(extends),
os.path.basename(extends),
[], download_options,
override, set(), user_defaults
)
data = _update(data, cfg_data)
# apply command-line options data = _extends(data, result, os.getcwd(), 'COMMAND_LINE_VALUE')
data = _update(data, cloptions)
# Set up versions section, if necessary # Set up versions section, if necessary
if 'versions' not in data['buildout']: if 'versions' not in data['buildout']:
...@@ -1539,10 +1517,10 @@ class Options(DictMixin): ...@@ -1539,10 +1517,10 @@ class Options(DictMixin):
raise zc.buildout.UserError("No section named %r" % iname) raise zc.buildout.UserError("No section named %r" % iname)
result.update(self._do_extend_raw(iname, raw, doing)) result.update(self._do_extend_raw(iname, raw, doing))
result = _annotate_section(result, "") _annotate_section(result, "")
data = _annotate_section(copy.deepcopy(data), "") data = _annotate_section(copy.deepcopy(data), "")
result = _update_section(result, data) _update_section(result, data)
result = _unannotate_section(result) _unannotate_section(result)
result.pop('<', None) result.pop('<', None)
return result return result
finally: finally:
...@@ -1831,101 +1809,107 @@ def _default_globals(): ...@@ -1831,101 +1809,107 @@ def _default_globals():
variable_template_split = re.compile('([$]{[^}]*})').split variable_template_split = re.compile('([$]{[^}]*})').split
def _open(
base, filename, seen, download_options,
override, downloaded, user_defaults
):
"""Open a configuration file and return the result as a dictionary,
Recursively open other files based on buildout options found. class _extends(object):
"""
download_options = _update_section(download_options, override)
raw_download_options = _unannotate_section(download_options)
newest = bool_option(raw_download_options, 'newest', 'false')
fallback = newest and not (filename in downloaded)
extends_cache = raw_download_options.get('extends-cache')
if extends_cache and variable_template_split(extends_cache)[1::2]:
raise ValueError(
"extends-cache '%s' may not contain ${section:variable} to expand."
% extends_cache
)
download = zc.buildout.download.Download(
raw_download_options, cache=extends_cache,
fallback=fallback, hash_name=True)
is_temp = False
downloaded_filename = None
if _isurl(filename):
downloaded_filename, is_temp = download(filename)
fp = open(downloaded_filename)
base = filename[:filename.rfind('/')]
elif _isurl(base):
if os.path.isabs(filename):
fp = open(filename)
base = os.path.dirname(filename)
else:
filename = base + '/' + filename
downloaded_filename, is_temp = download(filename)
fp = open(downloaded_filename)
base = filename[:filename.rfind('/')]
else:
filename = os.path.join(base, filename)
fp = open(filename)
base = os.path.dirname(filename)
downloaded.add(filename)
if filename in seen: def __new__(cls, defaults, *args):
if is_temp: self = super(_extends, cls).__new__(cls)
fp.close() self.seen = set()
os.remove(downloaded_filename) self.processing = []
raise zc.buildout.UserError("Recursive file include", seen, filename) self.extends = [defaults]
self._download_options = []
root_config_file = not seen self.collect(*args)
seen.append(filename) return self.merge()
filename_for_logging = filename
if downloaded_filename:
filename_for_logging = '%s (downloaded as %s)' % (
filename, downloaded_filename)
result = zc.buildout.configparser.parse(
fp, filename_for_logging, _default_globals)
fp.close()
if is_temp:
os.remove(downloaded_filename)
options = result.get('buildout', {})
extends = options.pop('extends', None)
if 'extended-by' in options:
raise zc.buildout.UserError(
'No-longer supported "extended-by" option found in %s.' %
filename)
result = _annotate(result, filename) def merge(self):
result = {}
for d in self.extends:
_update(result, d)
return result
if root_config_file and 'buildout' in result: def __getattr__(self, attr):
download_options = _update_section( if attr == 'download_options':
download_options, result['buildout'] # Compute processed options
) result_so_far = self.merge()
self.extends[:] = [result_so_far]
value = copy.deepcopy(result_so_far.get('buildout')) or {}
# Update with currently-being-processed options
for options in reversed(self._download_options):
_update_section(value, options)
value = _unannotate_section(value)
setattr(self, attr, value)
return value
return self.__getattribute__(attr)
def collect(self, result, base, filename):
options = result.get('buildout', {})
extends = options.pop('extends', '')
# Sanitize buildout options
if 'extended-by' in options:
raise zc.buildout.UserError(
'No-longer supported "extended-by" option found in %s.' %
filename)
_annotate(result, filename)
# Collect extends and unprocessed download options
self.processing.append(filename)
self._download_options.append(options)
for fextends in extends.split():
self.open(base, fextends)
self.extends.append(result)
del self.processing[-1], self._download_options[-1]
def open(self, base, filename):
# Determine file location
if _isurl(filename):
download = True
elif _isurl(base):
download = True
filename = urljoin(base + '/', filename)
else:
download = False
filename = os.path.realpath(
os.path.join(base, os.path.expanduser(filename)))
# Detect repetitions and loops
if filename in self.seen:
if filename in self.processing:
raise zc.buildout.UserError("circular extends: %s" % filename)
return
self.seen.add(filename)
if extends: # Fetch file
extends = extends.split() is_temp = False
eresult, user_defaults = _open( try:
base, extends.pop(0), seen, download_options, override, if download:
downloaded, user_defaults download_options = self.download_options
) extends_cache = download_options.get('extends-cache')
for fname in extends: if extends_cache and variable_template_split(extends_cache)[1::2]:
next_extend, user_defaults = _open( raise ValueError(
base, fname, seen, download_options, override, "extends-cache '%s' may not contain ${section:variable} to expand."
downloaded, user_defaults % extends_cache
) )
eresult = _update(eresult, next_extend) downloaded_filename, is_temp = zc.buildout.download.Download(
result = _update(eresult, result) download_options, cache=extends_cache,
else: fallback=bool_option(download_options, 'newest'),
if user_defaults: hash_name=True)(filename)
result = _update(user_defaults, result) filename_for_logging = '%s (downloaded as %s)' % (
user_defaults = {} filename, downloaded_filename)
seen.pop() base = filename[:filename.rfind('/')]
return result, user_defaults else:
downloaded_filename = filename_for_logging = filename
base = os.path.dirname(filename)
with open(downloaded_filename) as fp:
result = zc.buildout.configparser.parse(
fp, filename_for_logging, _default_globals)
finally:
if is_temp:
os.remove(downloaded_filename)
return self.collect(result, base, filename)
ignore_directories = '.svn', 'CVS', '__pycache__', '.git' ignore_directories = '.svn', 'CVS', '__pycache__', '.git'
...@@ -1980,8 +1964,7 @@ def _dists_sig(dists): ...@@ -1980,8 +1964,7 @@ def _dists_sig(dists):
result.append(os.path.basename(location)) result.append(os.path.basename(location))
return result return result
def _update_section(in1, s2): def _update_section(s1, s2):
s1 = copy.deepcopy(in1)
# Base section 2 on section 1; section 1 is copied, with key-value pairs # Base section 2 on section 1; section 1 is copied, with key-value pairs
# in section 2 overriding those in section 1. If there are += or -= # in section 2 overriding those in section 1. If there are += or -=
# operators in section 2, process these to add or subtract items (delimited # operators in section 2, process these to add or subtract items (delimited
...@@ -2013,13 +1996,12 @@ def _update_section(in1, s2): ...@@ -2013,13 +1996,12 @@ def _update_section(in1, s2):
s1[key] = section_key s1[key] = section_key
return s1 return s1
def _update(in1, d2): def _update(d1, d2):
d1 = copy.deepcopy(in1)
for section in d2: for section in d2:
if section in d1: if section in d1:
d1[section] = _update_section(d1[section], d2[section]) _update_section(d1[section], d2[section])
else: else:
d1[section] = copy.deepcopy(d2[section]) d1[section] = d2[section]
return d1 return d1
def _recipe(options): def _recipe(options):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment