Commit 7fded038 authored by Xavier Thompson's avatar Xavier Thompson

[feat] Reimplement the extends algorithm

The new algorithm avoids fetching the same extended file more than once
and correctly handles overriding values and += and -=:

The new algorithm starts as if there was a buildout file containing

```
[buildout]
extends =
  user/defaults.cfg # if it exists
  buildout.cfg # if it exists
  command_line_extends.cfg # if passed on the command line
```

The files are then fetched in depth-first-search postorder and fetching
child nodes in the order given by the extends directive, ignoring files
that have already been fetched.

The buildout dicts are then collected in order, and this linearisation
is then merged at the end, overriding the first configs collected with
the later ones. The first dict in the linearisation is not from a file,
but the dict of buildout's (hardcoded) defaults. This is equivalent to
acting as though every file that does not extend anything extends these
defaults.

The first time a file must be downloaded from a url, the linearisation
is merged with the configs already collected, and the resulting options
are then used to determine the download options for this download, and
every subsequent download.

This is a break with buildout's current logic for download options.

By analogy with classes in Python, we are computing a linearisation of
the class hierarchy to determine the method resolution order (MRO).
This algorithm is not the same as Python's MRO since Python 2.3 (C3).

It could be good to switch to a C3 linearisation like Python.
parent a1b8a4cf
......@@ -29,6 +29,11 @@ try:
except ImportError:
from UserDict import DictMixin
try:
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin
import zc.buildout.configparser
import copy
import datetime
......@@ -261,11 +266,14 @@ def _print_annotate(data, verbose, chosen_sections, basedir):
def _unannotate_section(section):
return {key: entry.value for key, entry in section.items()}
for key in section:
section[key] = section[key].value
return section
def _unannotate(data):
return {key: _unannotate_section(section) for key, section in data.items()}
for key in data:
_unannotate_section(data[key])
return data
def _format_picked_versions(picked_versions, required_by):
......@@ -355,54 +363,24 @@ class Buildout(DictMixin):
data['buildout']['directory'] = SectionKey(
os.path.dirname(config_file), 'COMPUTED_VALUE')
cloptions = dict(
(section, dict((option, SectionKey(value, 'COMMAND_LINE_VALUE'))
for (_, option, value) in v))
for (section, v) in itertools.groupby(sorted(cloptions),
lambda v: v[0])
)
override = copy.deepcopy(cloptions.get('buildout', {}))
result = {}
for section, option, value in cloptions:
result.setdefault(section, {})[option] = value
# load user defaults, which override defaults
options = result.setdefault('buildout', {})
extends = []
user_config = _get_user_config()
if use_user_defaults and os.path.exists(user_config):
download_options = data['buildout']
user_defaults, _ = _open(
os.path.dirname(user_config),
user_config, [], download_options,
override, set(), {}
)
for_download_options = _update(data, user_defaults)
else:
user_defaults = {}
for_download_options = copy.deepcopy(data)
# load configuration files
extends.append(user_config)
if config_file:
download_options = for_download_options['buildout']
cfg_data, _ = _open(
os.path.dirname(config_file),
config_file, [], download_options,
override, set(), user_defaults
)
data = _update(data, cfg_data)
# extends from command-line
if 'buildout' in cloptions:
cl_extends = cloptions['buildout'].pop('extends', None)
if cl_extends:
for extends in cl_extends.value.split():
download_options = for_download_options['buildout']
cfg_data, _ = _open(
os.path.dirname(extends),
os.path.basename(extends),
[], download_options,
override, set(), user_defaults
)
data = _update(data, cfg_data)
extends.append(config_file)
clextends = options.get('extends')
if clextends:
extends.append(clextends)
options['extends'] = '\n'.join(extends)
# apply command-line options
data = _update(data, cloptions)
data = _extends(data, result, os.getcwd(), 'COMMAND_LINE_VALUE')
# Set up versions section, if necessary
if 'versions' not in data['buildout']:
......@@ -1539,10 +1517,10 @@ class Options(DictMixin):
raise zc.buildout.UserError("No section named %r" % iname)
result.update(self._do_extend_raw(iname, raw, doing))
result = _annotate_section(result, "")
_annotate_section(result, "")
data = _annotate_section(copy.deepcopy(data), "")
result = _update_section(result, data)
result = _unannotate_section(result)
_update_section(result, data)
_unannotate_section(result)
result.pop('<', None)
return result
finally:
......@@ -1831,101 +1809,107 @@ def _default_globals():
variable_template_split = re.compile('([$]{[^}]*})').split
def _open(
base, filename, seen, download_options,
override, downloaded, user_defaults
):
"""Open a configuration file and return the result as a dictionary,
Recursively open other files based on buildout options found.
"""
download_options = _update_section(download_options, override)
raw_download_options = _unannotate_section(download_options)
newest = bool_option(raw_download_options, 'newest', 'false')
fallback = newest and not (filename in downloaded)
extends_cache = raw_download_options.get('extends-cache')
if extends_cache and variable_template_split(extends_cache)[1::2]:
raise ValueError(
"extends-cache '%s' may not contain ${section:variable} to expand."
% extends_cache
)
download = zc.buildout.download.Download(
raw_download_options, cache=extends_cache,
fallback=fallback, hash_name=True)
is_temp = False
downloaded_filename = None
if _isurl(filename):
downloaded_filename, is_temp = download(filename)
fp = open(downloaded_filename)
base = filename[:filename.rfind('/')]
elif _isurl(base):
if os.path.isabs(filename):
fp = open(filename)
base = os.path.dirname(filename)
else:
filename = base + '/' + filename
downloaded_filename, is_temp = download(filename)
fp = open(downloaded_filename)
base = filename[:filename.rfind('/')]
else:
filename = os.path.join(base, filename)
fp = open(filename)
base = os.path.dirname(filename)
downloaded.add(filename)
class _extends(object):
if filename in seen:
if is_temp:
fp.close()
os.remove(downloaded_filename)
raise zc.buildout.UserError("Recursive file include", seen, filename)
root_config_file = not seen
seen.append(filename)
filename_for_logging = filename
if downloaded_filename:
filename_for_logging = '%s (downloaded as %s)' % (
filename, downloaded_filename)
result = zc.buildout.configparser.parse(
fp, filename_for_logging, _default_globals)
fp.close()
if is_temp:
os.remove(downloaded_filename)
options = result.get('buildout', {})
extends = options.pop('extends', None)
if 'extended-by' in options:
raise zc.buildout.UserError(
'No-longer supported "extended-by" option found in %s.' %
filename)
def __new__(cls, defaults, *args):
self = super(_extends, cls).__new__(cls)
self.seen = set()
self.processing = []
self.extends = [defaults]
self._download_options = []
self.collect(*args)
return self.merge()
result = _annotate(result, filename)
def merge(self):
result = {}
for d in self.extends:
_update(result, d)
return result
if root_config_file and 'buildout' in result:
download_options = _update_section(
download_options, result['buildout']
)
def __getattr__(self, attr):
if attr == 'download_options':
# Compute processed options
result_so_far = self.merge()
self.extends[:] = [result_so_far]
value = copy.deepcopy(result_so_far.get('buildout')) or {}
# Update with currently-being-processed options
for options in reversed(self._download_options):
_update_section(value, options)
value = _unannotate_section(value)
setattr(self, attr, value)
return value
return self.__getattribute__(attr)
def collect(self, result, base, filename):
options = result.get('buildout', {})
extends = options.pop('extends', '')
# Sanitize buildout options
if 'extended-by' in options:
raise zc.buildout.UserError(
'No-longer supported "extended-by" option found in %s.' %
filename)
_annotate(result, filename)
# Collect extends and unprocessed download options
self.processing.append(filename)
self._download_options.append(options)
for fextends in extends.split():
self.open(base, fextends)
self.extends.append(result)
del self.processing[-1], self._download_options[-1]
def open(self, base, filename):
# Determine file location
if _isurl(filename):
download = True
elif _isurl(base):
download = True
filename = urljoin(base + '/', filename)
else:
download = False
filename = os.path.realpath(
os.path.join(base, os.path.expanduser(filename)))
# Detect repetitions and loops
if filename in self.seen:
if filename in self.processing:
raise zc.buildout.UserError("circular extends: %s" % filename)
return
self.seen.add(filename)
if extends:
extends = extends.split()
eresult, user_defaults = _open(
base, extends.pop(0), seen, download_options, override,
downloaded, user_defaults
)
for fname in extends:
next_extend, user_defaults = _open(
base, fname, seen, download_options, override,
downloaded, user_defaults
)
eresult = _update(eresult, next_extend)
result = _update(eresult, result)
else:
if user_defaults:
result = _update(user_defaults, result)
user_defaults = {}
seen.pop()
return result, user_defaults
# Fetch file
is_temp = False
try:
if download:
download_options = self.download_options
extends_cache = download_options.get('extends-cache')
if extends_cache and variable_template_split(extends_cache)[1::2]:
raise ValueError(
"extends-cache '%s' may not contain ${section:variable} to expand."
% extends_cache
)
downloaded_filename, is_temp = zc.buildout.download.Download(
download_options, cache=extends_cache,
fallback=bool_option(download_options, 'newest'),
hash_name=True)(filename)
filename_for_logging = '%s (downloaded as %s)' % (
filename, downloaded_filename)
base = filename[:filename.rfind('/')]
else:
downloaded_filename = filename_for_logging = filename
base = os.path.dirname(filename)
with open(downloaded_filename) as fp:
result = zc.buildout.configparser.parse(
fp, filename_for_logging, _default_globals)
finally:
if is_temp:
os.remove(downloaded_filename)
return self.collect(result, base, filename)
ignore_directories = '.svn', 'CVS', '__pycache__', '.git'
......@@ -1980,8 +1964,7 @@ def _dists_sig(dists):
result.append(os.path.basename(location))
return result
def _update_section(in1, s2):
s1 = copy.deepcopy(in1)
def _update_section(s1, s2):
# Base section 2 on section 1; section 1 is copied, with key-value pairs
# in section 2 overriding those in section 1. If there are += or -=
# operators in section 2, process these to add or subtract items (delimited
......@@ -2013,13 +1996,12 @@ def _update_section(in1, s2):
s1[key] = section_key
return s1
def _update(in1, d2):
d1 = copy.deepcopy(in1)
def _update(d1, d2):
for section in d2:
if section in d1:
d1[section] = _update_section(d1[section], d2[section])
_update_section(d1[section], d2[section])
else:
d1[section] = copy.deepcopy(d2[section])
d1[section] = d2[section]
return d1
def _recipe(options):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment