Commit ecb15c89 authored by Godefroid Chapelle's avatar Godefroid Chapelle Committed by GitHub

Merge pull request #378 from buildout/annotate_verbose

verbose mode and select sections for annotate command
parents d6fba392 35ffe41f
.installed.cfg
bin/
build/
include/
develop-eggs/
eggs/
parts/
......
......@@ -4,7 +4,9 @@ Change History
2.9.3 (unreleased)
==================
- Nothing changed yet.
- Add more verbosity to ``annotate`` results with ``-v``
- Select one or more sections with arguments after ``buildout annotate``.
2.9.2 (2017-03-06)
......
......@@ -117,6 +117,19 @@ where they came from. Try it!
.. _bootstrap-command:
.. code-block:: console
buildout -v annotate
Increase the verbosity of the output to display all steps that compute the final values used by buildout.
.. code-block:: console
buildout annotate versions
You can pass one or more section names as arguments to display annotation only for the given sections.
bootstrap
_________
......
......@@ -75,49 +75,172 @@ class MissingSection(zc.buildout.UserError, KeyError):
return "The referenced section, %r, was not defined." % self.args[0]
def _annotate_section(section, note):
def _annotate_section(section, source):
for key in section:
section[key] = (section[key], note)
section[key] = SectionKey(section[key], source)
return section
class SectionKey(object):
def __init__(self, value, source):
self.history = []
self.value = value
self.addToHistory("SET", value, source)
@property
def source(self):
return self.history[-1].source
def overrideValue(self, sectionkey):
self.value = sectionkey.value
if sectionkey.history[-1].operation not in ['ADD', 'REMOVE']:
self.addToHistory("OVERRIDE", sectionkey.value, sectionkey.source)
def setDirectory(self, value):
self.value = value
self.addToHistory("DIRECTORY", value, self.source)
def addToValue(self, added, source):
subvalues = self.value.split('\n') + added.split('\n')
self.value = "\n".join(subvalues)
self.addToHistory("ADD", added, source)
def removeFromValue(self, removed, source):
subvalues = [
v
for v in self.value.split('\n')
if v not in removed.split('\n')
]
self.value = "\n".join(subvalues)
self.addToHistory("REMOVE", removed, source)
def addToHistory(self, operation, value, source):
item = HistoryItem(operation, value, source)
self.history.append(item)
def printAll(self, key, basedir, verbose):
self.printKeyAndValue(key)
if verbose:
self.printVerbose(basedir)
else:
self.printTerse(basedir)
def printKeyAndValue(self, key):
lines = self.value.splitlines()
if len(lines) <= 1:
args = [key, "="]
if self.value:
args.append(" ")
args.append(self.value)
print_(*args, sep='')
else:
print_(key, "= ", lines[0], sep='')
for line in lines[1:]:
print_(line)
def printVerbose(self, basedir):
print_()
for item in reversed(self.history):
item.printAll(basedir)
print_()
def printTerse(self, basedir):
toprint = []
history = copy.deepcopy(self.history)
while history:
next = history.pop()
if next.operation in ["ADD", "REMOVE"]:
next.printShort(toprint, basedir)
else:
next.printShort(toprint, basedir)
break
for line in reversed(toprint):
if line.strip():
print_(line)
def __repr__(self):
return "<SectionKey value=%s source=%s>" % (
" ".join(self.value.split('\n')), self.source)
class HistoryItem(object):
def __init__(self, operation, value, source):
self.operation = operation
self.value = value
self.source = source
def printShort(self, toprint, basedir):
source = self.source_for_human(basedir)
if self.operation in ["OVERRIDE", "SET", "DIRECTORY"]:
toprint.append(" " + source)
elif self.operation == "ADD":
toprint.append("+= " + source)
elif self.operation == "REMOVE":
toprint.append("-= " + source)
def printOperation(self):
lines = self.value.splitlines()
if len(lines) <= 1:
print_(" ", self.operation, "VALUE =", self.value)
else:
print_(" ", self.operation, "VALUE =")
for line in lines:
print_(" ", " ", line)
def printSource(self, basedir):
if self.source in (
'DEFAULT_VALUE', 'COMPUTED_VALUE', 'COMMAND_LINE_VALUE'
):
prefix = "AS"
else:
prefix = "IN"
print_(" ", prefix, self.source_for_human(basedir))
def source_for_human(self, basedir):
if self.source.startswith(basedir):
return os.path.relpath(self.source, basedir)
else:
return self.source
def printAll(self, basedir):
self.printSource(basedir)
self.printOperation()
def __repr__(self):
return "<HistoryItem operation=%s value=%s source=%s>" % (
self.operation, " ".join(self.value.split('\n')), self.source)
def _annotate(data, note):
for key in data:
data[key] = _annotate_section(data[key], note)
return data
def _print_annotate(data):
def _print_annotate(data, verbose, chosen_sections, basedir):
sections = list(data.keys())
sections.sort()
print_()
print_("Annotated sections")
print_("="*len("Annotated sections"))
for section in sections:
if (not chosen_sections) or (section in chosen_sections):
print_()
print_('[%s]' % section)
keys = list(data[section].keys())
keys.sort()
for key in keys:
value, notes = data[section][key]
keyvalue = "%s= %s" % (key, value)
print_(keyvalue)
line = ' '
for note in notes.split():
if note == '[+]':
line = '+= '
elif note == '[-]':
line = '-= '
else:
print_(line, note)
line = ' '
print_()
sectionkey = data[section][key]
sectionkey.printAll(key, basedir, verbose)
def _unannotate_section(section):
for key in section:
value, note = section[key]
section[key] = value
section[key] = section[key].value
return section
def _unannotate(data):
for key in data:
data[key] = _unannotate_section(data[key])
......@@ -164,6 +287,7 @@ _buildout_default_options = _annotate_section({
'use-dependency-links': 'true',
}, 'DEFAULT_VALUE')
class Buildout(DictMixin):
def __init__(self, config_file, cloptions,
......@@ -173,12 +297,13 @@ class Buildout(DictMixin):
__doing__ = 'Initializing.'
# default options
data = dict(buildout=_buildout_default_options.copy())
_buildout_default_options_copy = copy.deepcopy(
_buildout_default_options)
data = dict(buildout=_buildout_default_options_copy)
self._buildout_dir = os.getcwd()
if config_file and not _isurl(config_file):
config_file = os.path.abspath(config_file)
base = os.path.dirname(config_file)
if not os.path.exists(config_file):
if command == 'init':
self._init_config(config_file, args)
......@@ -186,7 +311,8 @@ class Buildout(DictMixin):
# Sigh. This model of a buildout instance
# with methods is breaking down. :(
config_file = None
data['buildout']['directory'] = ('.', 'COMPUTED_VALUE')
data['buildout']['directory'] = SectionKey(
'.', 'COMPUTED_VALUE')
else:
raise zc.buildout.UserError(
"Couldn't open %s" % config_file)
......@@ -195,19 +321,16 @@ class Buildout(DictMixin):
"%r already exists." % config_file)
if config_file:
data['buildout']['directory'] = (os.path.dirname(config_file),
'COMPUTED_VALUE')
else:
base = None
data['buildout']['directory'] = SectionKey(
os.path.dirname(config_file), 'COMPUTED_VALUE')
cloptions = dict(
(section, dict((option, (value, 'COMMAND_LINE_VALUE'))
(section, dict((option, SectionKey(value, 'COMMAND_LINE_VALUE'))
for (_, option, value) in v))
for (section, v) in itertools.groupby(sorted(cloptions),
lambda v: v[0])
)
override = cloptions.get('buildout', {}).copy()
override = copy.deepcopy(cloptions.get('buildout', {}))
# load user defaults, which override defaults
if user_defaults:
......@@ -218,32 +341,35 @@ class Buildout(DictMixin):
os.path.expanduser('~'), '.buildout')
user_config = os.path.join(buildout_home, 'default.cfg')
if os.path.exists(user_config):
data_buildout_copy = copy.deepcopy(data['buildout'])
_update(data, _open(os.path.dirname(user_config), user_config,
[], data['buildout'].copy(), override,
[], data_buildout_copy, override,
set()))
# load configuration files
if config_file:
data_buildout_copy = copy.deepcopy(data['buildout'])
_update(data, _open(os.path.dirname(config_file), config_file, [],
data['buildout'].copy(), override, set()))
data_buildout_copy, override, set()))
# apply command-line options
_update(data, cloptions)
# Set up versions section, if necessary
if 'versions' not in data['buildout']:
data['buildout']['versions'] = ('versions', 'DEFAULT_VALUE')
data['buildout']['versions'] = SectionKey(
'versions', 'DEFAULT_VALUE')
if 'versions' not in data:
data['versions'] = {}
# Default versions:
versions_section_name = data['buildout']['versions'][0]
versions_section_name = data['buildout']['versions'].value
if versions_section_name:
versions = data[versions_section_name]
else:
versions = {}
versions.update(
dict((k, (v, 'DEFAULT_VALUE'))
dict((k, SectionKey(v, 'DEFAULT_VALUE'))
for (k, v) in (
# Prevent downgrading due to prefer-final:
('zc.buildout',
......@@ -262,7 +388,9 @@ class Buildout(DictMixin):
# file location
for name in ('download-cache', 'eggs-directory', 'extends-cache'):
if name in data['buildout']:
origdir, src = data['buildout'][name]
sectionkey = data['buildout'][name]
origdir = sectionkey.value
src = sectionkey.source
if '${' in origdir:
continue
if not os.path.isabs(origdir):
......@@ -270,7 +398,7 @@ class Buildout(DictMixin):
'COMPUTED_VALUE',
'COMMAND_LINE_VALUE'):
if 'directory' in data['buildout']:
basedir = data['buildout']['directory'][0]
basedir = data['buildout']['directory'].value
else:
basedir = self._buildout_dir
else:
......@@ -285,7 +413,7 @@ class Buildout(DictMixin):
if not os.path.isabs(absdir):
absdir = os.path.join(basedir, absdir)
absdir = os.path.abspath(absdir)
data['buildout'][name] = (absdir, src)
sectionkey.setDirectory(absdir)
self._annotated = copy.deepcopy(data)
self._raw = _unannotate(data)
......@@ -365,7 +493,7 @@ class Buildout(DictMixin):
versions = self[versions_section_name]
else:
# remove annotations
versions = dict((k, v[0]) for (k, v) in versions.items())
versions = dict((k, v.value) for (k, v) in versions.items())
options['versions'] # refetching section name just to avoid a warning
self.versions = versions
zc.buildout.easy_install.default_versions(versions)
......@@ -1106,7 +1234,13 @@ class Buildout(DictMixin):
runsetup = setup # backward compat.
def annotate(self, args=None):
_print_annotate(self._annotated)
verbose = self['buildout'].get('verbosity', 0) != 0
section = None
if args is None:
sections = []
else:
sections = args
_print_annotate(self._annotated, verbose, sections, self._buildout_dir)
def print_options(self, base_path=None):
for section in sorted(self._data):
......@@ -1279,7 +1413,7 @@ class Options(DictMixin):
result.update(self._do_extend_raw(iname, raw, doing))
result = _annotate_section(result, "")
data = _annotate_section(data.copy(), "")
data = _annotate_section(copy.deepcopy(data), "")
_update_section(result, data)
result = _unannotate_section(result)
result.pop('<', None)
......@@ -1405,7 +1539,7 @@ class Options(DictMixin):
return len(self.keys())
def copy(self):
result = self._raw.copy()
result = copy.deepcopy(self._raw)
result.update(self._cooked)
result.update(self._data)
return result
......@@ -1573,7 +1707,8 @@ def _open(base, filename, seen, dl_options, override, downloaded):
Recursively open other files based on buildout options found.
"""
_update_section(dl_options, override)
_dl_options = _unannotate_section(dl_options.copy())
dl_options_copy = copy.deepcopy(dl_options)
_dl_options = _unannotate_section(dl_options_copy)
newest = bool_option(_dl_options, 'newest', 'false')
fallback = newest and not (filename in downloaded)
download = zc.buildout.download.Download(
......@@ -1702,31 +1837,37 @@ def _update_section(s1, s2):
# in section 2 overriding those in section 1. If there are += or -=
# operators in section 2, process these to add or substract items (delimited
# by newlines) from the preexisting values.
s2 = s2.copy() # avoid mutating the second argument, which is unexpected
s2 = copy.deepcopy(s2) # avoid mutating the second argument, which is unexpected
# Sort on key, then on the addition or substraction operator (+ comes first)
for k, v in sorted(s2.items(), key=lambda x: (x[0].rstrip(' +'), x[0][-1])):
v2, note2 = v
if k.endswith('+'):
key = k.rstrip(' +')
implicit_value = SectionKey("", "IMPLICIT_VALUE")
# Find v1 in s2 first; it may have been defined locally too.
v1, note1 = s2.get(key, s1.get(key, ("", "")))
newnote = ' [+] '.join((note1, note2)).strip()
s2[key] = "\n".join((v1).split('\n') +
v2.split('\n')), newnote
section_key = s2.get(key, s1.get(key, implicit_value))
section_key.addToValue(v.value, v.source)
s2[key] = section_key
del s2[k]
elif k.endswith('-'):
key = k.rstrip(' -')
implicit_value = SectionKey("", "IMPLICIT_VALUE")
# Find v1 in s2 first; it may have been set by a += operation first
v1, note1 = s2.get(key, s1.get(key, ("", "")))
newnote = ' [-] '.join((note1, note2)).strip()
s2[key] = ("\n".join(
[v for v in v1.split('\n')
if v not in v2.split('\n')]), newnote)
section_key = s2.get(key, s1.get(key, implicit_value))
section_key.removeFromValue(v.value, v.source)
s2[key] = section_key
del s2[k]
s1.update(s2)
_update_verbose(s1, s2)
return s1
def _update_verbose(s1, s2):
for key, v2 in s2.items():
if key in s1:
v1 = s1[key]
v1.overrideValue(v2)
else:
s1[key] = v2
def _update(d1, d2):
for section in d2:
if section in d1:
......
......@@ -809,7 +809,7 @@ the origin of the value (file name or ``COMPUTED_VALUE``, ``DEFAULT_VALUE``,
bin-directory= bin
DEFAULT_VALUE
develop= recipes
/sample-buildout/buildout.cfg
buildout.cfg
develop-eggs-directory= develop-eggs
DEFAULT_VALUE
directory= /sample-buildout
......@@ -833,7 +833,7 @@ the origin of the value (file name or ``COMPUTED_VALUE``, ``DEFAULT_VALUE``,
offline= false
DEFAULT_VALUE
parts= data-dir
/sample-buildout/buildout.cfg
buildout.cfg
parts-directory= parts
DEFAULT_VALUE
prefer-final= true
......@@ -853,9 +853,178 @@ the origin of the value (file name or ``COMPUTED_VALUE``, ``DEFAULT_VALUE``,
<BLANKLINE>
[data-dir]
path= foo bins
/sample-buildout/buildout.cfg
buildout.cfg
recipe= recipes:mkdir
/sample-buildout/buildout.cfg
buildout.cfg
<BLANKLINE>
[versions]
zc.buildout = >=1.99
DEFAULT_VALUE
zc.recipe.egg = >=1.99
DEFAULT_VALUE
<BLANKLINE>
The ``annotate`` command is sensitive to the verbosity flag.
You get more information about the way values are computed::
>>> print_(system(buildout+ ' -v annotate'), end='')
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
<BLANKLINE>
Annotated sections
==================
<BLANKLINE>
[buildout]
allow-hosts= *
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = *
<BLANKLINE>
allow-picked-versions= true
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = true
<BLANKLINE>
bin-directory= bin
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = bin
<BLANKLINE>
develop= recipes
<BLANKLINE>
IN buildout.cfg
SET VALUE = recipes
<BLANKLINE>
develop-eggs-directory= develop-eggs
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = develop-eggs
<BLANKLINE>
directory= /sample-buildout
<BLANKLINE>
AS COMPUTED_VALUE
SET VALUE = /sample-buildout
<BLANKLINE>
eggs-directory= /sample-buildout/eggs
<BLANKLINE>
AS DEFAULT_VALUE
DIRECTORY VALUE = /sample-buildout/eggs
AS DEFAULT_VALUE
SET VALUE = eggs
<BLANKLINE>
executable= ...
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = ...
<BLANKLINE>
find-links=
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE =
<BLANKLINE>
install-from-cache= false
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = false
<BLANKLINE>
installed= .installed.cfg
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = .installed.cfg
<BLANKLINE>
log-format=
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE =
<BLANKLINE>
log-level= INFO
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = INFO
<BLANKLINE>
newest= true
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = true
<BLANKLINE>
offline= false
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = false
<BLANKLINE>
parts= data-dir
<BLANKLINE>
IN buildout.cfg
SET VALUE = data-dir
<BLANKLINE>
parts-directory= parts
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = parts
<BLANKLINE>
prefer-final= true
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = true
<BLANKLINE>
python= buildout
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = buildout
<BLANKLINE>
show-picked-versions= false
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = false
<BLANKLINE>
socket-timeout=
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE =
<BLANKLINE>
update-versions-file=
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE =
<BLANKLINE>
use-dependency-links= true
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = true
<BLANKLINE>
verbosity= 10
<BLANKLINE>
AS COMMAND_LINE_VALUE
SET VALUE = 10
<BLANKLINE>
versions= versions
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = versions
<BLANKLINE>
<BLANKLINE>
[data-dir]
path= foo bins
<BLANKLINE>
IN buildout.cfg
SET VALUE = foo bins
<BLANKLINE>
recipe= recipes:mkdir
<BLANKLINE>
IN buildout.cfg
SET VALUE = recipes:mkdir
<BLANKLINE>
<BLANKLINE>
[versions]
...
The output of the ``annotate`` command can be very long.
You can restrict the output to some sections by passing section names as arguments::
>>> print_(system(buildout+ ' annotate versions'), end='')
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
<BLANKLINE>
Annotated sections
==================
<BLANKLINE>
[versions]
zc.buildout= >=1.99
......@@ -1384,42 +1553,42 @@ operations::
option= a1 a2
a3 a4
a5
/sample-buildout/base.cfg
+= /sample-buildout/extension1.cfg
+= /sample-buildout/extension2.cfg
base.cfg
+= extension1.cfg
+= extension2.cfg
recipe=
/sample-buildout/base.cfg
base.cfg
<BLANKLINE>
[part2]
option= b1 b2 b3 b4
/sample-buildout/base.cfg
-= /sample-buildout/extension1.cfg
-= /sample-buildout/extension2.cfg
base.cfg
-= extension1.cfg
-= extension2.cfg
recipe=
/sample-buildout/base.cfg
base.cfg
<BLANKLINE>
[part3]
option= c1 c2
c3 c4 c5
/sample-buildout/base.cfg
+= /sample-buildout/extension1.cfg
base.cfg
+= extension1.cfg
recipe=
/sample-buildout/base.cfg
base.cfg
<BLANKLINE>
[part4]
option= d2
d3
d1
d4
/sample-buildout/base.cfg
+= /sample-buildout/extension1.cfg
-= /sample-buildout/extension1.cfg
base.cfg
+= extension1.cfg
-= extension1.cfg
recipe=
/sample-buildout/base.cfg
base.cfg
<BLANKLINE>
[part5]
option= h1 h2
/sample-buildout/extension1.cfg
extension1.cfg
[versions]
zc.buildout= >=1.99
DEFAULT_VALUE
......@@ -1427,6 +1596,77 @@ operations::
DEFAULT_VALUE
<BLANKLINE>
With more verbosity::
>>> print_(system(os.path.join('bin', 'buildout') + ' -v annotate'), end='')
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
<BLANKLINE>
Annotated sections
==================
...
[part1]
option= a1 a2
a3 a4
a5
<BLANKLINE>
IN extension2.cfg
ADD VALUE = a5
IN extension1.cfg
ADD VALUE = a3 a4
IN base.cfg
SET VALUE = a1 a2
<BLANKLINE>
...
[part2]
option= b1 b2 b3 b4
<BLANKLINE>
IN extension2.cfg
REMOVE VALUE = b1 b2 b3
IN extension1.cfg
REMOVE VALUE = b1 b2
IN base.cfg
SET VALUE = b1 b2 b3 b4
<BLANKLINE>
...
[part3]
option=
c1 c2
c3 c4 c5
<BLANKLINE>
IN extension1.cfg
ADD VALUE = c3 c4 c5
IN base.cfg
SET VALUE = c1 c2
<BLANKLINE>
...
[part4]
option=
d2
d3
d1
d4
<BLANKLINE>
IN extension1.cfg
REMOVE VALUE = d5
IN extension1.cfg
ADD VALUE =
d1
d4
IN base.cfg
SET VALUE =
d2
d3
d5
<BLANKLINE>
...
[part5]
option= h1 h2
<BLANKLINE>
IN extension1.cfg
SET VALUE = h1 h2
<BLANKLINE>
...
Cleanup::
>>> os.remove(os.path.join(sample_buildout, 'base.cfg'))
......
......@@ -3539,7 +3539,7 @@ def test_suite():
(re.compile('setuptools'), 'setuptools'),
(re.compile('Got zc.recipe.egg \S+'), 'Got zc.recipe.egg'),
(re.compile(r'zc\.(buildout|recipe\.egg)\s*= >=\S+'),
'zc.\1 = >=1.99'),
'zc.\\1 = >=1.99'),
])
) + manuel.capture.Manuel(),
'buildout.txt',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment