Commit 8be81fe4 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Julien Muchembled

Add referred parts' hash strings in __buildout_signature__, that invokes...

Add referred parts' hash strings in __buildout_signature__, that invokes rebuild of a part when one of its (recursive) dependencies are modified.

Also remove duplicates and sort entries in __buildout_signature__.
parent b096b83a
......@@ -310,6 +310,7 @@ class Buildout(DictMixin):
self._raw = _unannotate(data)
self._data = {}
self._parts = []
self._initializing = []
# provide some defaults before options are parsed
# because while parsing options those attributes might be
......@@ -839,14 +840,29 @@ class Buildout(DictMixin):
"Unexpected entry, %r, in develop-eggs directory.", f)
def _compute_part_signatures(self, parts):
# The same recipe may appear many times.
sig_cache= {}
# Compute recipe signature and add to options
for part in parts:
options = self.get(part)
if options is None:
options = self[part] = {}
recipe, entry = _recipe(options)
req = pkg_resources.Requirement.parse(recipe)
sig = _dists_sig(pkg_resources.working_set.resolve([req]))
try:
# Copy cached list, because sig.append is called later
sig = sig_cache[recipe][:]
except KeyError:
req = pkg_resources.Requirement.parse(recipe)
sig_result = sig_cache[recipe] = sorted(set(_dists_sig(pkg_resources.working_set.resolve([req]))))
sig = sig_result[:]
for dependency in sorted(options.depends):
m = md5()
for item in sorted(self[dependency].items()):
m.update(('%r\0%r\0' % item).encode())
sig.append('%s:%s' % (dependency, m.hexdigest()))
options['__buildout_signature__'] = ' '.join(sig)
def _read_installed_part_options(self):
......@@ -1178,21 +1194,32 @@ class Buildout(DictMixin):
v = ' '+v
print_("%s =%s" % (k, v))
def __getitem__(self, section):
__doing__ = 'Getting section %s.', section
def initialize(self, options, reqs, entry):
recipe_class = _install_and_load(reqs, 'zc.buildout', entry, self)
self._initializing.append(options)
try:
return self._data[section]
except KeyError:
pass
return recipe_class(self, options.name, options)
finally:
del self._initializing[-1]
def __getitem__(self, section):
__doing__ = 'Getting section %s.', section
try:
data = self._raw[section]
options = self._data[section]
except KeyError:
raise MissingSection(section)
options = self.Options(self, section, data)
self._data[section] = options
options._initialize()
try:
data = self._raw[section]
except KeyError:
raise MissingSection(section)
options = self.Options(self, section, data)
self._data[section] = options
options._initialize()
if self._initializing:
caller = self._initializing[-1]
if 'buildout' != section != caller.name:
caller.depends.add(section)
return options
def __setitem__(self, name, data):
......@@ -1275,6 +1302,7 @@ class Options(DictMixin):
self._raw = data
self._cooked = {}
self._data = {}
self.depends = set()
def _initialize(self):
name = self.name
......@@ -1292,17 +1320,9 @@ class Options(DictMixin):
return # buildout section can never be a part
if self.get('recipe'):
self.initialize()
self.recipe = self.buildout.initialize(self, *_recipe(self._data))
self.buildout._parts.append(name)
def initialize(self):
reqs, entry = _recipe(self._data)
buildout = self.buildout
recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout)
name = self.name
self.recipe = recipe_class(buildout, name, self)
def _do_extend_raw(self, name, data, doing):
if name == 'buildout':
return data
......@@ -1409,7 +1429,14 @@ class Options(DictMixin):
section, option = s
if not section:
section = self.name
v = self.buildout[section].get(option, None, seen, last=last)
options = self
else:
self.buildout._initializing.append(self)
try:
options = self.buildout[section]
finally:
del self.buildout._initializing[-1]
v = options.get(option, None, seen, last=last)
if v is None:
if option == '_buildout_section_name_':
v = self.name
......
......@@ -2544,7 +2544,7 @@ were created.
The ``.installed.cfg`` is only updated for the recipes that ran::
>>> cat(sample_buildout, '.installed.cfg')
... # doctest: +NORMALIZE_WHITESPACE
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
parts = debug d1 d2 d3 d4
......@@ -2574,7 +2574,7 @@ The ``.installed.cfg`` is only updated for the recipes that ran::
<BLANKLINE>
[d4]
__buildout_installed__ = /sample-buildout/data2-extra
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg== d2:...
path = /sample-buildout/data2-extra
recipe = recipes:mkdir
......@@ -2587,6 +2587,7 @@ Now, if we run the buildout without the install command::
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Uninstalling d4.
Uninstalling d2.
Uninstalling d1.
Uninstalling debug.
......@@ -2596,7 +2597,8 @@ Now, if we run the buildout without the install command::
Installing d2.
d2: Creating directory data2
Updating d3.
Updating d4.
Installing d4.
d4: Creating directory data2-extra
  • Due to a bug in manuel egg, we had no test results for doctests for a long time and we missed the fact that a more recent commit reverted to the old behaviour. Could it be dffdeffc?

    However, the behaviour of buildout at this point is meaningless. Buildout was previously invoked with install d3 d4 command-line arguments and I consider that d2 should have been reinstalled, because a section shall not access the option of another section if the latter is not properly installed.

    Does anyone use the install command in a way that fixing it is important?

    But we should at least revert this change so that this doctest continues.

    /cc @jerome

  • Sorry I did not reply ... I don't have much to say. I don't think we rely on invoking buildout with "install part1 part2" at all, but my opinion is that we should try to keep buildout behavior close to the original.

  • In !21 (merged) there are changes to revert the test to the original behavior

Please register or sign in to reply
We see the output of the debug recipe, and that ``data2`` was created. We
also see that ``d1`` and ``d2`` have gone away::
......
......@@ -179,18 +179,14 @@ def wait_until(label, func, *args, **kw):
time.sleep(0.01)
raise ValueError('Timed out waiting for: '+label)
class TestOptions(zc.buildout.buildout.Options):
def initialize(self):
pass
class Buildout(zc.buildout.buildout.Buildout):
def __init__(self):
zc.buildout.buildout.Buildout.__init__(
self, '', [('buildout', 'directory', os.getcwd())])
Options = TestOptions
def initialize(self, *args):
pass
def buildoutSetUp(test):
......
......@@ -2089,6 +2089,80 @@ def dealing_with_extremely_insane_dependencies():
Error: Couldn't find a distribution for 'pack5'.
"""
def test_part_pulled_by_recipe():
"""
>>> mkdir(sample_buildout, 'recipes')
>>> write(sample_buildout, 'recipes', 'test.py',
... '''
... class Recipe:
...
... def __init__(self, buildout, name, options):
... self.x = buildout[options['x']][name]
...
... def install(self):
... print self.x
... return ()
...
... update = install
... ''')
>>> write(sample_buildout, 'recipes', 'setup.py',
... '''
... from setuptools import setup
... setup(
... name = "recipes",
... entry_points = {'zc.buildout': ['test = test:Recipe']},
... )
... ''')
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = a
... [a]
... recipe = recipes:test
... x = b
... [b]
... <= a
... a = A
... b = B
... c = ${c:x}
... [c]
... x = c
... ''')
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Installing b.
B
Installing a.
A
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Updating b.
B
Updating a.
A
>>> cat('.installed.cfg') # doctest: +ELLIPSIS
[buildout]
...
[b]
__buildout_installed__ =
__buildout_signature__ = recipes-... c:...
...
[a]
__buildout_installed__ =
__buildout_signature__ = recipes-... b:...
...
"""
def read_find_links_to_load_extensions():
r"""
We'll create a wacky buildout extension that just announces itself when used:
......
......@@ -97,14 +97,14 @@ of extra requirements to be included in the working set.
We can see that the options were augmented with additional data
computed by the egg recipe by looking at .installed.cfg:
>>> cat(sample_buildout, '.installed.cfg')
>>> cat(sample_buildout, '.installed.cfg') # doctest: +ELLIPSIS
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link
parts = sample-part
<BLANKLINE>
[sample-part]
__buildout_installed__ =
__buildout_signature__ = ...
__buildout_signature__ = sample-... setuptools-...egg zc.buildout-... zc.recipe.egg-...
  • I don't see how setuptools signature can be setuptools-...egg here, it's always the egg name and the egg version. Do we have a test result where this test was passing ? In my development branch I changed this to remove the egg prefix here and test pass.

  • this is changed in !21 (merged)

Please register or sign in to reply
_b = /sample-buildout/bin
_d = /sample-buildout/develop-eggs
_e = /sample-buildout/eggs
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment