Commit 8be81fe4 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Julien Muchembled

Add referred parts' hash strings in __buildout_signature__, that invokes...

Add referred parts' hash strings in __buildout_signature__, that invokes rebuild of a part when one of its (recursive) dependencies are modified.

Also remove duplicates and sort entries in __buildout_signature__.
parent b096b83a
...@@ -310,6 +310,7 @@ class Buildout(DictMixin): ...@@ -310,6 +310,7 @@ class Buildout(DictMixin):
self._raw = _unannotate(data) self._raw = _unannotate(data)
self._data = {} self._data = {}
self._parts = [] self._parts = []
self._initializing = []
# provide some defaults before options are parsed # provide some defaults before options are parsed
# because while parsing options those attributes might be # because while parsing options those attributes might be
...@@ -839,14 +840,29 @@ class Buildout(DictMixin): ...@@ -839,14 +840,29 @@ class Buildout(DictMixin):
"Unexpected entry, %r, in develop-eggs directory.", f) "Unexpected entry, %r, in develop-eggs directory.", f)
def _compute_part_signatures(self, parts): def _compute_part_signatures(self, parts):
# The same recipe may appear many times.
sig_cache= {}
# Compute recipe signature and add to options # Compute recipe signature and add to options
for part in parts: for part in parts:
options = self.get(part) options = self.get(part)
if options is None: if options is None:
options = self[part] = {} options = self[part] = {}
recipe, entry = _recipe(options) recipe, entry = _recipe(options)
try:
# Copy cached list, because sig.append is called later
sig = sig_cache[recipe][:]
except KeyError:
req = pkg_resources.Requirement.parse(recipe) req = pkg_resources.Requirement.parse(recipe)
sig = _dists_sig(pkg_resources.working_set.resolve([req])) sig_result = sig_cache[recipe] = sorted(set(_dists_sig(pkg_resources.working_set.resolve([req]))))
sig = sig_result[:]
for dependency in sorted(options.depends):
m = md5()
for item in sorted(self[dependency].items()):
m.update(('%r\0%r\0' % item).encode())
sig.append('%s:%s' % (dependency, m.hexdigest()))
options['__buildout_signature__'] = ' '.join(sig) options['__buildout_signature__'] = ' '.join(sig)
def _read_installed_part_options(self): def _read_installed_part_options(self):
...@@ -1178,13 +1194,19 @@ class Buildout(DictMixin): ...@@ -1178,13 +1194,19 @@ class Buildout(DictMixin):
v = ' '+v v = ' '+v
print_("%s =%s" % (k, v)) print_("%s =%s" % (k, v))
def initialize(self, options, reqs, entry):
recipe_class = _install_and_load(reqs, 'zc.buildout', entry, self)
self._initializing.append(options)
try:
return recipe_class(self, options.name, options)
finally:
del self._initializing[-1]
def __getitem__(self, section): def __getitem__(self, section):
__doing__ = 'Getting section %s.', section __doing__ = 'Getting section %s.', section
try: try:
return self._data[section] options = self._data[section]
except KeyError: except KeyError:
pass
try: try:
data = self._raw[section] data = self._raw[section]
except KeyError: except KeyError:
...@@ -1193,6 +1215,11 @@ class Buildout(DictMixin): ...@@ -1193,6 +1215,11 @@ class Buildout(DictMixin):
options = self.Options(self, section, data) options = self.Options(self, section, data)
self._data[section] = options self._data[section] = options
options._initialize() options._initialize()
if self._initializing:
caller = self._initializing[-1]
if 'buildout' != section != caller.name:
caller.depends.add(section)
return options return options
def __setitem__(self, name, data): def __setitem__(self, name, data):
...@@ -1275,6 +1302,7 @@ class Options(DictMixin): ...@@ -1275,6 +1302,7 @@ class Options(DictMixin):
self._raw = data self._raw = data
self._cooked = {} self._cooked = {}
self._data = {} self._data = {}
self.depends = set()
def _initialize(self): def _initialize(self):
name = self.name name = self.name
...@@ -1292,17 +1320,9 @@ class Options(DictMixin): ...@@ -1292,17 +1320,9 @@ class Options(DictMixin):
return # buildout section can never be a part return # buildout section can never be a part
if self.get('recipe'): if self.get('recipe'):
self.initialize() self.recipe = self.buildout.initialize(self, *_recipe(self._data))
self.buildout._parts.append(name) self.buildout._parts.append(name)
def initialize(self):
reqs, entry = _recipe(self._data)
buildout = self.buildout
recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout)
name = self.name
self.recipe = recipe_class(buildout, name, self)
def _do_extend_raw(self, name, data, doing): def _do_extend_raw(self, name, data, doing):
if name == 'buildout': if name == 'buildout':
return data return data
...@@ -1409,7 +1429,14 @@ class Options(DictMixin): ...@@ -1409,7 +1429,14 @@ class Options(DictMixin):
section, option = s section, option = s
if not section: if not section:
section = self.name section = self.name
v = self.buildout[section].get(option, None, seen, last=last) options = self
else:
self.buildout._initializing.append(self)
try:
options = self.buildout[section]
finally:
del self.buildout._initializing[-1]
v = options.get(option, None, seen, last=last)
if v is None: if v is None:
if option == '_buildout_section_name_': if option == '_buildout_section_name_':
v = self.name v = self.name
......
...@@ -2544,7 +2544,7 @@ were created. ...@@ -2544,7 +2544,7 @@ were created.
The ``.installed.cfg`` is only updated for the recipes that ran:: The ``.installed.cfg`` is only updated for the recipes that ran::
>>> cat(sample_buildout, '.installed.cfg') >>> cat(sample_buildout, '.installed.cfg')
... # doctest: +NORMALIZE_WHITESPACE ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
[buildout] [buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
parts = debug d1 d2 d3 d4 parts = debug d1 d2 d3 d4
...@@ -2574,7 +2574,7 @@ The ``.installed.cfg`` is only updated for the recipes that ran:: ...@@ -2574,7 +2574,7 @@ The ``.installed.cfg`` is only updated for the recipes that ran::
<BLANKLINE> <BLANKLINE>
[d4] [d4]
__buildout_installed__ = /sample-buildout/data2-extra __buildout_installed__ = /sample-buildout/data2-extra
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg== __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg== d2:...
path = /sample-buildout/data2-extra path = /sample-buildout/data2-extra
recipe = recipes:mkdir recipe = recipes:mkdir
...@@ -2587,6 +2587,7 @@ Now, if we run the buildout without the install command:: ...@@ -2587,6 +2587,7 @@ Now, if we run the buildout without the install command::
>>> print_(system(buildout), end='') >>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes' Develop: '/sample-buildout/recipes'
Uninstalling d4.
Uninstalling d2. Uninstalling d2.
Uninstalling d1. Uninstalling d1.
Uninstalling debug. Uninstalling debug.
...@@ -2596,7 +2597,8 @@ Now, if we run the buildout without the install command:: ...@@ -2596,7 +2597,8 @@ Now, if we run the buildout without the install command::
Installing d2. Installing d2.
d2: Creating directory data2 d2: Creating directory data2
Updating d3. Updating d3.
Updating d4. Installing d4.
d4: Creating directory data2-extra
  • Due to a bug in manuel egg, we had no test results for doctests for a long time and we missed the fact that a more recent commit reverted to the old behaviour. Could it be dffdeffc?

    However, the behaviour of buildout at this point is meaningless. Buildout was previously invoked with install d3 d4 command-line arguments and I consider that d2 should have been reinstalled, because a section shall not access the option of another section if the latter is not properly installed.

    Does anyone use the install command in a way that fixing it is important?

    But we should at least revert this change so that this doctest continues.

    /cc @jerome

  • Sorry I did not reply ... I don't have much to say. I don't think we rely on invoking buildout with "install part1 part2" at all, but my opinion is that we should try to keep buildout behavior close to the original.

  • In !21 (merged) there are changes to revert the test to the original behavior

Please register or sign in to reply
We see the output of the debug recipe, and that ``data2`` was created. We We see the output of the debug recipe, and that ``data2`` was created. We
also see that ``d1`` and ``d2`` have gone away:: also see that ``d1`` and ``d2`` have gone away::
......
...@@ -179,18 +179,14 @@ def wait_until(label, func, *args, **kw): ...@@ -179,18 +179,14 @@ def wait_until(label, func, *args, **kw):
time.sleep(0.01) time.sleep(0.01)
raise ValueError('Timed out waiting for: '+label) raise ValueError('Timed out waiting for: '+label)
class TestOptions(zc.buildout.buildout.Options):
def initialize(self):
pass
class Buildout(zc.buildout.buildout.Buildout): class Buildout(zc.buildout.buildout.Buildout):
def __init__(self): def __init__(self):
zc.buildout.buildout.Buildout.__init__( zc.buildout.buildout.Buildout.__init__(
self, '', [('buildout', 'directory', os.getcwd())]) self, '', [('buildout', 'directory', os.getcwd())])
Options = TestOptions def initialize(self, *args):
pass
def buildoutSetUp(test): def buildoutSetUp(test):
......
...@@ -2089,6 +2089,80 @@ def dealing_with_extremely_insane_dependencies(): ...@@ -2089,6 +2089,80 @@ def dealing_with_extremely_insane_dependencies():
Error: Couldn't find a distribution for 'pack5'. Error: Couldn't find a distribution for 'pack5'.
""" """
def test_part_pulled_by_recipe():
"""
>>> mkdir(sample_buildout, 'recipes')
>>> write(sample_buildout, 'recipes', 'test.py',
... '''
... class Recipe:
...
... def __init__(self, buildout, name, options):
... self.x = buildout[options['x']][name]
...
... def install(self):
... print self.x
... return ()
...
... update = install
... ''')
>>> write(sample_buildout, 'recipes', 'setup.py',
... '''
... from setuptools import setup
... setup(
... name = "recipes",
... entry_points = {'zc.buildout': ['test = test:Recipe']},
... )
... ''')
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = a
... [a]
... recipe = recipes:test
... x = b
... [b]
... <= a
... a = A
... b = B
... c = ${c:x}
... [c]
... x = c
... ''')
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Installing b.
B
Installing a.
A
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Updating b.
B
Updating a.
A
>>> cat('.installed.cfg') # doctest: +ELLIPSIS
[buildout]
...
[b]
__buildout_installed__ =
__buildout_signature__ = recipes-... c:...
...
[a]
__buildout_installed__ =
__buildout_signature__ = recipes-... b:...
...
"""
def read_find_links_to_load_extensions(): def read_find_links_to_load_extensions():
r""" r"""
We'll create a wacky buildout extension that just announces itself when used: We'll create a wacky buildout extension that just announces itself when used:
......
...@@ -97,14 +97,14 @@ of extra requirements to be included in the working set. ...@@ -97,14 +97,14 @@ of extra requirements to be included in the working set.
We can see that the options were augmented with additional data We can see that the options were augmented with additional data
computed by the egg recipe by looking at .installed.cfg: computed by the egg recipe by looking at .installed.cfg:
>>> cat(sample_buildout, '.installed.cfg') >>> cat(sample_buildout, '.installed.cfg') # doctest: +ELLIPSIS
[buildout] [buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link
parts = sample-part parts = sample-part
<BLANKLINE> <BLANKLINE>
[sample-part] [sample-part]
__buildout_installed__ = __buildout_installed__ =
__buildout_signature__ = ... __buildout_signature__ = sample-... setuptools-...egg zc.buildout-... zc.recipe.egg-...
  • I don't see how setuptools signature can be setuptools-...egg here, it's always the egg name and the egg version. Do we have a test result where this test was passing ? In my development branch I changed this to remove the egg prefix here and test pass.

  • this is changed in !21 (merged)

Please register or sign in to reply
_b = /sample-buildout/bin _b = /sample-buildout/bin
_d = /sample-buildout/develop-eggs _d = /sample-buildout/develop-eggs
_e = /sample-buildout/eggs _e = /sample-buildout/eggs
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment