Commit b04a4fdf authored by Jérome Perrin's avatar Jérome Perrin

switch_softwaretype: run software type buildout in same process

This way buildout can reuse egg caches and it's a bit faster:

To run a simple instance buildout, from 2.837s it goes down to 1.875s.
To run slapos node instance 10 times just after requesting an ERP5 instance, it goes from ~112s to 98s

before:

    hyperfine "/srv/slapgrid/slappart4/srv/slapos/inst/slappart0/tmp/shared/python2.7/60364a13cc977dd5a894e0239ac889b9/bin/python2.7 /srv/slapgrid/slappart4/srv/slapos/inst/slappart0/tmp/soft/c63ba7265399450b28f9ea6d5667a5e7/bin/buildout -U"
    Benchmark #1: /srv/slapgrid/slappart4/srv/slapos/inst/slappart0/tmp/shared/python2.7/60364a13cc977dd5a894e0239ac889b9/bin/python2.7 /srv/slapgrid/slappart4/srv/slapos/inst/slappart0/tmp/soft/c63ba7265399450b28f9ea6d5667a5e7/bin/buildout -U
      Time (mean ± σ):      2.837 s ±  0.275 s    [User: 2.481 s, System: 0.285 s]
      Range (min … max):    2.482 s …  3.222 s    10 runs

after:

    hyperfine "/srv/slapgrid/slappart4/srv/slapos/inst/slappart0/tmp/shared/python2.7/60364a13cc977dd5a894e0239ac889b9/bin/python2.7 /srv/slapgrid/slappart4/srv/slapos/inst/slappart0/tmp/soft/c63ba7265399450b28f9ea6d5667a5e7/bin/buildout -U"
    Benchmark #1: /srv/slapgrid/slappart4/srv/slapos/inst/slappart0/tmp/shared/python2.7/60364a13cc977dd5a894e0239ac889b9/bin/python2.7 /srv/slapgrid/slappart4/srv/slapos/inst/slappart0/tmp/soft/c63ba7265399450b28f9ea6d5667a5e7/bin/buildout -U
      Time (mean ± σ):      1.875 s ±  0.067 s    [User: 1.660 s, System: 0.148 s]
      Range (min … max):    1.816 s …  2.038 s    10 runs
parent 63f1a49f
Pipeline #12021 failed with stage
...@@ -25,8 +25,45 @@ ...@@ -25,8 +25,45 @@
# #
############################################################################## ##############################################################################
import os, subprocess, sys
import six from zc.buildout.buildout import Buildout
class SubBuildout(Buildout):
"""Run buildout in buildout, partially copied from infrae.buildout
"""
def __init__(self, main_buildout, config, options, **kwargs):
# Use same logger
self._logger = main_buildout._logger
self._log_level = main_buildout._log_level
# Use same options
for opt in (
'offline',
'verbosity',
'newest',
'directory',
'eggs-directory',
'develop-eggs-directory',
):
if opt in main_buildout['buildout']:
options.append((
'buildout',
opt,
main_buildout['buildout'][opt],
))
# Use same slap connection
for k, v in main_buildout["slap-connection"].items():
options.append(('slap-connection', k, v))
Buildout.__init__(self, config, options, **kwargs)
def _setup_logging(self):
"""We don't want to setup any logging, since it's already done
by the main buildout.
"""
pass
class Recipe: class Recipe:
...@@ -39,26 +76,23 @@ class Recipe: ...@@ -39,26 +76,23 @@ class Recipe:
self.base = self.buildout[section][key] self.base = self.buildout[section][key]
def install(self): def install(self):
# XXX-Antoine: We gotta find a better way to do this. I tried to check options = [("buildout", "installed", ".installed-%s.cfg" % self.name)]
# out how slapgrid-cp was running buildout. But it is worse than that. profile = self.base
args = sys.argv[:]
for x in six.iteritems(self.buildout["slap-connection"]):
args.append("slap-connection:%s=%s" % x)
for x in "directory", "eggs-directory", "develop-eggs-directory":
args.append("buildout:%s=%s" % (x, self.buildout["buildout"][x]))
args.append("buildout:installed=.installed-%s.cfg" % self.name)
# Options.get (from zc.buildout) should deserialize.
try: try:
override = self.options["override"][self.software_type] # XXX this assume using slapos.buildout, which serializes arbitrary python objects for options
extended_profile = self.options["override"][self.software_type]
except (KeyError, TypeError): except (KeyError, TypeError):
buildout = self.base pass
else: else:
# unfortunately, buildout:extends does not work when given at command line options.append(["buildout", "extends", profile])
buildout = os.path.join(self.buildout["buildout"]["parts-directory"], profile = extended_profile
self.name + ".cfg")
with open(override) as src, open(buildout, "w", 0) as dst: sub_buildout = SubBuildout(
dst.write("[buildout]\nextends = %s\n\n" % self.base + src.read()) self.buildout,
subprocess.check_call(args + ["-oc", buildout]) profile,
return [] options,
)
sub_buildout.install([])
update = install update = install
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment