Simplify installation of recipe installing scripts

Several software use `slapos` command and expect this command to have networkcache support enabled, which is achieved in buildout by having `slapos.libnetworkcache` in the egg "working set" when generating the slapos script.

Introduce a dedicated section to install `slapos` command and use it in softwares which includes a slapos, testnode and slaprunner. Test softwares like slapos-sr-testing also need a slapos command, but they generate one with development version of slapos.core.

Before these changes, several software were installing `slapos` indirectly by installing `extra-eggs` from monitoring stack. The monitoring stack was also modified so that it uses a section with a prefix (`monitor-eggs`) and users of monitoring stacks were also modified to not use this `extra-eggs` part from monitoring stack, but install eggs through their own section.

Similar problem also existed with `slapos.toolbox`, which is installed by several softwares and stacks, for different reasons (because it contains lots of utilities), now that buildout detects parts installing the same path, only one parts can install the scripts. For this, define a section `slapos-toolbox` that software and stack should install, instead of listing the egg as `eggs` in a `zc.recipe.egg` section, because that would install the scripts again.

See merge request !729

software/apache-frontend/common.cfg

@@ -38,9 +38,9 @@ parts +=
   proxy-by-url
   http-proxy
 
-# Extent extra-eggs.
 [extra-eggs]
-eggs +=
+recipe = zc.recipe.egg
+eggs =
   websockify
   erp5.util
 

software/caddy-frontend/common.cfg

@@ -73,12 +73,7 @@ depends = ${caddyprofiledeps-develop:recipe}
 recipe = zc.recipe.egg
 eggs =
   caddyprofiledeps
-
-# Extent extra-eggs.
-[extra-eggs]
-eggs +=
   websockify
-  erp5.util
   collective.recipe.shelloutput
 
 [template-common]

software/erp5testnode/software.cfg

@@ -14,9 +14,9 @@ extends =
   ../../stack/monitor/buildout.cfg
 
 parts =
+  slapos-command
   slapos-cookbook
   template
-  lxml-python
   eggs
   zip
   git
@@ -25,25 +25,8 @@ parts =
 [eggs]
 recipe = zc.recipe.egg
 eggs =
+  erp5.util[testnode]
   ${lxml-python:egg}
-  zc.buildout
-  slapos.recipe.template
-  supervisor
-  jsonschema
-  hexagonit.recipe.download
-  lock_file
-  pytz
-  erp5.util
-  PyXML
-  z3c.etestbrowser
-
-[testnode]
-
-scripts =
-  testnode = erp5.util.testnode:main
-  slapgrid-cp = slapos.grid.slapgrid:runComputerPartition
-  slapgrid-sr = slapos.grid.slapgrid:runSoftwareRelease
-  slapproxy = slapos.proxy:main
 
 
 [template]

software/gateone/software.cfg

@@ -15,7 +15,7 @@ extends =
   ../../component/nginx/buildout.cfg
   ../../stack/slapos.cfg
 
-parts = 
+parts =
   slapos-cookbook
   kerberos
   eggs
@@ -24,7 +24,7 @@ parts =
   dash
   check-recipe
   template
-  
+
 
 [eggs]
 recipe = zc.recipe.egg
@@ -91,7 +91,6 @@ extra-context =
     key template_logrotate_base template-logrotate-base:rendered
     key template_gateone template-gateone:target
     raw gateone_bin ${buildout:bin-directory}/gateone
-    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
 
 [template-gateone]
 < = download-base

software/kvm/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = cad7d490b84a484b089e34b27a5675ff
+md5sum = 763f66b915839a9b073fb93f43a44493
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2

software/kvm/instance.cfg.in

@@ -26,10 +26,10 @@ RootSoftwareInstance = $${:default}
 
 # XXX - If this configuration is not generated by slapgrid, use empty values
 [storage-configuration]
-storage-home = 
-  
+storage-home =
+
 [network-information]
-global-ipv4-network = 
+global-ipv4-network =
 
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
@@ -87,7 +87,7 @@ extra-context =
   raw novnc_location ${noVNC:location}
   raw netcat_bin ${netcat:location}/bin/netcat
   raw python_executable ${buildout:executable}
-  raw python_eggs_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
+  raw python_eggs_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
   raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
   raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
   raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}

software/kvm/software.cfg

@@ -8,6 +8,7 @@ extends =
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/netcat/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../stack/slapos.cfg
@@ -40,9 +41,12 @@ parts = ${:common-parts}
 
 #XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu
 
-[extra-eggs]
-eggs +=
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+eggs =
   ${python-cffi:egg}
+  ${lxml-python:egg}
   websockify
   slapos.cookbook
   erp5.util

software/nginx-push-stream/software.cfg

 [buildout]
-extends = 
+extends =
   ../../stack/slapos.cfg
   ../../component/dash/buildout.cfg
   ../../component/nginx/buildout.cfg
 
 parts =
   slapos-cookbook
+  slapos-toolbox
   nginx-push-stream-module
   nginx-push-stream
   template
   template-nginx-service
   template-nginx
 
-[slapos-cookbook]
-eggs +=
-  slapos.toolbox
-
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = ec70348dd71b319590a5c5837f3d2e45
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = dc3a7734b08977f438b35d441a12f628
+md5sum = f342daca2e87f8fa44fd7b6a2a1c22f8
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -811,7 +811,7 @@ mode = 0744
 context =
     section supervisord supervisord
     section slaprunner slaprunner
-    raw python_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
+    raw python_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
 
 [supervisord-wrapper]
 recipe = slapos.cookbook:wrapper

software/slaprunner/software-py3.cfg

@@ -9,7 +9,7 @@ python = python3
 common-parts -=
   rdiff-backup
 
-[extra-eggs]
+[python-with-eggs]
 eggs -=
 # futures is a backport of Py3's concurrent.futures module
   futures

software/slaprunner/software.cfg

@@ -37,6 +37,7 @@ shared-part-list =
 # to avoid versioning issues
 
 common-parts =
+  slapos-command
   slapos-cookbook
   template
   instance-runner-import
@@ -50,6 +51,7 @@ common-parts =
 parts =
   ${:common-parts}
 
+
 [template-base]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}
@@ -141,8 +143,11 @@ filename = slapos-slapuser-script.in
 < = template-base
 output = ${buildout:directory}/buildout-shared-part-list
 
-[extra-eggs]
-eggs +=
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+eggs =
+  ${slapos-toolbox:eggs}
   erp5.util
   lock-file
   slapos.recipe.build
@@ -154,6 +159,11 @@ eggs +=
 # BBB: eggs used as recipe should be kept otherwise sections depending
 # on it can't be uninstalled
   collective.recipe.shelloutput
+scripts =
+  gunicorn
+  supervisord
+  supervisorctl
+
 
 [versions]
 Flask-Auth = 0.85

stack/monitor/buildout.cfg

@@ -7,13 +7,13 @@ extends =
   ../../component/dash/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/lxml-python/buildout.cfg
-  ../../component/pycurl/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
   ../../stack/logrotate/buildout.cfg
+  ../../stack/slapos.cfg
 
 parts =
   slapos-cookbook
-  extra-eggs
+  monitor-eggs
   monitor2-template
 
 [monitor-download-base]
@@ -30,25 +30,17 @@ destination = ${buildout:parts-directory}/monitor-scripts
 on-update = true
 
 [monitor-eggs]
-# XXX: merge with extra-eggs?
+recipe = zc.recipe.egg
+interpreter = monitor-pythonwitheggs
 eggs =
-  ${lxml-python:egg}
-  ${pycurl:egg}
-  ${python-cryptography:egg}
   plone.recipe.command
   collective.recipe.template
   cns.recipe.symlink
-  slapos.toolbox
-  slapos.core
-
-[extra-eggs]
-recipe = zc.recipe.egg
-interpreter = pythonwitheggs
-eggs =
-  ${monitor-eggs:eggs}
   psutil
   PyRSS2Gen
   Jinja2
+depends =
+  ${slapos-toolbox:recipe}
 
 
 # Monitor templates files
@@ -82,12 +74,12 @@ rendered = ${buildout:directory}/template-monitor.cfg
 context =
     key apache_location apache:location
     key template_logrotate_base template-logrotate-base:rendered
-    raw monitor_bin ${buildout:directory}/bin/monitor.bootstrap
-    raw monitor_collect ${buildout:directory}/bin/monitor.collect
-    raw monitor_statistic ${buildout:directory}/bin/monitor.statistic
-    raw monitor_runpromise ${buildout:directory}/bin/monitor.runpromise
-    raw monitor_genstatus ${buildout:directory}/bin/monitor.genstatus
-    raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite
+    raw monitor_bin ${buildout:bin-directory}/monitor.bootstrap
+    raw monitor_collect ${buildout:bin-directory}/monitor.collect
+    raw monitor_statistic ${buildout:bin-directory}/monitor.statistic
+    raw monitor_runpromise ${buildout:bin-directory}/monitor.runpromise
+    raw monitor_genstatus ${buildout:bin-directory}/monitor.genstatus
+    raw monitor_configwrite ${buildout:bin-directory}/monitor.configwrite
     raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
     raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
     raw curl_executable_location ${curl:location}/bin/curl
@@ -97,9 +89,9 @@ context =
     raw monitor_httpd_template ${monitor-httpd-conf:location}/${monitor-httpd-conf:filename}
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
-    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
+    raw python_with_eggs ${buildout:bin-directory}/${monitor-eggs:interpreter}
     raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
-    raw check_disk_space ${buildout:directory}/bin/check-free-disk
+    raw check_disk_space ${buildout:bin-directory}/check-free-disk
     raw bin_directory ${buildout:directory}/bin
 
 [versions]

stack/slapos.cfg

@@ -30,6 +30,8 @@ extends =
   ../component/python-cachecontrol/buildout.cfg
   ../component/python-cryptography/buildout.cfg
   ../component/python-PyYAML/buildout.cfg
+  ../component/pycurl/buildout.cfg
+
 
 # Separate from site eggs
 allowed-eggs-from-site-packages =
@@ -97,6 +99,22 @@ eggs =
   ${python-cryptography:egg}
   pyOpenSSL
   slapos.cookbook
+
+# slapos.toolbox containing utilities
+[slapos-toolbox]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${pycurl:egg}
+  ${python-cryptography:egg}
+  slapos.toolbox
+
+# Install a slapos command with networkcache enabled in ${buildout:bin-directory}
+[slapos-command]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.core
   slapos.libnetworkcache
 
 [versions]