Update Release Candidate

component/chromedriver/buildout.cfg

 # ChromeDriver - Webdriver for Chrome
-# http://chromedriver.chromium.org/
+# https://chromedriver.chromium.org/
 # This is a binary download with wrapper scripts.
 
 [buildout]
@@ -13,6 +13,10 @@ parts =
   chromedriver-wrapper
 
 
+[chromedriver-wrapper-91]
+<= chromedriver-wrapper
+wrapper-name = chromedriver-91
+part = ${chromedriver-91:location}
 
 [chromedriver-wrapper-2.41]
 <= chromedriver-wrapper
@@ -46,7 +50,7 @@ install =
 
 
 [chromedriver]
-<= chromedriver-2.41
+<= chromedriver-91
 
 [chromedriver-2.41]
 <= chromedriver-download
@@ -54,6 +58,11 @@ version = 2.41
 # Supports Chrome v67-69
 md5sum-x86_64 = fbd8b9561575054e0e7e9cc53b680a70
 
+[chromedriver-91]
+<= chromedriver-download
+version = 91.0.4472.101
+# Supports Chrome v91
+md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
 
 [chromedriver-download]
 # Installs chromedriver ${version}.

component/chromium/buildout.cfg

@@ -66,16 +66,27 @@ install =
   ))
     os.fchmod(f.fileno(), 0o755)
 
+[chromium-wrapper-91]
+<= chromium-wrapper
+wrapper-name = chromium-91
+part = ${chromium-91:location}
+
 [chromium-wrapper-69]
 <= chromium-wrapper
 wrapper-name = chromium-69
 part = ${chromium-69:location}
 
 
+[chromium]
+<= chromium-91
 
+[chromium-91]
+<= chromium-download
+version = 91.0.4472.114
 
-[chromium]
-<= chromium-69
+revision_x86-64 = 870763
+md5sum-x86_64 = 74eab41580469c2b8117cf396db823cb
+generation-x86_64 = 1617926496067901
 
 
 [chromium-69]
@@ -111,6 +122,7 @@ x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snaps
 library =
   ${atk:location}/lib
   ${at-spi2-atk:location}/lib
+  ${at-spi2-core:location}/lib
   ${alsa:location}/lib
   ${cairo:location}/lib
   ${cups:location}/lib
@@ -133,11 +145,14 @@ library =
   ${libXtst:location}/lib
   ${libXScrnSaver:location}/lib
   ${libXrandr:location}/lib
+  ${libdrm:location}/lib
   ${libexpat:location}/lib
   ${libffi:location}/lib
   ${libpng:location}/lib
   ${libpng12:location}/lib
   ${libxcb:location}/lib
+  ${libxkbcommon:location}/lib
+  ${libxshmfence:location}/lib
   ${libxml2:location}/lib
   ${mesa:location}/lib
   ${nspr:location}/lib

component/firefox/buildout.cfg

@@ -51,6 +51,11 @@ install =
   ))
     os.fchmod(f.fileno(), 0o755)
 
+[firefox-wrapper-78]
+<= firefox-wrapper
+wrapper-name = firefox-78
+part = ${firefox-78:location}
+
 [firefox-wrapper-68]
 <= firefox-wrapper
 wrapper-name = firefox-68
@@ -95,6 +100,12 @@ cache-dir =
 # would not be created.
 <= firefox-68
 
+[firefox-78]
+<= firefox-download
+version = 78.1.0esr
+i686-md5sum = 09595a1b9a99d17a618a51bc1f971e5e
+x86_64-md5sum = 06f4d488721ce7229d9a86cb4c6786f3
+
 [firefox-68]
 <= firefox-download
 version = 68.0.2esr

component/mariadb/mdev20693-mixin.cfg

 # To be extended after mariadb's buildout.cfg
 
-[mariadb]
+[mariadb-10.4]
 patches +=
   ${:_profile_base_location_}/mdev20693.patch#34ca907d6b36ba81d75bed118243f637

component/mesa/buildout.cfg

@@ -9,6 +9,7 @@ extends =
   ../pkgconfig/buildout.cfg
   ../xorg/buildout.cfg
   ../xz-utils/buildout.cfg
+  ../zlib/buildout.cfg
 
 parts =
   mesa
@@ -16,32 +17,33 @@ parts =
 [mesa]
 recipe = slapos.recipe.cmmi
 shared = true
-url = ftp://ftp.freedesktop.org/pub/mesa/11.0.3/mesa-11.0.3.tar.xz
-md5sum = bf9118bf0fbf360715cfe60baf7a1db5
+url = https://archive.mesa3d.org/mesa-18.0.0.tar.xz
+md5sum = c2a59fc5b56de3e197fa3a6023409e23
 configure-options =
-  --enable-static
   --disable-gles1
   --disable-gles2
-  --disable-dri
+  --enable-dri
   --disable-dri3
   --enable-egl
-  --disable-gbm
+  --enable-gbm
+  --enable-sysfs
   --disable-xvmc
   --disable-vdpau
   --disable-va
-  --enable-xlib-glx
-  --disable-shared-glapi
+  --disable-xlib-glx
+  --enable-shared-glapi
   --disable-driglx-direct
   --disable-gallium-llvm
   --with-gallium-drivers=
   --with-dri-drivers=
 environment =
   PATH=${autoconf:location}/bin:${bison:location}/bin:${flex:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${libdrm:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
   PYTHON2=${buildout:executable}
   ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
   AUTOCONF=${autoconf:location}/bin/autoconf
   AUTOMAKE=${automake:location}/bin/automake
+  LDFLAGS=-Wl,-rpath=${libdrm:location}/lib -Wl,-rpath=${zlib:location}/lib
 make-options =
   -j1
   LIBTOOL=${libtool:location}/bin/libtool
@@ -57,3 +59,4 @@ configure-options =
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${mesa:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
+  LDFLAGS=-Wl,-rpath=${libdrm:location}/lib -Wl,-rpath=${zlib:location}/lib

component/tempstorage/buildout.cfg

+# SlapOS component for tempstorage.
+# https://github.com/zopefoundation/tempstorage
+
+[buildout]
+extends =
+    ../ZODB/buildout.cfg
+    ../git/buildout.cfg
+
+
+# tempstorage provides tempstorage<X> depending on ZODB major version.
+#
+# - tempstorage >= 4 works only with ZODB5 because ZODB commit protocol was changed
+#   https://github.com/zopefoundation/tempstorage/commit/5cc223ea
+# - tempstorage <= 5.2 is vulnerable to data corruption in loadBefore
+#   https://github.com/zopefoundation/tempstorage/issues/8
+#   https://github.com/zopefoundation/tempstorage/pull/16
+# - tempstorage 3-nxd provides loadBefore backports to tempstorage 3.
+[tempstorage]
+recipe  = slapos.recipe.build
+depends = ${ZODB:egg}
+init =
+  # link/depend to tempstorage<ZODB.major>
+  zodb = self.buildout['ZODB']
+  zmajor = zodb['major']
+  tempstorage_x = self.buildout['tempstorage'+zmajor]
+  options['depends'] += '$${%s:egg}' % tempstorage_x.name
+  options['egg'] = tempstorage_x['egg']
+
+  # update [versions] from what is needed by tempstorage<X>
+  self.buildout.parse('[_tempstorage-versions]\n' + tempstorage_x['egg-versions'])
+  versions = self.buildout['versions']
+  versions.update(self.buildout['_tempstorage-versions'])
+  # propagate updated [versions] -> easy_install
+  # (buildout does this in Buildout constructor)
+  import zc.buildout.easy_install
+  zc.buildout.easy_install.default_versions(versions)
+
+
+# tempstorage5 is plain upstream egg
+[tempstorage5]
+recipe  = zc.recipe.egg:eggs
+egg     = tempstorage
+eggs    = ${:egg}
+
+egg-versions =
+  tempstorage = 5.2
+
+
+# tempstorage4-wc2 is tempstorage 3 + backports for loadBefore fixes
+[tempstorage4-wc2]
+recipe  = zc.recipe.egg:develop
+setup   = ${tempstorage4-wc2-repository:location}
+egg     = tempstorage
+egg-versions =
+  tempstorage =
+
+[tempstorage4-wc2-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/tempstorage.git
+branch  = 3-nxd
+revision= 77b49295db78
+location = ${buildout:parts-directory}/tempstorage
+git-executable = ${git:location}/bin/git
+
+
+# tempstorage4 is the same as tempstorage4-wc2 because of loadBefore fixes.
+[tempstorage4]
+<= tempstorage4-wc2

component/tesseract/buildout.hash.cfg

 [download-tessdata.py]
 filename = download-tessdata.py
-md5sum = 2d283a6d8662d6bb8c9de7b26162b702
+md5sum = 1f6cf2797f0daf9dec16c1d3a8c44907

component/tesseract/download-tessdata.py

@@ -19,9 +19,13 @@ def post_make_hook(options, buildout, env):
   for url in options['tessdata-urls'].splitlines():
     url, _, md5sum = url.partition('#')
     if url:
+      destination = os.path.join(
+          options['tessdata-location'],
+          os.path.basename(url),
+      )
       download(
           url,
           md5sum=md5sum,
-          path=os.path.join(options['tessdata-location'],
-                            os.path.basename(url)),
+          path=destination,
       )
+      os.chmod(destination, 0o750)

component/wendelin.core/buildout.cfg

@@ -38,6 +38,12 @@ GO   = ${go:exe}
 # wcfs needs this:
 [gowork]
 cpkgpath += ${zlib:location}/lib/pkgconfig
+[wendelin.core-env]
+# `pkg-config --libs zlib` emits only -L, but not -Wl,-rpath
+# better set it via gowork:environment when "VAR += ..." support is there
+# environment +=
+#   CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
+CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
 
 
 [wendelin.core-repository]

component/xorg/buildout.cfg

@@ -2,20 +2,25 @@
 extends =
   ../autoconf/buildout.cfg
   ../automake/buildout.cfg
+  ../bison/buildout.cfg
   ../bzip2/buildout.cfg
   ../dash/buildout.cfg
   ../freetype/buildout.cfg
   ../gnutls/buildout.cfg
+  ../icu/buildout.cfg
   ../intltool/buildout.cfg
   ../libtool/buildout.cfg
   ../libuuid/buildout.cfg
   ../libxml2/buildout.cfg
   ../libxslt/buildout.cfg
+  ../meson/buildout.cfg
+  ../ninja/buildout.cfg
   ../openssl/buildout.cfg
   ../patch/buildout.cfg
   ../perl/buildout.cfg
   ../perl-XML-Parser/buildout.cfg
   ../pkgconfig/buildout.cfg
+  ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
   ./buildout.hash.cfg
 
@@ -652,3 +657,43 @@ pkg_config_depends = ${libX11:location}/lib/pkgconfig:${xorgproto:pkg_config_dep
 environment =
   PKG_CONFIG_PATH=${:pkg_config_depends}
   PATH=${pkgconfig:location}/bin:%(PATH)s
+
+
+[libdrm]
+recipe = slapos.recipe.cmmi
+shared = true
+configure-command = ${meson:location}/bin/meson builddir --libdir=lib -Dprefix=@@LOCATION@@
+make-binary = ${ninja:location}/bin/ninja -C builddir
+url = https://dri.freedesktop.org/libdrm/libdrm-2.4.106.tar.xz
+md5sum = 4e316ae1966a1a63c31a3885313a8fb8
+pkg_config_depends = ${pciaccess:location}/lib/pkgconfig
+environment =
+  PKG_CONFIG_PATH=${:pkg_config_depends}
+  PATH=${xz-utils:location}/bin:${pkgconfig:location}/bin:${ninja:location}/bin:%(PATH)s
+  LDFLAGS=-Wl,-rpath=${pciaccess:location}/lib -Wl,-rpath=@@LOCATION@@/lib
+
+
+[libxshmfence]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://www.x.org/releases/individual/lib/libxshmfence-1.3.tar.gz
+md5sum = ab3940af0bd3d3cc91eb35ecd33a779a
+pkg_config_depends = ${xorgproto:pkg_config_depends}:${xorgproto:location}/share/pkgconfig
+environment =
+  PKG_CONFIG_PATH=${:pkg_config_depends}
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+
+
+[libxkbcommon]
+recipe = slapos.recipe.cmmi
+shared = true
+configure-command = ${meson:location}/bin/meson builddir --libdir=lib -Dprefix=@@LOCATION@@ -Denable-wayland=false -Denable-docs=false
+make-binary = ${ninja:location}/bin/ninja -C builddir
+
+url = https://xkbcommon.org/download/libxkbcommon-1.3.0.tar.xz
+md5sum = 00b5275ec1309a1d427a645de5861605
+pkg_config_depends = ${libxcb:pkg_config_depends}:${libxcb:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${icu4c:location}/lib/pkgconfig
+environment =
+  PKG_CONFIG_PATH=${:pkg_config_depends}
+  PATH=${xz-utils:location}/bin:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:%(PATH)s
+  LDFLAGS=-Wl,-rpath=${libxcb:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${icu4c:location}/lib -Wl,-rpath=@@LOCATION@@/lib

slapos/recipe/switch_softwaretype.py

@@ -25,9 +25,8 @@
 #
 ##############################################################################
 
-
-from zc.buildout.buildout import Buildout
-
+from zc.buildout.buildout import Buildout, MissingOption, MissingSection
+from zc.buildout import UserError
 
 class SubBuildout(Buildout):
   """Run buildout in buildout, partially copied from infrae.buildout
@@ -71,8 +70,21 @@ class Recipe:
     self.buildout = buildout
     self.options = options
     self.name = name
-    self.software_type = buildout["slap-configuration"]["slap-software-type"]
-    section, key = self.options[self.software_type].split(":")
+    try:
+      self.software_type = buildout["slap-configuration"]["slap-software-type"]
+    except (MissingSection, MissingOption):
+      raise UserError("The section to retrieve slap partition parameters "
+                      "(with slapos.cookbook:slapconfiguration recipe or a derived one) "
+                      "must be named [slap-configuration].")
+    try:
+      section, key = self.options[self.software_type].split(":")
+    except MissingOption:
+      raise MissingOption("This software type (%s) isn't mapped. RootSoftwareInstance "
+                      "is the default software type." % self.software_type)
+    except ValueError:
+      raise UserError("The software types in the section [%s] must be separated "
+                      "by a colon such as: 'section:key', where key is usually 'rendered'. "
+                      "Don't use: ${section:key}" % self.name)
     self.base = self.buildout[section][key]
 
   def install(self):

software/backupserver/buildout.hash.cfg

@@ -46,8 +46,8 @@ md5sum = ae4a0043414336a521b524d9c95f1c68
 
 [template-pullrdiffbackup]
 filename = instance-pullrdiffbackup.cfg.in
-md5sum = c1f00c251298c2ab7fd095e7c4571d3b
+md5sum = f2e6f30a0e8228cbfb93eaaae10fe884
 
 [template]
 filename = instance.cfg.in
-md5sum = 42021b325159dff29e4bd4e33b8ff2f3
+md5sum = 3df515def97f1e8a9f181514ae6ef03f

software/backupserver/instance-pullrdiffbackup.cfg.in

@@ -194,9 +194,9 @@ virtual-depends =
 
 [nginx-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = nginx_listen.py
-config-hostname = $${nginx-configuration:ip}
+config-host = $${nginx-configuration:ip}
 config-port = $${nginx-configuration:port}
 
 [nginx-configuration]
@@ -206,7 +206,7 @@ output = $${directory:etc}/nginx.cfg
 mode = 0600
 access_log = $${directory:log}/nginx-access.log
 error_log = $${directory:log}/nginx-error.log
-ip = $${slap-network-information:global-ipv6}
+ip = {{ partition_ipv6 }}
 port = 9443
 ssl_key = $${directory:ssl}/nginx.key
 ssl_csr = $${directory:ssl}/nginx.csr

software/backupserver/instance.cfg.in

@@ -13,32 +13,21 @@ rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filenam
 filename = instance-pullrdiffbackup.cfg
 extensions = jinja2.ext.do
 context =
-    key slave_instance_list instance-parameter:slave-instance-list
+    key slave_instance_list slap-configuration:slave-instance-list
+# partition_ipv6 is the random ipv6 allocated to the local partition
+    key partition_ipv6 slap-configuration:ipv6-random
 
 [switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = $${:pullrdiffbackup}
-# pullrdiffbackup = ${template-pullrdiffbackup:output}
-pullrdiffbackup = $${dynamic-template-pullrdiffbackup:rendered}
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = $${:pullrdiffbackup}
+pullrdiffbackup = dynamic-template-pullrdiffbackup:rendered
 
-[slap-connection]
-# part to migrate to new - separated words
-computer-id = $${slap_connection:computer_id}
-partition-id = $${slap_connection:partition_id}
-server-url = $${slap_connection:server_url}
-software-release-url = $${slap_connection:software_release_url}
-key-file = $${slap_connection:key_file}
-cert-file = $${slap_connection:cert_file}
-
-# [slap-parameter]
-# slave-instance-list = []
-
-[instance-parameter]
+[slap-configuration]
 # Fetches parameters defined in SlapOS Master for this instance.
 # Always the same.
 recipe = slapos.cookbook:slapconfiguration.serialised
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}

software/backupserver/software.cfg

@@ -102,5 +102,3 @@ mode = 0644
 
 [versions]
 gunicorn = 19.1.1
-plone.recipe.command = 1.1
-PyRSS2Gen = 1.1

software/caddy-frontend/buildout.hash.cfg

@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 28220d18308313d49a38d39c61a7e769
+md5sum = 8507a2ace2f789b92c522cc62ca5aace
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = ec421617d4932e8e1ae1e55a41fbfaab
+md5sum = 8beb438d06bbb0f917d13e182fb12d17
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = a745fb8d61a7e2646e3aa55edf73e5a6
+md5sum = 613f777a08373088cbaf7f51fd18ea70
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -106,7 +106,7 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
 
 [profile-kedifa]
 filename = instance-kedifa.cfg.in
-md5sum = 225ce18b9218de4169327a206051a92e
+md5sum = 16901e9eeb0d4f87e708ad91e7756f12
 
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -535,9 +535,9 @@ context =
 
 [trafficserver-promise-listen-port]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = trafficserver-port-listening.py
-config-hostname = ${trafficserver-variable:local-ip}
+config-host = ${trafficserver-variable:local-ip}
 config-port = ${trafficserver-variable:input-port}
 
 [trafficserver-ctl]
@@ -655,44 +655,44 @@ config-verification-script = ${promise-helper-last-configuration-state:rendered}
 
 [promise-caddy-frontend-v4-https]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = caddy_frontend_ipv4_https.py
-config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
+config-host = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${configuration:port}
 
 [promise-caddy-frontend-v4-http]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = caddy_frontend_ipv4_http.py
-config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
+config-host = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${configuration:plain_http_port}
 
 [promise-caddy-frontend-v6-https]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = caddy_frontend_ipv6_https.py
-config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
+config-host = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${configuration:port}
 
 [promise-caddy-frontend-v6-http]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = caddy_frontend_ipv6_http.py
-config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
+config-host = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${configuration:plain_http_port}
 
 [promise-backend-haproxy-http]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = backend_haproxy_http.py
-config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
+config-host = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${backend-haproxy-configuration:http-port}
 
 [promise-backend-haproxy-https]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = backend_haproxy_https.py
-config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
+config-host = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${backend-haproxy-configuration:https-port}
 
 [backend-haproxy-configuration]
@@ -981,9 +981,9 @@ context =
 
 [promise-slave-introspection-https]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = slave_introspection_https.py
-config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
+config-host = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${frontend-configuration:slave-introspection-https-port}
 
 [logrotate-entry-slave-introspection]

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -865,14 +865,14 @@ rendered = ${directory:etc}/nginx-rejected-slave.conf
 
 [promise-rejected-slave-publish-ip-port]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = rejected-slave-publish-ip-port-listening.py
-config-hostname = ${rejected-slave-publish-configuration:ip}
+config-host = ${rejected-slave-publish-configuration:ip}
 config-port = ${rejected-slave-publish-configuration:port}
 
 [rejected-slave-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 module = check_file_state
 name = rejected-slave.py
 config-filename = ${rejected-slave-json:rendered}

software/caddy-frontend/instance-kedifa.cfg.in

@@ -193,9 +193,9 @@ template = inline:
 
 [promise-expose-csr_id-ip-port]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = expose-csr_id-ip-port-listening.py
-config-hostname = ${expose-csr_id-configuration:ip}
+config-host = ${expose-csr_id-configuration:ip}
 config-port = ${expose-csr_id-configuration:port}
 
 [expose-csr_id]

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -633,9 +633,9 @@ template = inline:
 
 [promise-expose-csr_id-ip-port]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = expose-csr_id-ip-port-listening.py
-config-hostname = ${expose-csr_id-configuration:ip}
+config-host = ${expose-csr_id-configuration:ip}
 config-port = ${expose-csr_id-configuration:port}
 
 [expose-csr_id]

software/caddy-frontend/test/test.py

@@ -79,6 +79,8 @@ else:
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
+  # XXX Keep using slapos node instance --all, because of missing promises
+  SlapOSInstanceTestCase.slap._force_slapos_node_instance_all = True
 
 # ports chosen to not collide with test systems
 HTTP_PORT = '11080'

software/erp5/instance-erp5-input-schema.json

@@ -78,7 +78,7 @@
       "type": "string"
     },
     "developer-list": {
-      "description": "List of logins which should get the Developper role (required to modify portal_components' content), defaulting to inituser-login's value",
+      "description": "List of logins which should get the Developer role (required to modify portal_components' content), defaulting to inituser-login's value",
       "items": {
         "pattern": "^\\S+$",
         "type": "string"
@@ -215,7 +215,7 @@
               "type": "integer"
             },
             "timerserver-interval": {
-              "description": "Timerserver tick perdiod, in seconds, or 0 to disable",
+              "description": "Timerserver tick period, in seconds, or 0 to disable",
               "default": 5,
               "type": "number"
             },
@@ -439,7 +439,7 @@
                   "type": "string"
                 },
                 {
-                  "description": "Override value (parameter for maching nodes).",
+                  "description": "Override value (parameter for matching nodes).",
                   "type": [
                     "integer",
                     "string"
@@ -517,7 +517,7 @@
           "type": "boolean"
         },
         "node-count": {
-          "description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
+          "description": "Number of tests this instance can execute in parallel. This must be at least equal to the number of nodes configured on testnode running the test",
           "default": 3,
           "type": "integer"
         },
@@ -525,6 +525,101 @@
           "description": "Number of extra databases this instance tests will need.",
           "default": 3,
           "type": "integer"
+        },
+        "selenium": {
+          "default": {
+            "target": "firefox"
+          },
+          "examples": [
+            {
+              "target": "selenium-server",
+              "server-url": "https://selenium.example.com",
+              "desired-capabilities": {
+                "browserName": "firefox",
+                "version": "68.0.2esr",
+                "acceptInsecureCerts": true
+              }
+            },
+            {
+              "target": "selenium-server",
+              "server-url": "https://selenium.example.com",
+              "desired-capabilities": {
+                "browserName": "chrome",
+                "version": "91.0.4472.101"
+              }
+            }
+          ],
+          "oneOf": [
+            {
+              "type": "object",
+              "title": "Selenium Server",
+              "description": "Configuration for Selenium server",
+              "additionalProperties": false,
+              "required": [
+                "desired-capabilities",
+                "server-url",
+                "target"
+              ],
+              "properties": {
+                "target": {
+                  "description": "Target system",
+                  "type": "string",
+                  "const": "selenium-server"
+                },
+                "server-url": {
+                  "description": "URL of the selenium server",
+                  "type": "string",
+                  "format": "uri"
+                },
+                "verify-server-certificate": {
+                  "description": "Verify the SSL/TLS certificate of the selenium server when using HTTPS",
+                  "type": "boolean",
+                  "default": true
+                },
+                "server-ca-certificate": {
+                  "description": "PEM encoded bundle of CA certificates to verify the SSL/TLS certificate of the selenium server when using HTTPS",
+                  "type": "string",
+                  "default": "Root certificates from http://certifi.io/en/latest/"
+                },
+                "desired-capabilities": {
+                  "description": "Desired browser capabilities",
+                  "required": [
+                    "browserName"
+                  ],
+                  "type": "object",
+                  "properties": {
+                    "browserName": {
+                      "description": "Name of the browser being used",
+                      "type": "string",
+                      "examples": [
+                        "firefox",
+                        "chrome",
+                        "safari"
+                      ]
+                    },
+                    "version": {
+                      "description": "The browser version",
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            },
+            {
+              "type": "object",
+              "title": "Firefox",
+              "description": "Configuration for using firefox running as a sub-process",
+              "additionalProperties": false,
+              "properties": {
+                "target": {
+                  "description": "Target system",
+                  "const": "firefox",
+                  "type": "string",
+                  "default": "firefox"
+                }
+              }
+            }
+          ]
         }
       },
       "type": "object"

software/erp5/test/test/test_erp5.py

@@ -218,6 +218,45 @@ class TestBalancerPorts(ERP5InstanceTestCase):
         ]))
 
 
+class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test ERP5 can be instantiated with selenium server for test runner.
+  """
+  __partition_reference__ = 'sel'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        '_':
+        json.dumps({
+            'test-runner': {
+                'selenium': {
+                    "target": "selenium-server",
+                    "server-url": "https://example.com",
+                    "verify-server-certificate": False,
+                    "desired-capabilities": {
+                        "browserName": "firefox",
+                        "version": "68.0.2esr",
+                    }
+                }
+            }
+        })
+    }
+
+  def test_test_runner_configuration_json_file(self):
+    runUnitTest_script, = glob.glob(
+        self.computer_partition_root_path + "/../*/bin/runUnitTest.real")
+    config_file = None
+    with open(runUnitTest_script) as f:
+      for line in f:
+        if 'ERP5_TEST_RUNNER_CONFIGURATION' in line:
+          _, config_file = line.split('=')
+    assert config_file
+    with open(config_file.strip()) as f:
+      self.assertEqual(
+          f.read(),
+          json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner']))
+
+
 class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 can be instantiated without test runner.
   """

software/erp5testnode/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 307663d73ef3ef94b02567ecd322252e
 
 [template-default]
 filename = instance-default.cfg
-md5sum = d10958c62d0be8d0d051300d695f4f44
+md5sum = 24cc143b1886d443a4c29dcb8147a01c

software/erp5testnode/instance-default.cfg

@@ -243,9 +243,9 @@ instance-promises =
 
 [shellinabox-frontend-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = $${:_buildout_section_name_}.py
-config-hostname = $${shellinabox-frontend:hostname}
+config-host = $${shellinabox-frontend:hostname}
 config-port = $${shellinabox-frontend:port}
 
 [testnode-log-frontend-promise]

software/galene/test/README.md

+Tests for Galene software release

software/galene/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.galene'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Galene",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/galene/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from __future__ import unicode_literals
+
+import os
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '../software.cfg')))
+
+
+class TestGalene(SlapOSInstanceTestCase):
+  __partition_reference__ = 'G'
+
+  def setUp(self):
+    self.connection_parameters = self.computer_partition.getConnectionParameterDict()
+
+  def test_url_get(self):
+    resp = requests.get(self.connection_parameters['url'], verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+

software/gitlab/software.cfg

@@ -134,7 +134,7 @@ git-executable = ${git:location}/bin/git
 <= git-repository
 repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
 # 9.5.10 + NXD patches:
-revision = v9.5.10-8-gc290e22a08cb
+revision = v9.5.10-9-g69b0ffae00bf
 location = ${buildout:parts-directory}/gitlab
 
 [gitlab-shell-repository]

software/helloworld/buildout.hash.cfg

@@ -14,4 +14,4 @@
 # not need these here).
 [instance-profile]
 filename = instance.cfg.in
-md5sum = dcb9d2f540e0e397c9346c8b0c05f233
+md5sum = c771dee1ef9aedad7c6ebf9418afe08e

software/helloworld/instance.cfg.in

@@ -87,7 +87,7 @@ wrapper-path = ${directory:service}/helloweb-${:kind}
 # promise, that checks that helloweb service is alive
 [helloweb-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = helloweb-${:kind}.py
 
 {# macro to instantiate service of `kind` to listen on `port` #}
@@ -102,8 +102,8 @@ port    = {{ port }}
 [helloweb-{{ kind }}-promise]
 <= helloweb-promise
 kind    = {{ kind }}
-config-hostname= ${helloweb-{{ kind }}:ipv6}
-config-port    = {{ port }}
+config-host = ${helloweb-{{ kind }}:ipv6}
+config-port = {{ port }}
 {% endmacro %}
 
 # services instantiation

software/html5as-base/instance.cfg.in

@@ -18,10 +18,19 @@ recipe = slapos.recipe.template:jinja2
 template = {{ template_instance_html5as_target }}
 rendered = ${buildout:directory}/${:filename}
 filename = instance-html5as.cfg
-context = 
+context =
   section buildout buildout
   section parameter_list profile-common
+# partition_ipv6 is the random ipv6 allocated to the local partition
+  key partition_ipv6 slap-configuration:ipv6-random
 
 [switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = ${instance-html5as:rendered}
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = ${:default}
+default = instance-html5as:rendered
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}

software/html5as-base/instance_html5as.cfg.in

@@ -70,7 +70,7 @@ scgi_temp_path = ${:tmp}/scgi_temp_path
 nb_workers = 2
 
 # Network
-ip = ${slap-network-information:global-ipv6}
+ip = {{ partition_ipv6 }}
 port = 8081
 access_url = http://[${:ip}]:${:port}
 

software/html5as-base/software.cfg

@@ -27,7 +27,7 @@ recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/template.cfg
 template = ${:_profile_base_location_}/${:filename}
 filename = instance.cfg.in
-md5sum = 5a6ebc126bcad3cdff1b51fb51f82a35
+md5sum = 310aab063704794065ee3bc8f81fdc70
 mode = 0644
 context =
   section buildout buildout
@@ -43,7 +43,7 @@ context =
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
 _update_hash_filename_ = instance_html5as.cfg.in
-md5sum = 4a8c98cc5ca141f78f14fb9cec203cb8
+md5sum = 9b7ed68551cac5967915979383238669
 mode = 0644
 
 [template_nginx_conf]
@@ -71,8 +71,3 @@ mode = 0644
 recipe  = zc.recipe.egg
 eggs    =
     plone.recipe.command
-
-# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
-[versions]
-slapos.recipe.template = 4.4
-plone.recipe.command = 1.1

software/html5as/buildout.hash.cfg

@@ -17,11 +17,11 @@
 
 [template-cfg]
 filename = instance.cfg.in
-md5sum = 0a7aceffa5222e88125b72da42ddedd7
+md5sum = 9e486efe4ab1aba8cb72b04f6c6da8ad
 
 [instance_html5as]
 _update_hash_filename_ = instance_html5as.cfg.in
-md5sum = ec808dba866d85f7d37b85d75c13df2b
+md5sum = 191ec2a8b967a3944971709365bdcd2d
 
 [template_nginx_conf]
 _update_hash_filename_ = templates/nginx_conf.in
@@ -45,4 +45,4 @@ md5sum = 1c0ee16966e1fcdb3fd11c09f12ee2b2
 
 [template_instance_replicate]
 _update_hash_filename_ = instance_replicate.cfg.in
-md5sum = 38d1d352307f79c9c99bf2a80a5c76b8
+md5sum = 7ff7e11d05145115f53564ec1af205ef

software/html5as/instance.cfg.in

@@ -9,11 +9,9 @@ offline = true
 [profile-common]
 nginx_location = {{ nginx_location }}
 dash_location = {{ dash_location }}
-tar_location = {{ tar_location }}
-curl_location = {{ curl_location }}
 template_nginx_conf = {{ template_nginx_conf_target }}
 template_mime_types = {{ template_mime_types_target }} 
-template_launcher = {{ template_launcher_target }} 
+template_launcher = {{ template_launcher_target }}
 template_index_html = {{ template_index_html_target }}
 template_graceful  = {{ template_graceful_target }}
 template_monitor = {{ template_monitor }}
@@ -26,24 +24,43 @@ filename = instance-html5as.cfg
 context =
   section buildout buildout
   section parameter_list profile-common
+# partition_ipv6 is the random ipv6 allocated to the local partition
+  key partition_ipv6 slap-configuration:ipv6-random
+# slapparameter_dict: dictionary of all parameters
+  key slapparameter_dict slap-configuration:configuration
+  jsonkey default_parameter_dict :default-parameters
+default-parameters =
+  {
+    "title": "",
+    "download_url": null,
+    "port": 8081,
+    "monitor-httpd-port": 8197
+  }
 
 [instance-replicate]
 recipe = slapos.recipe.template:jinja2
+extensions = jinja2.ext.do
 template = {{ template_instance_replicate }}
 rendered = ${buildout:directory}/${:filename}
 filename = instance-replicate-html5as.cfg
 context =
   section buildout buildout
   section parameter_list profile-common
-  key slapparameter_dict slap-parameters:configuration
+  key slapparameter_dict slap-configuration:configuration
+  jsonkey default_parameter_dict :default-parameters
+default-parameters =
+  {
+    "download_url": null,
+    "replicate-quantity": 1
+  }
 
 [switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = ${instance-html5as:rendered}
-replicate = ${instance-replicate:rendered}
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = ${:default}
+default = instance-html5as:rendered
+replicate = instance-replicate:rendered
 
-# Section  needed to be added manually here to retrieve parameters
-[slap-parameters]
+[slap-configuration]
 recipe = slapos.cookbook:slapconfiguration
 computer = ${slap-connection:computer-id}
 partition = ${slap-connection:partition-id}

software/html5as/instance_html5as.cfg.in

@@ -3,10 +3,13 @@
 # Deploy html5as instance
 #
 #############################
+# parameter_dict: a dictionary with the default parameters from instance.cfg.in
+# replaces the values with the parameters of the instance request if there are any
+{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
+
 [buildout]
 parts =
     nginx_conf
-    downloader
     mime_types
     launcher
     nginx-graceful
@@ -58,7 +61,6 @@ service = ${directory:etc}/service
 log = ${directory:var}/log
 run = ${directory:var}/run
 backup = ${directory:srv}/backup
-data = ${directory:srv}/html5as
 
 [tempdirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -76,8 +78,8 @@ scgi_temp_path = ${:tmp}/scgi_temp_path
 nb_workers = 2
 
 # Network
-ip = ${slap-network-information:global-ipv6}
-port = ${slap-parameter:port}
+ip = {{ partition_ipv6 }}
+port = {{ parameter_dict['port'] }}
 access_url = http://[${:ip}]:${:port}
 
 # Paths
@@ -88,15 +90,13 @@ path_access_log = ${basedirectory:log}/nginx.access.log
 path_error_log = ${basedirectory:log}/nginx.error.log
 path_tmp = ${tempdirectory:tmp}
 # Docroot
-docroot = ${basedirectory:data}
-default_index = ${basedirectory:data}/index.html
+docroot = ${downloader:location}
+default_index = ${:docroot}/index.html
 # Config files
 path_nginx_conf = ${directory:etc}/nginx.conf
 path_mime_types = ${directory:etc}/mime_types
 # Binaries
 path_shell = {{ parameter_list['dash_location'] }}/bin/dash
-curl-binary = {{ parameter_list['curl_location'] }}/bin/curl
-tar-binary = {{ parameter_list['tar_location'] }}/bin/tar
 
 # Executables
 bin_launcher = ${basedirectory:service}/launcher
@@ -130,28 +130,48 @@ context =
 
 # Command to put content in the docroot
 [downloader]
-recipe = plone.recipe.command
-# This section will fail if the command fails.
-stop-on-error = true
+recipe = slapos.recipe.build
+# Path where the recipe stores any produced file,
+# it will be automatically removed at the beginning of "install".
+location = ${directory:srv}/html5as
+# All the keys in this section will be available as a dict called "self.options"
+# We add: or '', otherwise jinja2 will render a 'None' string
+url = {{ parameter_dict['download_url'] or '' }}
+default_index_html = ${default_index_html:rendered}
 # If a tarball is passed as a parameter in download url
 # it's content will be served by the instance.
 # If the parameter is not provided it fallback to the default template
-command = 
-  rm -rf ${html5as:docroot}/*; 
-  URL="${slap-parameter:download_url}"; 
-  if [ -n "$URL" ];
-  then
-    ${html5as:curl-binary} -Lks $URL | ${html5as:tar-binary} xzv -C ${html5as:docroot} --strip-components 1;
-  else
-    cp ${default_index_html:rendered} ${html5as:docroot}/;
-  fi
+install =
+  import os, shutil
+  buildout_offline = self.buildout['buildout']['offline']
+  try:
+    # Allow to do self.download() which can only be used in "online" mode
+    self.buildout['buildout']['offline'] = 'false'
+    if self.options['url']:
+      # Use fonctions from the slapos.recipe.build repository
+      # Download a file from a URL to a temporary path
+      file = self.download(self.options['url'])
+      # Create a directory and extract the file that are compressed inside
+      extract_dir = self.extract(file)
+      # Return the right directory path
+      workdir = guessworkdir(extract_dir)
+      # Recursively copy directory
+      self.copyTree(workdir, location)
+    else:
+      # Create directory and copy the default template inside
+      os.makedirs(location)
+      shutil.copy(self.options['default_index_html'], location)
+  finally:
+    # reset the parameter
+    self.buildout['buildout']['offline'] = buildout_offline
 
 [default_index_html]
 recipe = slapos.recipe.template:jinja2
 template = {{ parameter_list['template_index_html'] }}
 rendered = ${directory:srv}/index.html
+title = {{ parameter_dict['title'] }}
 context =
-  key title slap-parameter:title
+  key title :title
 
 ### Nginx Graceful
 [nginx-graceful]
@@ -165,19 +185,19 @@ context =
 # Port Listening checking promise
 [port-listening-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = nginx-port-listening.py
-config-hostname = ${html5as:ip}
+config-host = ${html5as:ip}
 config-port = ${html5as:port}
 
 # Use a port different from the default one in order to be able to
 # use it in a SlapOS webrunner or a Theia SlapOS Runner
 [monitor-instance-parameter]
-monitor-httpd-port = 8197
+monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
 
 # Monitor Stack also provides logrotate stack. We only need to extend
 # the logrotate-entry-base defined in instance-logrotate-base.cfg.in .
-# More parameters can be added following the logrotate-entry-base section  
+# More parameters can be added following the logrotate-entry-base section
 [logrotate-entry-nginx]
 <= logrotate-entry-base
 name = nginx
@@ -191,7 +211,7 @@ recipe = slapos.cookbook:publish
 # be deployed. The parameters needed for accessing monitoring will be published
 <= monitor-publish
 server_url = ${html5as:access_url}
-title = Title ${slap-parameter:title}!
+title = Title {{ parameter_dict['title'] }}!
 # Add dependency to the promise so that frontend sections are processed
 # and there is no need to declare the new part in buildout:parts
 server-cdn-url = ${html5as-frontend-promise:url}
@@ -202,7 +222,7 @@ server-cdn-url = ${html5as-frontend-promise:url}
 <= slap-connection
 # Recipe used to make requests
 recipe = slapos.cookbook:requestoptional
-name = HTM5AS frontend
+name = HTML5AS frontend
 # Specify the software url of the frontend software release
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 # It is not a dedicated instance but an instance allocated on a shared instance
@@ -220,8 +240,3 @@ name = html5as-http-frontend.py
 url = ${html5as-frontend:connection-secure_access}
 config-url = ${:url}
 config-check-secure = 1
-
-[slap-parameter]
-title =
-download_url =
-port = 8081

software/html5as/instance_replicate.cfg.in

-{% set replicate_quantity = slapparameter_dict.pop('replicate-quantity', '1') | int %}
+{%- set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
+{%- set replicate_quantity = int(parameter_dict['replicate-quantity']) %}
+# Set default title and port for each replicate based on requested quantity
+{%- for i in range(1, replicate_quantity + 1) %}
+{%- do parameter_dict.setdefault("title-%d" % i, "") %}
+{%- do parameter_dict.setdefault("port-%d" % i, 8081 + i) %}
+{%- do parameter_dict.setdefault("monitor-httpd-port-%d" % i, 8197 + i) %}
+{%- endfor %}
+
+# Standard buildout section
 [buildout]
 parts =
     publish-connection-information
@@ -7,6 +16,11 @@ eggs-directory = {{ buildout['eggs-directory'] }}
 develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
 offline = true
 
+################################
+# Sections to Request instances
+################################
+
+# Macro section sharing request parameters
 [instance-request-base]
 <= slap-connection
 recipe = slapos.cookbook:request
@@ -17,29 +31,25 @@ software-type = default
 # What parameter are neede to be retrieved
 return = server_url server-cdn-url monitor-setup-url
 # Provided parameters
-config-title = ${slap-parameter:title}
-config-download_url = ${slap-parameter:download_url}
+# We add: or '', otherwise jinja2 will render a 'None' string
+config-download_url = {{ parameter_dict['download_url'] or '' }}
 
+# Create request section in a loop.
 {% for i in range(1, replicate_quantity + 1) %}
 # Request a normal html5as instance
 [instance-{{ i }}]
 <= instance-request-base
 # Name of the instance
 name = instance-html5as-{{ i }}
-config-port = ${slap-parameter:port-{{ i }}}
-config-title = ${slap-parameter:title-{{ i }}}
-{%   if "sla-%s-computer-guid" % i in slapparameter_dict -%}
-sla-computer_guid = {{ slapparameter_dict["sla-%s-computer-guid" % i] }}
+config-port = {{ parameter_dict["port-%s" % i] }}
+config-title = {{ parameter_dict["title-%s" % i] }}
+config-monitor-httpd-port = {{ parameter_dict["monitor-httpd-port-%s" % i] }}
+{%   if "sla-%s-computer-guid" % i in parameter_dict -%}
+sla-computer_guid = {{ parameter_dict["sla-%s-computer-guid" % i] }}
 {%   endif -%}
 {% endfor %}
 
-[slap-parameter]
-download_url =
-{% for i in range(1, replicate_quantity + 1) %}
-title-{{ i }} =
-port-{{ i }} = 808{{ i }}
-{% endfor %}
-
+# Publish information to connect to the two instances
 [publish-connection-information]
 recipe = slapos.cookbook:publish
 {% for i in range(1, replicate_quantity + 1) %}

software/html5as/software.cfg

@@ -13,8 +13,6 @@ extends =
 # In this example we extend needed components for html5as.
   ../../component/nginx/buildout.cfg
   ../../component/dash/buildout.cfg
-  ../../component/tar/buildout.cfg
-  ../../component/curl/buildout.cfg
 
 parts =
 # Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
@@ -36,8 +34,6 @@ context =
   section buildout buildout
   key nginx_location nginx:location
   key dash_location dash:location
-  key curl_location curl:location
-  key tar_location tar:location
   key template_nginx_conf_target template_nginx_conf:target
   key template_mime_types_target template_mime_types:target
   key template_launcher_target template_launcher:target
@@ -79,7 +75,3 @@ mode = 0644
 
 [template_instance_replicate]
 <= download-base
-
-# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
-[versions]
-slapos.recipe.template = 4.4

software/htmlvalidatorserver/buildout.hash.cfg

@@ -22,8 +22,8 @@ md5sum = 87781e6bcb523bb8434888d5f984f36c
 
 [template-validator]
 filename = instance-validator.cfg.in
-md5sum = 2be286e367e37ce9e504170cd3a08007
+md5sum = 9d12472bb2e337d3cc18f2cc6f235425
 
 [template]
 filename = instance.cfg.in
-md5sum = 2b4d33e9ef1082dd4d6a53f55b391772
+md5sum = 94fc13254c819cba33b03f30251bc469

software/htmlvalidatorserver/instance-validator.cfg.in

@@ -49,15 +49,15 @@ recipe = slapos.recipe.template
 url = ${template-tomcat-configuration:output}
 output = $${basedirectory:catalina_conf}/server.xml
 mode = 0600
-ip = $${slap-network-information:global-ipv6}
+ip = {{ partition_ipv6 }}
 port = 8899
 scheme = https
 
 [tomcat-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = tomcat_listen.py
-config-hostname = $${tomcat-configuration:ip}
+config-host = $${tomcat-configuration:ip}
 config-port = $${tomcat-configuration:port}
 
 #################################

software/htmlvalidatorserver/instance.cfg.in

@@ -6,29 +6,26 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = $${:validator}
-validator = ${template-validator:output}
-
-[slap-connection]
-# part to migrate to new - separated words
-computer-id = $${slap_connection:computer_id}
-partition-id = $${slap_connection:partition_id}
-server-url = $${slap_connection:server_url}
-software-release-url = $${slap_connection:software_release_url}
-key-file = $${slap_connection:key_file}
-cert-file = $${slap_connection:cert_file}
+[dynamic-template-validator]
+recipe = slapos.recipe.template:jinja2
+template = ${template-validator:output}
+rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
+filename = instance-validator.cfg
+context =
+# partition_ipv6 is the random ipv6 allocated to the local partition
+    key partition_ipv6 slap-configuration:ipv6-random
 
-# [slap-parameter]
-# slave-instance-list = []
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = $${:validator}
+validator = dynamic-template-validator:rendered
 
-[instance-parameter]
+[slap-configuration]
 # Fetches parameters defined in SlapOS Master for this instance.
 # Always the same.
 recipe = slapos.cookbook:slapconfiguration.serialised
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}

software/htmlvalidatorserver/software.cfg

@@ -49,5 +49,4 @@ mode = 0644
 # 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
 # but '1.3.4nxd2-SlapOSPatched001'.
 gunicorn = 19.1.1
-plone.recipe.command = 1.1
 inotifyx = 0.2.2

software/jscrawler/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 6c17361a49cfc47564063b867aab6e8c
 
 [template-jscrawler]
 filename = instance-jscrawler.cfg.jinja2.in
-md5sum = bdf9e67077cd5c3c140974bcc56422ad
+md5sum = fece076231740b414612da5ef3a1685a
 
 [template-jscrawler-builder]
 filename = template-jscrawler.builder.sh.in

software/jscrawler/instance-jscrawler.cfg.jinja2.in

@@ -60,9 +60,9 @@ log = ${httpd-wrapper:log-file}
 
 [httpd-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = httpd-listen.py
-config-hostname = ${httpd-wrapper:host}
+config-host = ${httpd-wrapper:host}
 config-port = ${httpd-wrapper:port}
 
 [jscrawler-wrapper]

software/jstestnode/instance-jstestnode-input-schema.json

@@ -23,6 +23,31 @@
       "example": "https://softinst1234.host.vifib.net/"
     },
     "test-runner": {
+      "default": {
+        "target": "firefox"
+      },
+      "examples": [
+        {
+          "target": "selenium-server",
+          "server-url": "https://selenium.example.com",
+          "desired-capabilities": {
+            "browserName": "firefox",
+            "version": "68.0.2esr",
+            "acceptInsecureCerts": true
+          }
+        },
+        {
+          "target": "selenium-server",
+          "server-url": "https://selenium.example.com",
+          "desired-capabilities": {
+            "browserName": "chrome",
+            "version": "91.0.4472.101"
+          }
+        },
+        {
+          "target": "node"
+        }
+      ],
       "oneOf": [
         {
           "type": "object",
@@ -46,14 +71,14 @@
               "format": "uri"
             },
             "verify-server-certificate": {
-              "description": "Verify the SSL/TLS Certificats of the selenium server when using HTTPS",
+              "description": "Verify the SSL/TLS certificate of the selenium server when using HTTPS",
               "type": "boolean",
               "default": true
             },
             "server-ca-certificate": {
-              "description": "PEM encoded bundle of CA Certificates to verify the SSL/TLS Certificate of the selenium server when using HTTPS",
+              "description": "PEM encoded bundle of CA certificates to verify the SSL/TLS certificate of the selenium server when using HTTPS",
               "type": "string",
-              "default": "root certificates from http://certifi.io/en/latest/"
+              "default": "Root certificates from http://certifi.io/en/latest/"
             },
             "desired-capabilities": {
               "description": "Desired browser capabilities",
@@ -63,8 +88,13 @@
               "type": "object",
               "properties": {
                 "browserName": {
-                  "description": "Name of the browser being used, for example firefox, chrome",
-                  "type": "string"
+                  "description": "Name of the browser being used",
+                  "type": "string",
+                  "examples": [
+                    "firefox",
+                    "chrome",
+                    "safari"
+                  ]
                 },
                 "version": {
                   "description": "The browser version",
@@ -77,7 +107,7 @@
         {
           "type": "object",
           "title": "Firefox",
-          "description": "Configuration for Firefox",
+          "description": "Configuration for using firefox running as a sub-process",
           "additionalProperties": false,
           "properties": {
             "target": {

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = bf0c01ac7493693bb57ebef00bb20fa0
+md5sum = d0f96be4e80b96e6ac33f6d474767b13
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
 
 [template-nbd]
 filename = instance-nbd.cfg.jinja2
-md5sum = 6ea26f88252bf899c966d0f5675e7176
+md5sum = 259e06f289f68297e0609e4ab1af8e86
 
 [template-ansible-promise]
 filename = template/ansible-promise.in
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = be750fb62f7057c97dd6c6887b2149cc
+md5sum = a502782244d1be536b732ebb40725f47
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
@@ -75,7 +75,7 @@ md5sum = fb330a796fadb6cd5c85217f80a42af3
 
 [template-httpd]
 filename = instance-kvm-http.cfg.in
-md5sum = d657884d02105deffddee0edae4b50a6
+md5sum = f4bcde62e008c2da9c65617ba7f73f08
 
 [image-download-controller]
 _update_hash_filename_ = template/image-download-controller.py

software/kvm/instance-kvm-cluster-input-schema.json

@@ -217,7 +217,7 @@
             },
             "enable-device-hotplug": {
               "title": "Enable device hotplug mode",
-              "description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
+              "description": "Allows to increase amount of RAM (ram-size) and CPU (cmp-count) without restart of the VM process, up to defined maximums (ram-max-size and cpu-max-count). Operation system have to support online addition of RAM and CPU.",
               "type": "boolean",
               "default": false
             },

software/kvm/instance-kvm-http.cfg.in

@@ -65,7 +65,7 @@ stop-on-error = true
 
 [httpd-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = apache-httpd.py
-config-hostname = ${apache-conf:ip}
+config-host = ${apache-conf:ip}
 config-port = ${apache-conf:port}

software/kvm/instance-kvm-input-schema.json

@@ -5,7 +5,7 @@
   "properties": {
     "enable-device-hotplug": {
       "title": "Enable device hotplug mode",
-      "description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
+      "description": "Allows to increase amount of RAM (ram-size) and CPU (cmp-count) without restart of the VM process, up to defined maximums (ram-max-size and cpu-max-count). Operation system have to support online addition of RAM and CPU.",
       "type": "boolean",
       "default": false
     },

software/kvm/instance-kvm.cfg.jinja2

@@ -437,13 +437,17 @@ disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
 pid-file-path = ${kvm-controller-parameter-dict:pid-file}
 socket-path = ${kvm-controller-parameter-dict:socket-path}
 
-enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
-smp-count = ${kvm-controller-parameter-dict:cpu-count}
+{%- set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() == 'true' %}
 smp-max-count = {{ cpu_max_count }}
-
-ram-size = ${kvm-controller-parameter-dict:ram-size}
 ram-max-size = {{ ram_max_size }}
+{%- if enable_device_hotplug %}
 init-ram-size = 1024
+init-smp-count = 1
+{%- else %}
+init-ram-size = ${kvm-controller-parameter-dict:ram-size}
+init-smp-count = ${kvm-controller-parameter-dict:cpu-count}
+{%- endif %}
+
 mac-address = ${create-mac:mac-address}
 tap-mac-address = ${create-tap-mac:mac-address}
 
@@ -582,9 +586,9 @@ command-line = ${kvm-controller:rendered}
 
 [kvm-vnc-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = vnc_promise.py
-config-hostname = ${kvm-parameter-dict:vnc-ip}
+config-host = ${kvm-parameter-dict:vnc-ip}
 config-port = ${kvm-parameter-dict:vnc-port}
 
 [kvm-disk-image-corruption-bin]
@@ -704,9 +708,9 @@ wrapper = ${directory:bin}/websockify
 
 [novnc-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = novnc_promise.py
-config-hostname = ${novnc-instance:ip}
+config-host = ${novnc-instance:ip}
 config-port = ${novnc-instance:port}
 
 
@@ -797,9 +801,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [httpd-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = httpd.py
-config-hostname = ${httpd:host}
+config-host = ${httpd:host}
 config-port = ${httpd:port}
 {% endif %}
 

software/kvm/instance-nbd.cfg.jinja2

@@ -65,9 +65,9 @@ key = ${gen-passwd:passwd}
 
 [onetimeupload-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = onetimeupload_promise.py
-config-hostname = ${onetimeupload-instance:ip}
+config-host = ${onetimeupload-instance:ip}
 config-port = ${onetimeupload-instance:port}
 
 [publish-connection-information]

software/kvm/template/template-kvm-run.in

@@ -37,9 +37,9 @@ mac_address = '{{ parameter_dict.get("mac-address") }}'
 tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
 tap_ipv6_addr = '{{ parameter_dict.get("tap-ipv6-addr") }}'
 numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
-ram_size = {{ parameter_dict.get("ram-size") }}
 ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
 init_ram_size = {{ parameter_dict.get("init-ram-size") }}
+init_smp_count = {{ parameter_dict.get("init-smp-count") }}
 pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
 external_disk_number = {{ parameter_dict.get("external-disk-number") }}
 external_disk_size = {{ parameter_dict.get("external-disk-size") }}
@@ -78,7 +78,6 @@ if not disk_info_list:
 {%- endfor %}
   })
 
-smp_count = {{ parameter_dict.get("smp-count") }}
 smp_max_count = {{ parameter_dict.get("smp-max-count") }}
 machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
 cpu_model = '{{ parameter_dict.get("cpu-model") }}'.strip()
@@ -267,12 +266,8 @@ if use_tap == 'true':
             tap_interface, vhost),
           '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
 
-if enable_device_hotplug != 'true':
-  smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
-  ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
-else:
-  smp = '1,maxcpus=%s' % smp_max_count
-  ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
+smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
+ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
 
 kvm_argument_list = [qemu_path,
   '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',

software/kvm/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       install_requires=[
           'slapos.core',
           'slapos.cookbook',
+          'slapos.toolbox',
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',

software/kvm/test/test.py

@@ -46,6 +46,7 @@ import time
 import shutil
 import sys
 
+from slapos.qemuqmpclient import QemuQMPWrapper
 from slapos.proxy.db_version import DB_VERSION
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -60,6 +61,8 @@ if has_kvm:
     os.path.abspath(
       os.path.join(os.path.dirname(__file__), '..',
                    'software%s.cfg' % ("-py3" if six.PY3 else ""))))
+  # XXX Keep using slapos node instance --all, because of missing promises
+  InstanceTestCase.slap._force_slapos_node_instance_all = True
 else:
   setUpModule, InstanceTestCase = None, unittest.TestCase
 
@@ -203,6 +206,108 @@ i0:whitelist-firewall-{hash} RUNNING""",
     )
 
 
+@skipUnlessKvm
+class TestMemoryManagement(InstanceTestCase, KvmMixin):
+  __partition_reference__ = 'i'
+
+  def getKvmProcessInfo(self, switch_list):
+    return_list = []
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
+                 if 'kvm-' in q['name']][0]['pid']
+      kvm_process = psutil.Process(kvm_pid)
+      get_next = False
+      for entry in kvm_process.cmdline():
+        if get_next:
+          return_list.append(entry)
+          get_next = False
+        elif entry in switch_list:
+          get_next = True
+    return kvm_pid, return_list
+
+  def test(self):
+    kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+    self.assertEqual(
+      ['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
+      info_list
+    )
+    self.rerequestInstance({
+      'ram-size': '1536',
+      'cpu-count': '2',
+    })
+    self.slap.waitForInstance(max_retry=10)
+    kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+    self.assertEqual(
+      ['2,maxcpus=3', '1536M,slots=128,maxmem=2048M'],
+      info_list
+    )
+
+    # assert that process was restarted
+    self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
+
+  def tearDown(self):
+    self.rerequestInstance({})
+    self.slap.waitForInstance(max_retry=10)
+
+  def test_enable_device_hotplug(self):
+    def getHotpluggedCpuRamValue():
+      qemu_wrapper = QemuQMPWrapper(os.path.join(
+        self.computer_partition_root_path, 'var', 'qmp_socket'))
+      ram_mb = sum(
+        [q['size']
+         for q in qemu_wrapper.getMemoryInfo()['hotplugged']]) / 1024 / 1024
+      cpu_count = len(
+        [q['CPU'] for q in qemu_wrapper.getCPUInfo()['hotplugged']])
+      return {'cpu_count': cpu_count, 'ram_mb': ram_mb}
+
+    kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+    self.assertEqual(
+      ['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
+      info_list
+    )
+    self.assertEqual(
+      getHotpluggedCpuRamValue(),
+      {'cpu_count': 0, 'ram_mb': 0}
+    )
+
+    parameter_dict = {
+      'enable-device-hotplug': 'true',
+      # to avoid restarts the max RAM and CPU has to be static
+      'ram-max-size': '2048',
+      'cpu-max-count': '4',
+    }
+    self.rerequestInstance(parameter_dict)
+    self.slap.waitForInstance(max_retry=2)
+    kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+
+    self.assertEqual(
+      ['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
+      info_list
+    )
+    self.assertEqual(
+      getHotpluggedCpuRamValue(),
+      {'cpu_count': 0, 'ram_mb': 0}
+    )
+    self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
+    parameter_dict.update(**{
+      'ram-size': '1536',
+      'cpu-count': '2'
+    })
+    self.rerequestInstance(parameter_dict)
+    self.slap.waitForInstance(max_retry=10)
+    kvm_pid_3, info_list = self.getKvmProcessInfo(['-smp', '-m'])
+
+    self.assertEqual(
+      ['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
+      info_list
+    )
+    self.assertEqual(kvm_pid_2, kvm_pid_3, "Unexpected: KVM restarted")
+    self.assertEqual(
+      getHotpluggedCpuRamValue(),
+      {'cpu_count': 1, 'ram_mb': 512}
+    )
+
+
 class MonitorAccessMixin(object):
   def sqlite3_connect(self):
     sqlitedb_file = os.path.join(
@@ -223,7 +328,7 @@ class MonitorAccessMixin(object):
       }
       return db.execute(
         "SELECT reference, xml, connection_xml, partition_reference,"
-              " software_release, requested_state, software_type"
+        " software_release, requested_state, software_type"
         " FROM partition%s"
         " WHERE slap_state='busy'" % DB_VERSION).fetchall()
     finally:

software/monitor/test/test.py

@@ -142,6 +142,23 @@ class EdgeSlaveMixin(MonitorTestMixin):
   instance_max_retry = 20
   expected_connection_parameter_dict = {}
 
+  @classmethod
+  def setUpClass(cls):
+    # XXX we run these tests with --all as a workaround for the fact that after
+    # requesting new shared instances we don't have promise to wait for the
+    # processing of these shared instances to be completed.
+    # The sequence is something like this:
+    #   - `requestEdgetestSlaves` will request edgetest partition
+    #   - first `waitForInstance` will process the edgetest partition, which will
+    #     request a edgebot partition, but without promise to wait for the
+    #     processing to be finished, so the first run of `slapos node instance`
+    #     exits with success code and `waitForInstance` return.
+    #   - second `waitForInstance` process the edgebot partition.
+    # Once we implement a promise (or something similar) here, we should not
+    # have to use --all
+    cls.slap._force_slapos_node_instance_all = True
+    return super(EdgeSlaveMixin, cls).setUpClass()
+
   @classmethod
   def getInstanceSoftwareType(cls):
     return 'edgetest'

software/plantuml/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = d1dff3fc39eefc57b36dbaa195b6890e
+md5sum = c962079a88a6ce97d8ce20fa4e8edfd1
 
 [tomcat-server-xml]
 filename = server.xml.in

software/plantuml/instance.cfg.in

@@ -87,9 +87,9 @@ instance-promises =
 
 [tomcat-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = $${:_buildout_section_name_}.py
-config-hostname= $${tomcat-instance:ip}
+config-host = $${tomcat-instance:ip}
 config-port = $${tomcat-instance:port}
 
 [publish-connection-parameter]

software/powerdns/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
 
 [template-powerdns]
 filename = instance-powerdns.cfg
-md5sum = 0920200cb05a68b1b4a161a927d9488f
+md5sum = c04c3b490e7f9f35af3d204a9df51f35
 
 [template-pdns-configuration]
 _update_hash_filename_ = template/pdns.conf.jinja2
@@ -26,7 +26,7 @@ md5sum = 20c37ea06a8fa405bc02470d5115fd11
 
 [template-dns-replicate]
 _update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
-md5sum = 504d15f0bbf0e515d5ff16070f1ac802
+md5sum = 4ff993a39da03d9d66d7c0f98efeb1e0
 
 [iso-list]
 _update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -73,9 +73,9 @@ sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
 
 [{{promise_section_title}}]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = {{promise_section_title}}.py
-config-hostname = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
+config-host = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
 config-port = {{ '${' ~ request_section_title ~ ':connection-powerdns-port}' }}
 
 {%  do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}

software/powerdns/instance-powerdns.cfg

@@ -137,9 +137,9 @@ extra-context =
 # Promises
 [pdns-promise-listen-port]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = pdns-port-listening.py
-config-hostname = $${pdns:ipv4}
+config-host = $${pdns:ipv4}
 config-port = $${pdns:port}
 
 [publish-connection-informations]

software/proftpd/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = efb4238229681447aa7fe73898dffad4
 
 [instance-default]
 filename = instance-default.cfg.in
-md5sum = dae19ec06f8da9fa2980a6d2bdf3da54
+md5sum = c6dce31a36e4e13de62687e9888aeb77
 
 [proftpd-config-file]
 filename = proftpd-config-file.cfg.in

software/proftpd/instance-default.cfg.in

@@ -86,9 +86,9 @@ template = inline:{{ slapparameter_dict['ssh-key'] | indent }}
 
 [proftpd-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = ${:_buildout_section_name_}.py
-config-hostname = ${proftpd:ipv6}
+config-host = ${proftpd:ipv6}
 config-port = ${proftpd:sftp-port}
 
 

software/pureftpd/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = 134ed1cf1f6de63b14425031eb5c9043
+md5sum = 500b773d1a63a6a895f9b8038a582b05

software/pureftpd/instance.cfg.in

@@ -33,12 +33,12 @@ pureftpd-dir = ${:srv}/pureftpd/
 
 [check-port-listening-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = check_nginx_port.py
 
 [pureftpd-listen-promise]
 <= check-port-listening-promise
-config-hostname = ${pureftpd:ipv6}
+config-host = ${pureftpd:ipv6}
 config-port = ${pureftpd:ftp-port}
 
 [pureftpd-userinfo]

software/re6stnet/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 71531ed9c9b79fa769ab367e7ea2d2a5
 
 [template-re6stnet]
 filename = instance-re6stnet.cfg.in
-md5sum = 2309889cf3f5bc57ba63d389e662fc21
+md5sum = 870c34cf58acaaee21c71182dd3cb0cf
 
 [template-apache-conf]
 filename = apache.conf.in

software/re6stnet/instance-re6stnet.cfg.in

@@ -170,16 +170,16 @@ context =
 
 [re6st-registry-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = re6st-registry.py
-config-hostname = ${re6st-registry:ipv4}
+config-host = ${re6st-registry:ipv4}
 config-port = ${re6st-registry:port}
 
 [apache-registry-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = apache-re6st-registry.py
-config-hostname = ${apache-conf:ipv6}
+config-host = ${apache-conf:ipv6}
 config-port = ${apache-conf:port}
 
 [publish]

software/repman/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 1d6eba3984b3e2009682f6ce49b8ac4d
+md5sum = 0c173313b48d0005c46d968db1c8ab5f
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in

software/repman/instance-repman.cfg.jinja2.in

@@ -216,23 +216,23 @@ depends =
 
 [proxysql-{{ name }}-admin-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = proxysql-{{ name }}-admin-port-listening.py
-config-hostname=  {{ ipv4 }}
+config-host =  {{ ipv4 }}
 config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}
 
 [proxysql-{{ name }}-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = proxysql-{{ name }}-port-listening.py
-config-hostname=  {{ ipv4 }}
+config-host =  {{ ipv4 }}
 config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 
 [proxysql-{{ name }}-ipv6-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = proxysql-{{ name }}-ipv6-port-listening.py
-config-hostname=  {{ ip }}
+config-host =  {{ ip }}
 config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 
 {% set service_name = "proxysql-" ~ name -%}
@@ -403,16 +403,16 @@ context =
 
 [repman-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = repman_service_listen.py
-config-hostname = ${repman-parameter:ipv4}
+config-host = ${repman-parameter:ipv4}
 config-port = ${repman-parameter:port}
 
 [repman-listen-ssl-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = repman_service_ssl_listen.py
-config-hostname = ${repman-parameter:ipv4}
+config-host = ${repman-parameter:ipv4}
 config-port = ${repman-parameter:secure-port}
 
 [nginx-conf]

software/restic-rest-server/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance.cfg.in]
+filename = instance.cfg.in
+md5sum = fee2097f3d12fd4bcfbc2698ecf49afc

software/restic-rest-server/instance.cfg.in

+{% import "caucase" as caucase with context %}
+
+[buildout]
+parts =
+  promises
+  publish-connection-parameter
+extends =
+  {{ template_monitor }}
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-configuration]
+# frontend reads from from a part named [slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+srv = ${:home}/srv
+tmp = ${:home}/tmp
+var-log = ${:var}/log
+service = ${:etc}/service
+promise = ${:etc}/promise
+rest-server-data-dir = ${:srv}/restic
+backup-caucased = ${:srv}/backup/caucased/
+
+# Macros
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/${:_buildout_section_name_}
+
+[check-url-available-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promise}/${:_buildout_section_name_}
+dash_path = {{ dash_bin }}
+curl_path = {{ curl_bin }}
+
+# Caucase
+[rest-server-certificate]
+key-file = ${directory:etc}/${:_buildout_section_name_}.key
+cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
+common-name = ${:_buildout_section_name_}
+ca-file = ${directory:etc}/${:_buildout_section_name_}.ca.crt
+crl-file = ${directory:etc}/${:_buildout_section_name_}.crl
+
+{{
+caucase.updater(
+    prefix='rest-server-certificate',
+    buildout_bin_directory=buildout['bin-directory'],
+    updater_path='${directory:service}/rest-server-certificate-updater',
+    url='${caucased:url}',
+    data_dir='${directory:srv}/caucase-updater',
+    crt_path='${rest-server-certificate:cert-file}',
+    ca_path='${rest-server-certificate:ca-file}',
+    crl_path='${rest-server-certificate:crl-file}',
+    key_path='${rest-server-certificate:key-file}',
+    template_csr='${rest-server-certificate-prepare-csr:csr}',
+    openssl=openssl_bin,
+)}}
+
+[rest-server-certificate-csr-config]
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = inline:
+  [req]
+  prompt = no
+  req_extensions = req_ext
+  distinguished_name = dn
+  [ dn ]
+  CN = restic-rest-server
+  [ req_ext ]
+  subjectAltName = @alt_names
+  [ alt_names ]
+  IP.1 = ${instance-parameter:ipv4-random}
+  IP.2 = ${instance-parameter:ipv6-random}
+
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
+
+[rest-server-certificate-prepare-csr]
+recipe = plone.recipe.command
+command =
+  if [ ! -f '${:csr}' ] ; then
+   {{ openssl_bin }} req \
+      -newkey rsa:2048 \
+      -batch \
+      -new \
+      -nodes \
+      -keyout /dev/null \
+      -config '${rest-server-certificate-csr-config:rendered}' \
+      -out '${:csr}'
+  fi
+stop-on-error = true
+csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
+
+[caucased]
+port = 18080
+ip = ${instance-parameter:ipv6-random}
+netloc = [${:ip}]:${:port}
+url = http://${:netloc}/
+
+{{
+caucase.caucased(
+    prefix='caucased',
+    buildout_bin_directory=buildout['bin-directory'],
+    caucased_path='${directory:service}/caucased',
+    backup_dir='${directory:backup-caucased}',
+    data_dir='${directory:srv}/caucased',
+    netloc='${caucased:netloc}',
+    tmp='${directory:tmp}',
+    service_auto_approve_count=1,
+    user_auto_approve_count=0,
+    key_len=2048,
+)}}
+
+[rest-server-password]
+recipe = slapos.cookbook:generate.password
+user = backup
+
+[rest-server-htpassword]
+recipe = plone.recipe.command
+command = 
+  if [ ! -f '${:htpassword}' ] ; then
+    {{ htpasswd_bin }} \
+      -b \
+      -B \
+      -c ${:htpassword} \
+      ${rest-server-password:user} \
+      ${rest-server-password:passwd}
+  fi
+htpassword = ${directory:rest-server-data-dir}/.htpasswd
+stop-on-error = true
+
+[rest-server]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ gowork_bin }}/rest-server \
+    --listen [${instance-parameter:ipv6-random}]:${:port}
+    --log ${directory:var-log}/${:_buildout_section_name_}-access.log
+    --path ${directory:rest-server-data-dir}
+    --tls
+    --tls-cert ${rest-server-certificate:cert-file}
+    --tls-key ${rest-server-certificate:key-file}
+    --prometheus
+wrapper-path = ${directory:service}/rest-server
+
+port = 19080
+ip = ${instance-parameter:ipv6-random}
+url = https://[${:ip}]:${:port}
+depends =
+  ${rest-server-htpassword:recipe}
+
+[rest-server-listen-promise]
+<= check-port-listening-promise
+hostname= ${rest-server:ip}
+port = ${rest-server:port}
+
+[frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Rest Server Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = ${rest-server:url}
+return = domain secure_access
+
+[frontend-available-promise]
+<= check-url-available-promise
+url = ${frontend:connection-secure_access}
+check-secure = 1
+
+
+[promises]
+recipe =
+instance-promises =
+  ${caucased-promise:recipe}
+  ${rest-server-certificate-promise:recipe}
+  ${rest-server-listen-promise:path}
+  ${frontend-available-promise:path}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+rest-server-user = ${rest-server-password:user}
+rest-server-password = ${rest-server-password:passwd}
+url = ${frontend:connection-secure_access}
+caucase-url = ${caucased:url}

software/restic-rest-server/software.cfg

+[buildout]
+
+extends =
+  ../../component/openssl/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/golang/buildout.cfg
+  ../../component/restic/buildout.cfg
+  ../../stack/caucase/buildout.cfg
+  ../../stack/slapos.cfg
+  buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  caucase-eggs
+  instance.cfg.in
+  restic
+  gowork
+
+[python]
+part = python3
+
+[gowork]
+install +=
+  ${git.github.com_restic_rest-server:location}:./cmd/...
+
+[git.github.com_restic_rest-server]
+<= go-git-package
+go.importpath = github.com/restic/rest-server
+repository =  https://github.com/restic/rest-server
+revision = v0.10.0-0-g9313f19
+
+[instance.cfg.in]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/instance.cfg
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+context =
+  section buildout buildout
+  key gowork_bin gowork:bin
+  raw openssl_bin ${openssl:location}/bin/openssl
+  raw htpasswd_bin ${apache:location}/bin/htpasswd
+  raw dash_bin ${dash:location}/bin/dash
+  raw curl_bin ${curl:location}/bin/curl
+  key template_monitor monitor2-template:rendered
+import-list =
+  file caucase caucase-jinja2-library:target

software/restic-rest-server/test/README.md

+Tests for restic ReST Server software release

software/restic-rest-server/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.restic_rest_server'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' restic Rest Server",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/restic-rest-server/test/test.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import glob
+import os
+import subprocess
+import tempfile
+import urllib.parse
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestResticRestServer(SlapOSInstanceTestCase):
+
+  def setUp(self):
+    self.connection_parameters = \
+        self.computer_partition.getConnectionParameterDict()
+    parsed_url = urllib.parse.urlparse(self.connection_parameters['url'])
+    self.url_with_credentials = parsed_url._replace(
+        netloc='{}:{}@[{}]:{}'.format(
+            self.connection_parameters['rest-server-user'],
+            self.connection_parameters['rest-server-password'],
+            parsed_url.hostname,
+            parsed_url.port,
+        )).geturl()
+
+    self.ca_cert = self._getCaucaseServiceCACertificate()
+
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+        prefix="ca.crt.pem",
+        mode="w",
+        delete=False,
+    )
+    ca_cert.write(
+        requests.get(
+            urllib.parse.urljoin(
+                self.connection_parameters['caucase-url'],
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
+  def test_http_get(self):
+    resp = requests.get(self.connection_parameters['url'], verify=self.ca_cert)
+    self.assertEqual(resp.status_code, requests.codes.unauthorized)
+
+    resp = requests.get(
+        urllib.parse.urljoin(
+            self.url_with_credentials,
+            '/metrics',
+        ),
+        verify=self.ca_cert,
+    )
+    # a random metric
+    self.assertIn('process_cpu_seconds_total', resp.text)
+    resp.raise_for_status()
+
+  def test_backup_scenario(self):
+    restic_bin = os.path.join(
+        self.computer_partition_root_path,
+        'software_release',
+        'go.work',
+        'bin',
+        'restic',
+    )
+
+    def get_restic_output(*args, **kw):
+      return subprocess.check_output(
+          (
+              restic_bin,
+              '--cacert',
+              self.ca_cert,
+              '--password-file',
+              password_file.name,
+              '--repo',
+              'rest:' + self.url_with_credentials,
+          ) + args,
+          universal_newlines=True,
+          **kw,
+      )
+
+    with tempfile.TemporaryDirectory() as work_directory,\
+        tempfile.NamedTemporaryFile(mode='w') as password_file:
+      password_file.write('secret')
+      password_file.flush()
+
+      with open(os.path.join(work_directory, 'data'), 'w') as f:
+        f.write('data to backup')
+      with self.assertRaises(subprocess.CalledProcessError) as exc_context:
+        get_restic_output('snapshots', stderr=subprocess.PIPE)
+      self.assertIn('Is there a repository at the following location?',
+                    exc_context.exception.stderr)
+
+      out = get_restic_output('init')
+      self.assertIn('created restic repository', out)
+
+      out = get_restic_output('backup', work_directory)
+      self.assertIn('Added to the repo', out)
+
+      out = get_restic_output('snapshots')
+      self.assertEqual(out.splitlines()[-1], '1 snapshots')
+      snapshot_id = out.splitlines()[2].split()[0]
+      backup_path = out.splitlines()[2].split()[-1]
+      restore_directory = os.path.join(work_directory, 'restore')
+
+      out = get_restic_output(
+          'restore',
+          snapshot_id,
+          '--target',
+          restore_directory,
+      )
+      self.assertIn('restoring <Snapshot', out)
+      with open(os.path.join(restore_directory, backup_path, 'data')) as f:
+        self.assertEqual(f.read(), 'data to backup')

software/seleniumserver/buildout.hash.cfg

@@ -15,8 +15,8 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = c4ac5de141ae6a64848309af03e51d88
+md5sum = 0084214fae4ee1aad2c878aa393757af
 
 [template-selenium]
 filename = instance-selenium.cfg.in
-md5sum = eea51d25c292c7ea305229184e380814
+md5sum = 884196ea35de35fa9159517912441ce6

software/seleniumserver/instance-selenium.cfg.in

@@ -61,7 +61,7 @@ command-line =
 
 # newSessionWaitTimeout: let clients wait in the queue when no node are available
 # maxSession: to accept enough clients
-hostname = $${instance-parameter:ipv4-random}
+hostname = $${slap-configuration:ipv4-random}
 port = 4444
 base-url = http://$${:hostname}:$${:port}
 url = $${:base-url}/wd/hub
@@ -94,7 +94,7 @@ environment =
   XORG_LOCK_DIR=$${directory:tmp}
   DISPLAY=$${xvfb-instance:display}
   FONTCONFIG_FILE=$${fontconfig-conf:rendered}
-hostname = $${instance-parameter:ipv4-random}
+hostname = $${slap-configuration:ipv4-random}
 
 
 [selenium-server-node-instance-firefox-52]
@@ -115,11 +115,23 @@ capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX
 java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
 port = 7779
 
+[selenium-server-node-instance-firefox-78]
+<= selenium-server-node-instance
+capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-78:version},firefox_binary=${firefox-wrapper-78:location}
+java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
+port = 7780
+
 [selenium-server-node-instance-chromium-69]
 <= selenium-server-node-instance
 capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-69:version},chrome_binary=${chromium-wrapper-69:location}
 java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-2.41:location}
-port = 7780
+port = 7781
+
+[selenium-server-node-instance-chromium-91]
+<= selenium-server-node-instance
+capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-91:version},chrome_binary=${chromium-wrapper-91:location}
+java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-91:location}
+port = 7782
 
 
 [selenium-server-admin-password]
@@ -152,7 +164,7 @@ template = inline:
       $${:path-hub}
     }
   }
-ip = $${instance-parameter:ipv6-random}
+ip = $${slap-configuration:ipv6-random}
 hostname = [$${:ip}]
 port = 9443
 path-admin = /grid/console
@@ -180,7 +192,7 @@ recipe = slapos.cookbook:userinfo
 recipe = slapos.cookbook:free_port
 minimum = 22222
 maximum = 22231
-ip = $${slap-network-information:global-ipv6}
+ip = $${slap-configuration:ipv6-random}
 hostname = $${:ip}
 
 [ssh-keygen-base]
@@ -251,7 +263,7 @@ recipe = plone.recipe.command
 stop-on-error = true
 location = $${buildout:directory}/.ssh
 authorized-keys-file = $${:location}/authorized_keys
-command = mkdir -p $${:location} && echo '$${instance-parameter:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
+command = mkdir -p $${:location} && echo '$${slap-configuration:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
 
 
 [promises]
@@ -264,27 +276,29 @@ instance-promises =
   $${selenium-server-node-firefox-52-listen-promise:name}
   $${selenium-server-node-firefox-60-listen-promise:name}
   $${selenium-server-node-firefox-68-listen-promise:name}
+  $${selenium-server-node-firefox-78-listen-promise:name}
   $${selenium-server-node-instance-chromium-69-listen-promise:name}
+  $${selenium-server-node-instance-chromium-91-listen-promise:name}
 
 
 [check-port-listening-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = $${:_buildout_section_name_}.py
 
 [sshd-listen-promise]
 <= check-port-listening-promise
-config-hostname = $${sshd-address:hostname}
+config-host = $${sshd-address:hostname}
 config-port = $${sshd-address:port}
 
 [selenium-server-frontend-listen-promise]
 <= check-port-listening-promise
-config-hostname = $${selenium-server-frontend-instance:ip}
+config-host = $${selenium-server-frontend-instance:ip}
 config-port = $${selenium-server-frontend-instance:port}
 
 [selenium-server-hub-listen-promise]
 <= check-port-listening-promise
-config-hostname = $${selenium-server-hub-instance:hostname}
+config-host = $${selenium-server-hub-instance:hostname}
 config-port = $${selenium-server-hub-instance:port}
 
 # Promise waiting for all nodes to be registered
@@ -294,8 +308,8 @@ module = check_command_execute
 name = $${:_buildout_section_name_}.py
 config-command =
   $${selenium-server-check-nodes-registered:rendered} $${selenium-server-hub-instance:api-url} $${:expected-node-count}
-# We have 4 nodes with 3 slots each
-expected-node-count = 12
+# We have 6 nodes with 3 slots each
+expected-node-count = 18
 
 [selenium-server-check-nodes-registered]
 recipe = slapos.recipe.template:jinja2
@@ -313,24 +327,34 @@ template =
 
 [selenium-server-node-firefox-52-listen-promise]
 <= check-port-listening-promise
-config-hostname = $${selenium-server-node-instance-firefox-52:hostname}
+config-host = $${selenium-server-node-instance-firefox-52:hostname}
 config-port = $${selenium-server-node-instance-firefox-52:port}
 
 [selenium-server-node-firefox-60-listen-promise]
 <= check-port-listening-promise
-config-hostname = $${selenium-server-node-instance-firefox-60:hostname}
+config-host = $${selenium-server-node-instance-firefox-60:hostname}
 config-port = $${selenium-server-node-instance-firefox-60:port}
 
 [selenium-server-node-firefox-68-listen-promise]
 <= check-port-listening-promise
-config-hostname = $${selenium-server-node-instance-firefox-68:hostname}
+config-host = $${selenium-server-node-instance-firefox-68:hostname}
 config-port = $${selenium-server-node-instance-firefox-68:port}
 
+[selenium-server-node-firefox-78-listen-promise]
+<= check-port-listening-promise
+config-host = $${selenium-server-node-instance-firefox-78:hostname}
+config-port = $${selenium-server-node-instance-firefox-78:port}
+
 [selenium-server-node-instance-chromium-69-listen-promise]
 <= check-port-listening-promise
-config-hostname = $${selenium-server-node-instance-chromium-69:hostname}
+config-host = $${selenium-server-node-instance-chromium-69:hostname}
 config-port = $${selenium-server-node-instance-chromium-69:port}
 
+[selenium-server-node-instance-chromium-91-listen-promise]
+<= check-port-listening-promise
+config-host = $${selenium-server-node-instance-chromium-91:hostname}
+config-port = $${selenium-server-node-instance-chromium-91:port}
+
 
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish
@@ -347,7 +371,7 @@ ssh-fingerprint = $${ssh-key-fingerprint:fingerprint}
 run-node-command = PORT=7999 bash -c 'trap '"'"'kill -TERM $SSHPID; wait $SSHPID '"'"' TERM INT; ssh -L 4444:$${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port} -R $PORT:127.0.0.1:$PORT -p $${sshd-service:port} $${sshd-service:username}@$${sshd-service:ip} & SSHPID=$!; java -jar selenium-server-standalone-3.14.0.jar -role node -host 127.0.0.1 -port $PORT ; wait "$SSHPID"'
 
 
-[instance-parameter]
+[slap-configuration]
 recipe = slapos.cookbook:slapconfiguration
 computer = $${slap-connection:computer-id}
 partition = $${slap-connection:partition-id}

software/seleniumserver/instance.cfg.in

@@ -6,15 +6,21 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
+[dynamic-template-selenium]
+recipe = slapos.recipe.template:jinja2
+template = ${template-selenium:output}
+rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
+filename = instance-selenium.cfg
+context =
+    key partition_ipv6 slap-configuration:ipv6-random
+
 [switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = ${template-selenium:output}
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = $${:default}
+default = dynamic-template-selenium:rendered
 
-[slap-connection]
-# part to migrate to new - separated words
-computer-id = $${slap_connection:computer_id}
-partition-id = $${slap_connection:partition_id}
-server-url = $${slap_connection:server_url}
-software-release-url = $${slap_connection:software_release_url}
-key-file = $${slap_connection:key_file}
-cert-file = $${slap_connection:cert_file}
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}

software/seleniumserver/software.cfg

@@ -73,5 +73,6 @@ post-install =
 post-install =
   ${symlink-extra-fonts-to-firefox-fonts-dir:install}
 
-[versions]
-plone.recipe.command = 1.1
+[firefox-78]
+post-install =
+  ${symlink-extra-fonts-to-firefox-fonts-dir:install}

software/seleniumserver/test/test.py

@@ -426,6 +426,14 @@ class TestFirefox68(
   desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='68.0.2esr')
   user_agent = 'Gecko/20100101 Firefox/68.0'
 
+class TestFirefox78(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
+  desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='78.1.0esr')
+  user_agent = 'Gecko/20100101 Firefox/78.0'
+
 
 class TestChrome69(
     BrowserCompatibilityMixin,
@@ -434,3 +442,12 @@ class TestChrome69(
 ):
   desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
   user_agent = 'Chrome/69.0.3497.0'
+
+
+class TestChrome91(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
+  desired_capabilities = dict(DesiredCapabilities.CHROME, version='91.0.4472.114')
+  user_agent = 'Chrome/91.0.4472.0'

software/slapos-master/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = c341609c8d0cfe240a3a2ae35bd713b6
+md5sum = 84f099cc9852c4f53a075dccbb3880f0
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 9c67c77eab5195b2674e340fb44c48a2
+md5sum = c7c0bb9abbd0f8cc6c7956d83a61c4b3
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -2,6 +2,7 @@
 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
+{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
 {% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%}
 {#
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
@@ -38,7 +39,7 @@ mode = 644
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
 {% set zope_virtualhost_monster_backend_dict = {} %}
-{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
+{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
 {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
 {% for family_name, parameter_id_list in sorted(
   slapparameter_dict['zope-family-dict'].iteritems()) -%}
@@ -59,19 +60,19 @@ mode = 644
 {%     set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %}
 {%     if test_runner_address_list -%}
 {%       set test_runner_backend_mapping = {} %}
-{%       set test_runner_apache_url_list = [] %}
+{%       set test_runner_balancer_url_list = [] %}
 {%       set test_runner_external_port = next_port() %}
 {%       for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %}
 {%         do test_runner_backend_mapping.__setitem__(
                 'unit_test_' ~ i,
                 'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
-{%         do test_runner_apache_url_list.append(
+{%         do test_runner_balancer_url_list.append(
                 'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
 {%       endfor %}
 {%       do zope_virtualhost_monster_backend_dict.__setitem__(
               (ipv4, test_runner_external_port),
               ( ssl_authentication, test_runner_backend_mapping ) ) -%}
-{%       do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
+{%       do test_runner_url_dict.__setitem__(family_name, test_runner_balancer_url_list) -%}
 {%     endif -%}
 {%   endfor -%}
 
@@ -123,6 +124,23 @@ caucase-key = ${directory:apache-conf}/apache-caucase.pem
 ca-cert =  ${directory:apache-conf}/ca.crt
 crl = ${directory:apache-conf}/crl.pem
 
+[simplefile]
+< = jinja2-template-base
+template = inline:{{ '{{ content }}' }}
+
+{% macro simplefile(section_name, file_path, content, mode='') -%}
+{%   set content_section_name = section_name ~ '-content' -%}
+[{{  content_section_name }}]
+content = {{ dumps(content) }}
+
+[{{  section(section_name) }}]
+< = simplefile
+rendered = {{ file_path }}
+context = key content {{content_section_name}}:content
+mode = {{ mode }}
+{%- endmacro %}
+
+
 [apache-ssl]
 {% if ssl_parameter_dict.get('key') -%}
 key = ${apache-ssl-key:rendered}
@@ -185,9 +203,9 @@ input = inline:
 [{{ section('apache-promise') }}]
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = apache.py
-config-hostname = {{ ipv4 }}
+config-host = {{ ipv4 }}
 config-port = {{ apache_dict.values()[0][0] }}
 
 [{{ section('publish') }}]
@@ -273,8 +291,6 @@ template = inline:
   {% endfor %}
 
 [apachedex-parameters]
-# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
-# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
 apache-log-list = ${apache-conf-parameter-dict:access-log}
 configuration = ${monitor-apachedex-report-config:rendered}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}

software/slapos-master/instance-erp5.cfg.in

@@ -221,7 +221,10 @@ return =
 {%- if test_runner_enabled %}
   test-runner-address-list
 {% endif %}
-{% set bt5_default_list = 'erp5_full_text_mroonga_catalog slapos_configurator' -%}
+{% set bt5_default_list = [
+  'erp5_full_text_mroonga_catalog',
+  'slapos_configurator',
+] -%}
 {% if has_jupyter -%}
 {%   do bt5_default_list.append('erp5_data_notebook') -%}
 {% endif -%}
@@ -254,6 +257,7 @@ config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
 config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
 config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
 config-wcfs_enable = {{ dumps(wcfs_enable) }}
+config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
 software-type = zope
 
 {% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}

software/slapos-master/test/test/__init__.py

@@ -30,11 +30,21 @@ import os
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
-setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+
+_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
 
 
+setup_module_executed = False
+def setUpModule():
+  # slapos.testing.testcase's only need to be executed once
+  global setup_module_executed
+  if not setup_module_executed:
+    _setUpModule()
+  setup_module_executed = True
+
+
 class ERP5InstanceTestCase(SlapOSInstanceTestCase):
   """ERP5 base test case
   """

software/slapos-master/test/test/test_erp5.py

@@ -41,31 +41,50 @@ import requests
 
 from . import ERP5InstanceTestCase
 from . import setUpModule
-setUpModule  # pyflakes
+setUpModule # pyflakes
 
 
 class TestPublishedURLIsReachableMixin(object):
   """Mixin that checks that default page of ERP5 is reachable.
   """
-  def _checkERP5IsReachable(self, url):
+
+  def _checkERP5IsReachable(self, base_url, site_id, verify):
+    # We access ERP5 trough a "virtual host", which should make
+    # ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
+    # as base.
+    virtual_host_url = urlparse.urljoin(
+        base_url,
+        '/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
+        .format(site_id))
+
     # What happens is that instantiation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
-    # instantiated, but zope is still busy creating the site and haproxy
-    # replies with 503 Service Unavailable when zope is not started yet, with
-    # 404 when erp5 site is not created, with 500 when mysql is not yet
-    # reachable, so we retry in a loop until we get a succesful response.
-    for i in range(1, 60):
-      # XXX can we get CA from caucase already ?
-      r = requests.get(url, verify=False)
-      if r.status_code != requests.codes.ok:
-        delay = i * 2
-        self.logger.warn(
-          "ERP5 was not available, sleeping for %ds and retrying", delay)
-        time.sleep(delay)
-        continue
-      r.raise_for_status()
-      break
-
+    # instantiated, but zope is still busy creating the site and haproxy replies
+    # with 503 Service Unavailable when zope is not started yet, with 404 when
+    # erp5 site is not created, with 500 when mysql is not yet reachable, so we
+    # configure this requests session to retry.
+    # XXX we should probably add a promise instead
+    session = requests.Session()
+    session.mount(
+        base_url,
+        requests.adapters.HTTPAdapter(
+            max_retries=requests.packages.urllib3.util.retry.Retry(
+                total=60,
+                backoff_factor=.5,
+                status_forcelist=(404, 500, 503))))
+
+    r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
+    self.assertEqual(r.status_code, requests.codes.found)
+    # access on / are redirected to login form, with virtual host preserved
+    self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
+
+    # login page can be rendered and contain the text "ERP5"
+    r = session.get(
+        urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
+        verify=verify,
+        allow_redirects=False,
+    )
+    self.assertEqual(r.status_code, requests.codes.ok)
     self.assertIn("ERP5", r.text)
 
   def test_published_family_default_v6_is_reachable(self):
@@ -73,15 +92,18 @@ class TestPublishedURLIsReachableMixin(object):
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
+      param_dict['family-default-v6'],
+      param_dict['site-id'],
+      verify=False)
 
   def test_published_family_default_v4_is_reachable(self):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
-
+      param_dict['family-default'],
+      param_dict['site-id'],
+      verify=False)
 
 class TestDefaultParameters(
   ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
@@ -99,6 +121,30 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wsgi': False})}
 
+class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test ERP5 Jupyter notebook
+  """
+  __partition_reference__ = 'jupyter'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'jupyter': {'enable': True}})}
+
+  def test_jupyter_notebook_is_reachable(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+
+    self.assertEqual(
+      'https://[%s]:8888/tree' % self._ipv6_address,
+      param_dict['jupyter-url']
+    )
+
+    result = requests.get(
+      param_dict['jupyter-url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [requests.codes.found, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
 
 class TestApacheBalancerPorts(ERP5InstanceTestCase):
   """Instantiate with two zope families, this should create for each family:
@@ -132,19 +178,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     self.assertTrue(parsed.port)
 
   def test_published_family_parameters(self):
-    # when we request two families, we have two published family-{family_name}
-    # URLs
+    # when we request two families, we have two published family-{family_name} URLs
     param_dict = self.getRootPartitionConnectionParameterDict()
     for family_name in ('family1', 'family2'):
       self.checkValidHTTPSURL(
           param_dict['family-{family_name}'.format(family_name=family_name)])
       self.checkValidHTTPSURL(
-          param_dict['family-{family_name}-v6'.format(
-            family_name=family_name)])
+          param_dict['family-{family_name}-v6'.format(family_name=family_name)])
 
   def test_published_test_runner_url(self):
-    # each family's also a list of test test runner URLs, by default 3 per
-    # family
+    # each family's also a list of test test runner URLs, by default 3 per family
     param_dict = self.getRootPartitionConnectionParameterDict()
     for family_name in ('family1', 'family2'):
       family_test_runner_url_list = param_dict[
@@ -154,8 +197,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
         self.checkValidHTTPSURL(url)
 
   def test_zope_listen(self):
-    # we requested 3 zope in family1 and 5 zopes in family2, we should have 8
-    # zope running.
+    # we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
     self.assertEqual(
@@ -190,46 +232,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     ])
 
 
-class TestDisableTestRunner(
-  ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
-  """Test ERP5 can be instantiated without test runner.
-  """
-  __partition_reference__ = 'distr'
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {'_': json.dumps({'test-runner': {'enabled': False}})}
-
-  def test_no_runUnitTestScript(self):
-    """No runUnitTest script should be generated in any partition.
-    """
-    # self.computer_partition_root_path is the path of root partition.
-    # we want to assert that no scripts exist in any partition.
-    bin_programs = map(
-      os.path.basename,
-      glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
-
-    self.assertTrue(bin_programs)  # just to check the glob was correct.
-    self.assertNotIn('runUnitTest', bin_programs)
-    self.assertNotIn('runTestSuite', bin_programs)
-
-  def test_no_apache_testrunner_port(self):
-    # Apache only listen on two ports, there is no apache ports allocated for
-    # test runner
-    with self.slap.instance_supervisor_rpc as supervisor:
-      all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [p for p in all_process_info if p['name'] == 'apache']
-    apache_process = psutil.Process(process_info['pid'])
-    self.assertEqual(
-        sorted([socket.AF_INET, socket.AF_INET6]),
-        sorted(
-            c.family
-            for c in apache_process.connections()
-            if c.status == 'LISTEN'
-        ))
-
-
-class TestZopeNodeParameterOverride(
-  ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test override zope node parameters
   """
   __partition_reference__ = 'override'
@@ -244,7 +247,7 @@ class TestZopeNodeParameterOverride(
         "server": {},
         "cache-size-bytes": "20MB",
         "cache-size-bytes!": [
-          ("bb-0", 1 << 20),
+          ("bb-0", 1<<20),
           ("bb-.*", "500MB"),
         ],
         "pool-timeout": "10m",
@@ -315,7 +318,7 @@ class TestZopeNodeParameterOverride(
     partition = self.getComputerPartitionPath('zope-bb')
     for zope in xrange(5):
       checkConf({
-          "cache-size-bytes": "500MB" if zope else 1 << 20,
+          "cache-size-bytes": "500MB" if zope else 1<<20,
         }, {
           "cache-size": None,
         })

software/slapos-master/test/test/test_wcfs.py

+# Copyright (C) 2021  Nexedi SA and Contributors.
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import json
+import os.path
+import unittest
+
+from slapos.grid.utils import md5digest
+
+from . import ERP5InstanceTestCase, setUpModule as _setUpModule
+from .test_erp5 import TestPublishedURLIsReachableMixin
+
+# skip tests when software release is built with wendelin.core 1.
+def setUpModule():
+  _setUpModule()
+
+  cls = ERP5InstanceTestCase
+  if not os.path.exists(
+    os.path.join(
+      cls.slap.software_directory,
+      md5digest(cls.getSoftwareURL()),
+      'bin', 'wcfs')):
+    raise unittest.SkipTest("built with wendelin.core 1")
+
+
+class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test Wendelin Core File System
+  """
+  __partition_reference__ = 'wcfs'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'wcfs': {'enable': True}})}
+
+  def test_wcfs_accessible(self):
+    """Verify that wcfs filesystem is basically accessible.
+
+       - we can read .wcfs/zurl
+       - its content is equal to published `serving-zurl`
+    """
+    zurl = json.loads(
+             self.getComputerPartition('wcfs').getConnectionParameter('_')
+           )['serving-zurl']
+
+    mntpt = lookupMount(zurl)
+    zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
+    self.assertEqual(zurl_, zurl)
+
+
+# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
+def lookupMount(zurl):
+  for line in readfile('/proc/mounts').splitlines():
+    # <zurl> <mountpoint> fuse.wcfs ...
+    zurl_, mntpt, typ, _ = line.split(None, 3)
+    if typ != 'fuse.wcfs':
+      continue
+    if zurl_ == zurl:
+      return mntpt
+  raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
+
+# readfile returns content of file @path.
+def readfile(path):
+  with open(path, 'r') as f:
+    return f.read()

software/slapos-sr-testing/software-py3.cfg

@@ -13,6 +13,7 @@ extra-eggs +=
 [template]
 extra =
 # The following list is for SR whose buildout runs only with Python 3.
+  galene ${slapos.test.galene-setup:setup}
   helloworld ${slapos.test.helloworld-setup:setup}
   jupyter ${slapos.test.jupyter-setup:setup}
   monitor ${slapos.test.monitor-setup:setup}
@@ -20,3 +21,4 @@ extra =
   powerdns ${slapos.test.powerdns-setup:setup}
   proftpd ${slapos.test.proftpd-setup:setup}
   repman ${slapos.test.repman-setup:setup}
+  restic-rest-server ${slapos.test.restic_rest_server-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -5,6 +5,7 @@ extends =
   ../../component/curl/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/git/buildout.cfg
+  ../../component/pycurl/buildout.cfg
   ../../component/faketime/buildout.cfg
   ../../component/pillow/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
@@ -98,6 +99,11 @@ setup = ${slapos-repository:location}/software/proftpd/test/
 egg = slapos.test.re6stnet
 setup = ${slapos-repository:location}/software/re6stnet/test/
 
+[slapos.test.restic_rest_server-setup]
+<= setup-develop-egg
+egg = slapos.test.restic_rest_server
+setup = ${slapos-repository:location}/software/restic-rest-server/test/
+
 [slapos.test.seleniumserver-setup]
 <= setup-develop-egg
 egg = slapos.test.seleniumserver
@@ -183,6 +189,11 @@ setup = ${slapos-repository:location}/software/repman/test/
 egg = slapos.test.jscrawler
 setup = ${slapos-repository:location}/software/jscrawler/test/
 
+[slapos.test.galene-setup]
+<= setup-develop-egg
+egg = slapos.test.galene
+setup = ${slapos-repository:location}/software/galene/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -204,6 +215,7 @@ extra-eggs =
   ${lxml-python:egg}
   ${slapos.core-setup:egg}
   ${pillow-python:egg}
+  ${pycurl:egg}
   erp5.util
   ${python-pynacl:egg}
   ${python-cryptography:egg}
@@ -225,6 +237,7 @@ extra-eggs =
   ${slapos.test.powerdns-setup:egg}
   ${slapos.test.proftpd-setup:egg}
   ${slapos.test.re6stnet-setup:egg}
+  ${slapos.test.restic_rest_server-setup:egg}
   ${slapos.test.seleniumserver-setup:egg}
   ${slapos.test.slaprunner-setup:egg}
   ${slapos.test.jupyter-setup:egg}
@@ -237,6 +250,7 @@ extra-eggs =
   ${slapos.test.repman-setup:egg}
   ${slapos.test.caucase-setup:egg}
   ${slapos.test.jscrawler-setup:egg}
+  ${slapos.test.galene-setup:egg}
   ${slapos.test.html5as-setup:egg}
   ${slapos.test.html5as-base-setup:egg}
   ${slapos.test.fluentd-setup:egg}

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 9f367deb7597957e7108bee719b78bcc
+md5sum = ebdecea5c1653ed752e06fbc8be7e6b2
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -622,16 +622,16 @@ monitor-password = $${monitor-publish-parameters:monitor-password}
 
 [slaprunner-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = slaprunner.py
-config-hostname = $${slaprunner:ipv6}
+config-host = $${slaprunner:ipv6}
 config-port = $${slaprunner:runner_port}
 
 [runner-sshd-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = runner-sshd.py
-config-hostname = $${slap-network-information:global-ipv6}
+config-host = $${slap-network-information:global-ipv6}
 config-port = $${runner-sshd-port:port}
 
 [slap-parameter]
@@ -866,9 +866,9 @@ log = $${runnerdirectory:home}/instance/*/.slapgrid/log/instance.log $${runnerdi
 
 [supervisord-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = supervisord.py
-config-hostname = $${slaprunner:ipv4}
+config-host = $${slaprunner:ipv4}
 config-port = $${supervisord:port}
 
 [slapos-supervisord-promise]
@@ -879,9 +879,9 @@ config-command = ${buildout:bin-directory}/slapos node supervisorctl --cfg=$${sl
 
 [slapos-proxy-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = slaproxy.py
-config-hostname = $${slaprunner:ipv4}
+config-host = $${slaprunner:ipv4}
 config-port = $${slaprunner:proxy_port}
 
 # XXX Monitor

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 11274189a4492204e046fef45fdd809c
+md5sum = 557dec61b7bf58601051575234e7fd05
 
 [instance]
 _update_hash_filename_ = instance.cfg.in

software/theia/instance-theia.cfg.jinja.in

@@ -77,16 +77,16 @@ instance-promises =
 
 [theia-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = $${:_buildout_section_name_}.py
-config-hostname = $${theia-instance:ip}
+config-host = $${theia-instance:ip}
 config-port = $${theia-instance:port}
 
 [frontend-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = $${:_buildout_section_name_}.py
-config-hostname = $${frontend-instance:ip}
+config-host = $${frontend-instance:ip}
 config-port = $${frontend-instance:port}
 
 [remote-frontend-url-available-promise]
@@ -107,10 +107,10 @@ config-check-secure = 1
 
 [slapos-standalone-listen-promise]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 # XXX promise plugins can not contain "slapos" in their names
 name = standalone-listen-promise.py
-config-hostname = $${slapos-standalone-instance:hostname}
+config-host = $${slapos-standalone-instance:hostname}
 config-port = $${slapos-standalone-instance:port}
 
 [slapos-autorun-promise]

software/theia/test/dummy/software.cfg

@@ -16,4 +16,4 @@ init =
     json.dump(dict(os.environ), f)
 
 [versions]
-slapos.recipe.build = 0.46
+slapos.recipe.build = 0.47

software/turnserver/buildout.hash.cfg

@@ -15,12 +15,12 @@
 
 [instance-cfg]
 filename = instance.cfg.in
-md5sum = 776c7de2054f78ba79382c22d85018be
+md5sum = b43d5e8d1fc2d0eeb54f91cefe6a5bae
 
 [template-turnserver]
 filename = instance-turnserver.cfg.jinja2.in
-md5sum = df8768f165036dbe0435bd1678b9deb3
+md5sum = 7af3318d7249e9afe22436d9fe200159
 
 [template-insecure-turnserver]
 filename = instance-insecure-turnserver.cfg.jinja2.in
-md5sum = e88678b9f322251a201860a13f4db6b8
+md5sum = 3db65c3a16eb76ab438ac3817d1a5fea

software/turnserver/instance-insecure-turnserver.cfg.jinja2.in

 {% set part_list = [] -%}
-{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
+{%- set parameter = dict(default_parameter_dict, **slapparameter_dict) %}
+{%- set server_name = parameter['server-name'] %}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -17,9 +18,11 @@ plugins = ${:etc}/plugin
 recipe = slapos.cookbook:generate.password
 bytes = 8
 
-{% set turn_port = slapparameter_dict.get('port', 3478) -%}
-{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
-{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
+{% set turn_port = parameter['port'] -%}
+{% set turn_tls_port = parameter['tls-port'] -%}
+# listening-ip parameter is mandatory
+{% set listening_ip = slapparameter_dict['listening-ip'] -%}
+
 [turnserver-config]
 recipe = collective.recipe.template
 user = nxdturn
@@ -27,11 +30,11 @@ input = inline:
   listening-port={{ turn_port }}
   lt-cred-mech
   realm={{ server_name }}
-{% if slapparameter_dict.get('external-ip', '') %}
-  external-ip={{ slapparameter_dict['external-ip'] }}
+{% if parameter['external-ip'] %}
+  external-ip={{ parameter['external-ip'] }}
 {% endif %}
   fingerprint
-  listening-ip={{ listining_ip }}
+  listening-ip={{ listening_ip }}
   server-name={{ server_name }}
   no-stdout-log
   simple-log
@@ -54,9 +57,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [promise-check-turnserver-port]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = turnserver-port-listening.py
-config-hostname = {{ listining_ip }}
+config-host = {{ listening_ip }}
 config-port = {{ turn_port }}
 
 [publish-connection-information]

software/turnserver/instance-turnserver.cfg.jinja2.in

-{% set part_list = [] -%}
-{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
+{%- set part_list = [] -%}
+{%- set parameter = dict(default_parameter_dict, **slapparameter_dict) %}
+{%- set server_name = parameter['server-name'] %}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -31,9 +32,9 @@ mode = {{ mode }}
 {% do part_list.append(section_name) -%}
 {%- endmacro %}
 
-{% if slapparameter_dict.get('ssl-key') and slapparameter_dict.get('ssl-crt') -%}
-{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', slapparameter_dict.get('ssl-crt')) }}
-{{ simplefile('ssl-key', '${turnserver-ssl:key}', slapparameter_dict.get('ssl-key'), 600) }}
+{% if parameter['ssl-key'] and parameter['ssl-crt'] -%}
+{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', parameter['ssl-crt']) }}
+{{ simplefile('ssl-key', '${turnserver-ssl:key}', parameter['ssl-key'], 600) }}
 {% else -%}
 {%  do part_list.append('gen-certificate') -%}
 [gen-certificate]
@@ -57,9 +58,9 @@ secret-file = ${directory:etc}/.turnsecret
 command =
   if [ ! -s "${:secret-file}" ]; then
     cat <<EOF > ${:secret-file}
-    [turnserver]
-    secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
-    EOF
+  [turnserver]
+  secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
+  EOF
   fi
   chmod 600 ${:secret-file}
 
@@ -68,9 +69,11 @@ recipe = slapos.cookbook:zero-knowledge.read
 file-path = ${gen-secret:secret-file}
 secret =
 
-{% set turn_port = slapparameter_dict.get('port', 3478) -%}
-{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
-{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
+{% set turn_port = parameter['port'] -%}
+{% set turn_tls_port = parameter['tls-port'] -%}
+# listening-ip parameter is mandatory
+{% set listening_ip = slapparameter_dict['listening-ip'] -%}
+
 [turnserver-config]
 recipe = collective.recipe.template
 input = inline:
@@ -80,9 +83,9 @@ input = inline:
   lt-cred-mech
   use-auth-secret
   static-auth-secret=${read-secret:secret}
-  listening-ip={{ listining_ip }}
-{% if slapparameter_dict.get('external-ip', '') %}
-  external-ip={{ slapparameter_dict['external-ip'] }}
+  listening-ip={{ listening_ip }}
+{% if parameter['external-ip'] %}
+  external-ip={{ parameter['external-ip'] }}
 {% endif %}
   server-name={{ server_name }}
   realm={{ server_name }}
@@ -120,16 +123,16 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [promise-check-turnserver-port]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = turnserver-port-listening.py
-config-hostname = {{ listining_ip }}
+config-host = {{ listening_ip }}
 config-port = {{ turn_port }}
 
 [promise-check-turnserver-tls-port]
 <= monitor-promise-base
-module = check_port_listening
+module = check_socket_listening
 name = turnserver-tls-port-listening.py
-config-hostname = {{ listining_ip }}
+config-host = {{ listening_ip }}
 config-port = {{ turn_tls_port }}
 
 [publish-connection-information]