Merge branch 'master' into master+ZODB4-wc2

* master: (56 commits)
  golang: v↑ go1.17  (1.17.3 -> 1.17.6)
  golang: v↑ go1.16  (1.16.10 -> 1.16.13)
  stack/erp5: Shorten haproxy socket path
  software/erp5/test: fix flaky test_activity_processing
  component/apache: version up 2.4.52, including security fixes.
  component/trafficserver: version up 9.1.1.
  stack/slapos: version up lxml 4.6.5, including security fixes.
  ninja: simplify configure step
  consul, packer, phantomjs: enable shared
  version up: zc.buildout 2.7.1+slapos015, slapos.recipe.build 0.53, slapos.recipe.cmmi 0.19
  version up: CMake 3.22.1
  component/userhosts: use github.com/figiel/hosts for compatibility with recent libc
  software/erp5/upgrade_test: backport new fixes in old software
  software/erp5/upgrade_test: skip software checks
  component/coturn: build with slapos libnsl
  component/apache-php: build with slapos libnsl
  repman: improve parameters used for cluster
  component/theia: version up 1.21.0
  software/theia: update python packages in python-language-server-requirements
  component/theia: update plugins with current versions
  ...

component/apache-php/buildout.cfg

@@ -23,6 +23,7 @@ extends =
   ../imagemagick/buildout.cfg
   ../icu/buildout.cfg
   ../openssl/buildout.cfg
+  ../libnsl/buildout.cfg
 
 [php-redis]
 recipe = slapos.recipe.cmmi
@@ -112,7 +113,7 @@ environment =
   PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${libzip:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
   CPPFLAGS=-I${libzip:location}/include
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib -L${libnsl:location}/lib -Wl,-rpath -Wl,${libnsl:location}/lib 
   TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
   HOME=${apache:location}
 

component/apache/buildout.cfg

@@ -39,9 +39,9 @@ configure-options =
 [apache]
 recipe = slapos.recipe.cmmi
 shared = true
-version = 2.4.51
+version = 2.4.52
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = d2793fc1c8cb8ba355cee877d1f2d46d
+md5sum = a94ae42b84309d5ef6e613ae825b92fa
 configure-options = --disable-static
                     --enable-authn-alias
                     --enable-bucketeer

component/chromedriver/buildout.cfg

@@ -15,37 +15,29 @@ parts =
 
 [chromedriver-wrapper-91]
 <= chromedriver-wrapper
-wrapper-name = chromedriver-91
-part = ${chromedriver-91:location}
+part = chromedriver-91
 
 [chromedriver-wrapper-2.41]
 <= chromedriver-wrapper
-wrapper-name = chromedriver-2.41
-part = ${chromedriver-2.41:location}
+part = chromedriver-2.41
 
 [chromedriver-wrapper]
 # generate a wrapper named ${:wrapper-name} setting $LD_LIBRARY_PATH
 # to run chromedriver installed from ${:part}
-wrapper-name = chromedriver
-part = ${chromedriver:location}
+part = chromedriver
 
 recipe = slapos.recipe.build
-location = ${buildout:bin-directory}/${:wrapper-name}
+location = ${buildout:bin-directory}/${:part}
+init =
+  self.buildout[options['part']]
 install =
   import os
-  chromedriver = options['part']
-  part = self.buildout[os.path.basename(chromedriver)]
+  part = self.buildout[options['part']]
   with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
-  cd {}
-  export LD_LIBRARY_PATH=$PWD:{}
-  export PATH={}:$PATH
-  exec ./chromedriver "$@"
-  """.format(
-    chromedriver,
-    ':'.join(part['library'].split()),
-    ':'.join(part['path'].split()),
-  ))
+  export LD_LIBRARY_PATH=%s
+  exec %s/chromedriver "$@"
+  """ % (':'.join(part['library'].split()), part['location']))
     os.fchmod(f.fileno(), 0o755)
 
 
@@ -67,18 +59,13 @@ md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
 [chromedriver-download]
 # Installs chromedriver ${version}.
 # This chromedriver is not usable directly, it needs a wrapper.
-version =
-md5sum-x86_64 =
-
-recipe = slapos.recipe.build
-x86_64-linux-gnu = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_linux64.zip ${:md5sum-x86_64}
+recipe = slapos.recipe.build:download-unpacked
+url = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_${:_url}.zip
 library =
   ${nss:location}/lib
   ${nspr:location}/lib
   ${libX11:location}/lib
-path =
-install =
-  import os, shutil
-  extract_dir = self.extract(self.download(*options[multiarch()].split()))
-  os.mkdir(location)
-  shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
+
+[chromedriver-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux64
+md5sum = ${:md5sum-x86_64}

component/chromium/buildout.cfg

@@ -11,7 +11,6 @@
 # in the system.
 extends =
   ../alsa/buildout.cfg
-  ../coreutils/buildout.cfg
   ../cups/buildout.cfg
   ../dbus/buildout.cfg
   ../findutils/buildout.cfg
@@ -37,30 +36,30 @@ parts =
 
 
 [chromium-wrapper]
-# Install a chromium wrapper named ${:wrapper-name} in buildout's bin-directory,
-# wrapping chromium installed in ${:part}
+# Install a chromium wrapper named ${:part} in buildout's bin-directory,
+# wrapping chromium installed in [${:part}]
 # This [chromium-wrapper] installs the default chromium with name `chromium` and
 # can also be used as a macro to install under a different name.
-wrapper-name = chromium
-part = ${chromium:location}
+part = chromium
 
 recipe = slapos.recipe.build
-location = ${buildout:bin-directory}/${:wrapper-name}
+location = ${buildout:bin-directory}/${:part}
+init =
+  self.buildout[options['part']]
 install =
   import os
-  chromium = options['part']
-  part = self.buildout[os.path.basename(chromium)]
+  part = self.buildout[options['part']]
   with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
-  cd {}
+  d=%s
   # this also needs a $FONTCONFIG_FILE set, otherwise system fonts will be used and if
   # no system fonts are available, chrome will refuse to start with this error:
   # FATAL:platform_font_linux.cc(83)] Check failed: InitDefaultFont(). Could not find the default font
-  export LD_LIBRARY_PATH=$PWD:{}
-  export PATH={}:$PATH
-  exec ./chrome --disable-setuid-sandbox --no-sandbox --disable-gpu $@
-  """.format(
-    chromium,
+  export LD_LIBRARY_PATH=$d:%s
+  export PATH=%s:$PATH
+  exec $d/chrome --disable-setuid-sandbox --no-sandbox --disable-gpu "$@"
+  """ % (
+    part['location'],
     ':'.join(part['library'].split()),
     ':'.join(part['path'].split()),
   ))
@@ -68,13 +67,11 @@ install =
 
 [chromium-wrapper-91]
 <= chromium-wrapper
-wrapper-name = chromium-91
-part = ${chromium-91:location}
+part = chromium-91
 
 [chromium-wrapper-69]
 <= chromium-wrapper
-wrapper-name = chromium-69
-part = ${chromium-69:location}
+part = chromium-69
 
 
 [chromium]
@@ -101,24 +98,7 @@ generation-x86_64 = 1532051976706023
 [chromium-download]
 # macro to download a binary build of chromium and generate a
 # wrapper as chrome-slapos in the part directory
-
-# the full version tag
-version =
-
-# needs the following URL parts:
-# revision is from the "base position"
-revision_x86-64 =
-# generation is in the final download URL
-generation-x86_64 =
-# this is the md5sum of the downloaded archive
-md5sum-x86_64 =
-
-
-recipe = slapos.recipe.build
-
-x86_64-linux-gnu = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
-
-
+recipe = slapos.recipe.build:download-unpacked
 library =
   ${atk:location}/lib
   ${at-spi2-atk:location}/lib
@@ -166,12 +146,7 @@ library =
   ${zlib:location}/lib
 path =
   ${fontconfig:location}/bin
-install =
-  url, md5sum = self.options[multiarch()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  self.copyTree(guessworkdir(extract_dir), location)
-  # XXX adjust some permissions
-  import os
-  os.system("'${findutils:location}/bin/find' '%s' -type d -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
-  os.system("'${findutils:location}/bin/find' '%s' -type f -executable -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
-  os.system("'${findutils:location}/bin/find' '%s' -type f -exec '${coreutils:location}/bin/chmod' a+r {} \\;" % location)
+
+[chromium-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+url = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media
+md5sum = ${:md5sum-x86_64}

component/cmake/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 [cmake]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://cmake.org/files/v3.21/cmake-3.21.3.tar.gz
-md5sum = c0feb5855604f68b09bdb3acb623619e
+url = https://cmake.org/files/v3.22/cmake-3.22.1.tar.gz
+md5sum = 83802be08c114bb34729300206488321
 environment =
   CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
   CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib

component/consul/buildout.cfg

@@ -4,7 +4,14 @@ parts =
 
 [consul]
 recipe = slapos.recipe.build:download-unpacked
+shared = true
 version = 0.8.3
-base = https://releases.hashicorp.com/consul/${:version}/consul_${:version}
-i386-linux-gnu   = ${:base}_linux_386.zip    dfdc0eedd79baab7e6bc56c1582fd02e
-x86_64-linux-gnu = ${:base}_linux_amd64.zip  d6bc0898ea37ae2198370a9e1978d1bb
+url = https://releases.hashicorp.com/consul/${:version}/consul_${:version}_${:_url}.zip
+
+[consul:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux_386
+md5sum = dfdc0eedd79baab7e6bc56c1582fd02e
+
+[consul:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux_amd64
+md5sum = d6bc0898ea37ae2198370a9e1978d1bb

component/coturn/buildout.cfg

@@ -3,6 +3,7 @@ extends =
   ../sqlite3/buildout.cfg
   ../openssl/buildout.cfg
   ../libevent/buildout.cfg
+  ../libnsl/buildout.cfg
 parts = coturn
 
 [coturn]
@@ -13,4 +14,4 @@ environment =
   PATH=${sqlite3:location}/bin:%(PATH)s
   CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
   CFLAGS=-I${libevent2:location}/include
-  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib -L${libnsl:location}/lib -Wl,-rpath -Wl,${libnsl:location}/lib

component/cpio/963304-remove-superfluous-declaration-of-program_name

+author	Sergey Poznyakoff <gray@gnu.org>
+
+https://git.savannah.gnu.org/cgit/cpio.git/commit/?id=641d3f489cf6238bb916368d4ba0d9325a235afb
+
+* src/global.c: Remove superfluous declaration of program_name
+
+diff --git a/src/global.c b/src/global.c
+index fb3abe9..acf92bc 100644
+--- a/src/global.c
++++ b/src/global.c
+@@ -184,9 +184,6 @@ unsigned int warn_option = 0;
+ /* Extract to standard output? */
+ bool to_stdout_option = false;
+ 
+-/* The name this program was run with.  */
+-char *program_name;
+-
+ /* A pointer to either lstat or stat, depending on whether
+    dereferencing of symlinks is done for input files.  */
+ int (*xstat) ();

component/cpio/buildout.cfg

 # GNU cpio copies files into or out of a cpio or tar archive. The archive can be another file on the disk, a magnetic tape, or a pipe.
 [buildout]
+extends =
+  ../patch/buildout.cfg
 parts = cpio
 
 [cpio]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/gnu/cpio/cpio-2.12.tar.bz2
-md5sum = 93eea9f07c0058c097891c73e4955456
+url = https://ftp.gnu.org/gnu/cpio/cpio-2.13.tar.bz2
+md5sum = f3438e672e3fa273a7dc26339dd1eed6
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/963304-remove-superfluous-declaration-of-program_name#1aadb39448eedaf49d03cf45f9d78ea4
+environment =
+  PATH=${patch:location}/bin:%(PATH)s

component/firefox/buildout.cfg

@@ -3,6 +3,7 @@ extends =
   ../alsa/buildout.cfg
   ../dbus/buildout.cfg
   ../fontconfig/buildout.cfg
+  ../fonts/buildout.cfg
   ../gtk-2/buildout.cfg
   ../gtk-3/buildout.cfg
   ../libffi/buildout.cfg
@@ -22,59 +23,52 @@ parts =
 
 
 [firefox-wrapper]
-# Install a firefox wrapper names ${:wrapper-name} in buildout's bin-directory,
-# wrapping firefox installed in ${:part}
+# Install a firefox wrapper named ${:part} in buildout's bin-directory,
+# wrapping firefox installed in [${:part}]
 # This [firefox-wrapper] installs the default firefox with name `firefox` and
 # can also be used as a macro to install under a different name.
-wrapper-name = firefox
-part = ${firefox:location}
+part = firefox
 
 recipe = slapos.recipe.build
-location = ${buildout:bin-directory}/${:wrapper-name}
+location = ${buildout:bin-directory}/${:part}
+fonts-conf = ${firefox-default-fonts-conf:rendered}
+init =
+  self.buildout[options['part']]
 install =
   import os
-  firefox = options['part']
-  part = self.buildout[os.path.basename(firefox)]
+  part = self.buildout[options['part']]
   with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
-  cd {}
-  export LD_LIBRARY_PATH=$PWD:{}
-  export PATH={}:$PATH
+  d=%s
+  export LD_LIBRARY_PATH=$d:%s
+  export PATH=%s:$PATH
   # BBB use a default fonts.conf for compatibility, but it's software instance
   # responsability to build a fonts.conf with the fonts they want.
-  [ -z $FONTCONFIG_FILE ] && export FONTCONFIG_FILE=${firefox-default-fonts-conf:rendered}
-  exec ./firefox "$@"
-  """.format(
-    firefox,
+  [ "$FONTCONFIG_FILE" ] || export FONTCONFIG_FILE=%s
+  exec $d/firefox "$@"
+  """ % (
+    part['location'],
     ':'.join(part['library'].split()),
     ':'.join(part['path'].split()),
+    options['fonts-conf'],
   ))
     os.fchmod(f.fileno(), 0o755)
 
 [firefox-wrapper-78]
 <= firefox-wrapper
-wrapper-name = firefox-78
-part = ${firefox-78:location}
+part = firefox-78
 
 [firefox-wrapper-68]
 <= firefox-wrapper
-wrapper-name = firefox-68
-part = ${firefox-68:location}
+part = firefox-68
 
 [firefox-wrapper-60]
 <= firefox-wrapper
-wrapper-name = firefox-60
-part = ${firefox-60:location}
+part = firefox-60
 
 [firefox-wrapper-52]
 <= firefox-wrapper
-wrapper-name = firefox-52
-part = ${firefox-52:location}
-
-[firefox-wrapper-51]
-<= firefox-wrapper
-wrapper-name = firefox-51
-part = ${firefox-51:location}
+part = firefox-52
 
 [firefox-default-fonts-conf]
 recipe = slapos.recipe.template:jinja2
@@ -124,20 +118,12 @@ version = 52.9.0esr
 i686-md5sum = 9aa18888b7812670208490609d75c9bc
 x86_64-md5sum = 9336d70f45070c743d08e5473b783a7a
 
-[firefox-51]
-<= firefox-download
-version = 51.0.1
-i686-md5sum = 9a5b67e9d759a1e4df004294a24b2b43
-x86_64-md5sum = bd93f2652d1d90d59ae462439a93c85f
-
-
 [firefox-download]
 recipe = slapos.recipe.build
 slapos_promise =
   file:firefox
 
-i386-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 ${:i686-md5sum}
-x86_64-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ${:x86_64-md5sum}
+url = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/${:_url}/en-US/firefox-${:version}.tar.bz2
 
 library =
   ${alsa:location}/lib
@@ -185,12 +171,19 @@ path =
   ${fontconfig:location}/bin
 
 install =
-  url, md5sum = options[multiarch()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  self.copyTree(guessworkdir(extract_dir), location)
+  self.copyTree(guessworkdir(self.extract(self.download())), location)
   ${:post-install}
 post-install =
 
+[firefox-download:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux-i686
+md5sum = ${:i686-md5sum}
+
+[firefox-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux-x86_64
+md5sum = ${:x86_64-md5sum}
+
+
 [geckodriver]
 # Current geckodriver installed as ${buildout:bin-directory}/geckodriver
 <= geckodriver-0.24.0
@@ -207,47 +200,26 @@ version = 0.22.0
 i686-md5sum = 6de7544753fda56fbaa8382dcac99aaa
 x86_64-md5sum = 81746200ce5841e00cabf3b8ea7db542
 
-[geckodriver-0.21.0]
-<= geckodriver-base
-version = 0.21.0
-i686-md5sum = 9fc1657dd1b94272d0cdb3b29ca80f79
-x86_64-md5sum = 6bc36d4fd4975e296cdb3fa3c5e26a41
-
-[geckodriver-0.19.0]
-<= geckodriver-base
-version = 0.19.0
-i686-md5sum = 07cd383c8aef8ea5ef194a506141afd6
-x86_64-md5sum = ca6935a72fd0527d15a78a17a35e56e8
-
-[geckodriver-0.17.0]
-<= geckodriver-base
-version = 0.17.0
-i686-md5sum = 79b1a158f96d29942a111c0905f1c807
-x86_64-md5sum = be18faeea6e7db9db6990d8667e2298f
-
 [geckodriver-0.16.1]
 <= geckodriver-base
 version = 0.16.1
 i686-md5sum = not not on github
 x86_64-md5sum = 57dfd55d4759d9878eb75b4c0123d00c
 
-[geckodriver-0.14.0]
-<= geckodriver-base
-version = 0.14.0
-i686-md5sum = b5836f5a944fe9f3ed1a67c7b342c6a7
-x86_64-md5sum = 4a185d3179862a35104603b9274452e7
-
-
 [geckodriver-base]
 # Installs geckodriver ${version}
 recipe = slapos.recipe.build
+url = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-${:_url}.tar.gz
 location = ${buildout:bin-directory}/${:_buildout_section_name_}
+install =
+  import os, shutil
+  extract_dir = self.extract(self.download(options['url'], options['md5sum']))
+  shutil.copy(os.path.join(extract_dir, 'geckodriver'), location)
 
-i386-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
-x86_64-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
+[geckodriver-base:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux32
+md5sum = ${:i686-md5sum}
 
-install =
-  import shutil
-  url, md5sum = options[multiarch()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  shutil.copy(extract_dir + '/geckodriver', location)
+[geckodriver-base:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux64
+md5sum = ${:x86_64-md5sum}

component/golang/buildout.cfg

@@ -59,8 +59,8 @@ environment-extra =
 
 [golang1.16]
 <= golang-common
-url = https://golang.org/dl/go1.16.10.src.tar.gz
-md5sum = 49f0a54f0bdcba297bac194d8dafe431
+url = https://golang.org/dl/go1.16.13.src.tar.gz
+md5sum = 1c076f952d9af57590a36fa7d36f695a
 
 # go1.16 needs go1.4 to bootstrap
 environment-extra =
@@ -68,8 +68,8 @@ environment-extra =
 
 [golang1.17]
 <= golang-common
-url = https://golang.org/dl/go1.17.3.src.tar.gz
-md5sum = 3ea82e5966275f405f0db4f52511bb6e
+url = https://golang.org/dl/go1.17.6.src.tar.gz
+md5sum = dc57f93f323e9f8189e5ffc1f223e346
 
 # go1.17 needs go1.4 to bootstrap
 environment-extra =

component/headless-chromium/buildout.cfg

@@ -45,9 +45,9 @@ gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 # called "src".
 name = src
 
-# There's nothing special about version 92.0.4515.107. It just happened
-# to be the current Chromium stable version at the time of writing.
-version = 92.0.4515.107
+# 96.0.4664.129 version is the latest stable version in December 2021.
+# Note that we need a version compiling without python2
+version = 96.0.4664.129
 
 
 [headless-chromium]

component/libnsl/buildout.cfg

+[buildout]
+extends =
+  ../xz-utils/buildout.cfg
+  ../libtirpc/buildout.cfg
+
+[libnsl]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/thkukuk/libnsl/releases/download/v1.3.0/libnsl-1.3.0.tar.xz
+md5sum = 9214f674bd0c2bcfdd6c1da0cadb061f
+make-options = 
+  CFLAGS=-I${libtirpc:location}/include/tirpc
+  LDFLAGS='-L${libtirpc:location}/lib -Wl,-rpath=${libtirpc:location}/lib -ltirpc'
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/libreoffice-bin/buildout.cfg

@@ -8,26 +8,15 @@ parts =
 
 [libreoffice-bin]
 recipe = slapos.recipe.build
-# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
 version = 5.2.4.2
-url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
-# supported architectures md5sums
-md5sum_x86 = 7a0b33a2d18f06143258c428c32de213
-md5sum_x86-64 = cbea6cd17063b5bcbe0cb32f7819f0cf
+url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/${:_url1}/LibreOffice_${:version}_${:_url2}_rpm.tar.gz
 
 # where office code can be found?
 officedir = libreoffice5.2
 install =
   import os
   import sys
-  arch = {
-    'i386-linux-gnu': ('x86', 'x86'),
-    'x86_64-linux-gnu': ('x86_64', 'x86-64'),
-  }[multiarch()]
-  url = options['url'] % arch
-  md5sum = options['md5sum_' + arch[1]]
-  extract_dir = self.extract(self.download(url, md5sum))
-  workdir = guessworkdir(extract_dir)
+  workdir = guessworkdir(self.extract(self.download()))
   storagedir = os.path.join(workdir, 'storage')
   os.mkdir(storagedir)
   rpmsdir = os.path.join(workdir, 'RPMS')
@@ -48,3 +37,13 @@ install =
 # helper binaries
 cpio = ${cpio:location}/bin/cpio
 rpm2cpio = ${rpm2cpio:target}
+
+[libreoffice-bin:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url1 = x86
+_url2 = Linux_x86
+md5sum = 7a0b33a2d18f06143258c428c32de213
+
+[libreoffice-bin:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url1 = x86_64
+_url2 = Linux_x86-64
+md5sum = cbea6cd17063b5bcbe0cb32f7819f0cf

component/libtirpc/buildout.cfg

+[buildout]
+parts = libtirpc
+
+[libtirpc]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://downloads.sourceforge.net/libtirpc/libtirpc-1.3.2.tar.bz2
+md5sum = cf4ca51f3fc401bea61c702c69171ab0
+configure-options = --disable-gssapi

component/nginx/buildout.cfg

@@ -61,8 +61,8 @@ mime = ${nginx:location}/conf/mime.types
 [nginx-push-stream-module]
 recipe = slapos.recipe.build:download-unpacked
 shared = true
-url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.4.0.tar.gz
-md5sum = d9cba621b8739e13bdb5e02b9425f205
+url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.5.5.tar.gz
+md5sum = 9bb5237a93c957130aaf7033ed3c0989
 
 [nginx-push-stream]
 <= nginx-common

component/ninja/buildout.cfg

 [buildout]
-extends =
-  ../defaults.cfg
 parts =
   ninja
 
@@ -10,7 +8,7 @@ shared = true
 version = 1.10.2
 url = https://github.com/ninja-build/ninja/archive/refs/tags/v${:version}.tar.gz
 md5sum = 639f75bc2e3b19ab893eaf2c810d4eb4
-configure-command = ${python:executable} configure.py
+configure-command = ${buildout:executable} configure.py
 configure-options = --bootstrap --verbose
 make-binary = true
 post-install =

component/p7zip/buildout.cfg

 [buildout]
+extends =
+  ../patch/buildout.cfg
 parts =
   p7zip
 
 [p7zip]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://downloads.sf.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
-version = 9.38.1
-md5sum = 6cba8402ccab2370d3b70c5e28b3d651
+url = https://downloads.sourceforge.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
+version = 16.02
+md5sum = a0128d661cfe7cc8c121e73519c54fbf
+patch-binary = ${patch:location}/bin/patch
+patch-options = -p1
+patch-base = https://sources.debian.org/data/main/p/p7zip/16.02%2Bdfsg-8/debian/patches/
+patches =
+  ${:patch-base}/12-CVE-2016-9296.patch#f0f5dc36d97d9a71a6da3adba717fbf2
+  ${:patch-base}/13-CVE-2017-17969.patch#8399bdc913cf8cf8194ecdb5487f48d9
+  ${:patch-base}/14-Fix-g%2B%2B-warning.patch#9db8fbfbc8d624669c095240621460bd
+  ${:patch-base}/15-Fix-FTBFS-gcc10.patch#50ebfd79c715d8b5d649e57a1da2ce34
 configure-command = rm -r DOC
-make-binary =
-make-options =
-make-targets = make -j1 7z install
-  DEST_HOME=@@LOCATION@@
+make-targets = 7z
+post-install = make install DEST_HOME=%(location)s

component/packer/buildout.cfg

@@ -4,7 +4,14 @@ parts =
 
 [packer]
 recipe = slapos.recipe.build:download-unpacked
+shared = true
 version = 0.7.5
-base = https://releases.hashicorp.com/packer/${:version}/packer_${:version}
-i386-linux-gnu   = ${:base}_linux_386.zip    a545108a0ccfde7c1e74de6c4e6fdded
-x86_64-linux-gnu = ${:base}_linux_amd64.zip  f343d709b84db494e8d6ec38259aa4a6
+url = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_${:_url}.zip
+
+[packer:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux_386
+md5sum = a545108a0ccfde7c1e74de6c4e6fdded
+
+[packer:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux_amd64
+md5sum = f343d709b84db494e8d6ec38259aa4a6

component/phantomjs/buildout.cfg

@@ -7,23 +7,36 @@ parts =
 
 [phantomjs]
 recipe = slapos.recipe.build
+shared = true
 slapos_promise =
   file:phantomjs-slapos
 
-base = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux
-i386-linux-gnu   = ${:base}-i686.tar.bz2   9c1426eef5b04679d65198b1bdd6ef88
-x86_64-linux-gnu = ${:base}-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
-
+url = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-${:_url}.tar.bz2
 install =
   import os
-  extract_dir = self.extract(self.download(*options[multiarch()].split()))
-  workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, location)
+  self.copyTree(guessworkdir(self.extract(self.download())), location)
   with open(os.path.join(location, "phantomjs-slapos"), 'w') as wrapper:
     wrapper.write("""#!/bin/sh -e
-  cd %(location)s
-  export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
-  export PATH=${fontconfig:location}/bin:$PATH
-  exec %(location)s/bin/phantomjs "$@"
-  """ % options)
+  export LD_LIBRARY_PATH=%s
+  export PATH=%s:$PATH
+  exec %s/bin/phantomjs "$@"
+  """ % (
+    ':'.join(options['library'].split()),
+    ':'.join(options['path'].split()),
+    location,
+  ))
     os.fchmod(wrapper.fileno(), 0o755)
+library =
+  ${freetype:location}/lib
+  ${fontconfig:location}/lib
+  ${libexpat:location}/lib
+path =
+  ${fontconfig:location}/bin
+
+[phantomjs:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux-i686
+md5sum = 9c1426eef5b04679d65198b1bdd6ef88
+
+[phantomjs:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux-x86_64
+md5sum = f278996c3edd0e8d8ec4893807f27d71

component/postfix/buildout.cfg

@@ -8,6 +8,7 @@ extends =
   ../patch/buildout.cfg
   ../pcre/buildout.cfg
   ../cyrus-sasl/buildout.cfg
+  ../libnsl/buildout.cfg
   ../m4/buildout.cfg
 
 [postfix]
@@ -20,7 +21,9 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
 configure-command = make
-configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
+configure-options = makefiles CCARGS=${:configure-options-CCARGS} AUXLIBS=${:configure-options-AUXLIBS}
+configure-options-CCARGS = '-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl -I${libnsl:location}/include'
+configure-options-AUXLIBS = '-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -L${libtirpc:location}/lib -L${libnsl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib  -Wl,-rpath=${libnsl:location}/lib' 
 make-targets = non-interactive-package install_root=${:location}
 environment =
   PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s

component/python-2.7/buildout.cfg

@@ -42,6 +42,7 @@ patches =
   ${:_profile_base_location_}/pytracemalloc_pep445.patch#9f3145817afa2b7fad801fde8447e396
   ${:_profile_base_location_}/disabled_module_list.patch#e038a8016475574c810cbaaf0e42f4ac
   ${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
+  ${:_profile_base_location_}/py27-subproc-closefds-fast.patch#e495e44491694a8972da11739206f2e6
 url =
   http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =

component/python-2.7/py27-subproc-closefds-fast.patch

+From 9a79fa64c6298998d04a5ae6d699222a93cd0cb8 Mon Sep 17 00:00:00 2001
+From: Kirill Smelkov <kirr@nexedi.com>
+Date: Fri, 24 Dec 2021 13:55:08 +0300
+Subject: [PATCH] [py27] subprocess: Use /proc/self/fd/ to know opened file
+ descriptors
+
+... to avoid O(MAXFD) slowdown on closefds=True.
+
+This is semantic backport of what Python3 does.
+---
+ Lib/subprocess.py | 23 ++++++++++++++++++++++-
+ 1 file changed, 22 insertions(+), 1 deletion(-)
+
+diff --git a/Lib/subprocess.py b/Lib/subprocess.py
+index 1f2da0ffbe8..300557220ac 100644
+--- a/Lib/subprocess.py
++++ b/Lib/subprocess.py
+@@ -869,7 +869,28 @@ def pipe_cloexec(self):
+ 
+ 
+         def _close_fds(self, but):
+-            if hasattr(os, 'closerange'):
++            # try to retrieve list of opened file descriptors to avoid being slow on large MAXFD
++            # https://www.erp5.com/project_section/vifib/forum/python2--subprocess.Popen-is-slow-when-close_fds-True-and-nofile-ulimit-is-high-PO01sREVXn
++            # https://bugs.python.org/issue8052
++            # https://github.com/python/cpython/commit/8facece99a59
++            fdv = None
++            try:
++                fdv = os.listdir('/proc/self/fd')
++            except OSError as e:
++                if e.errno != errno.ENOENT:
++                    raise
++            if fdv is not None:
++                for fd_ in fdv:
++                    fd = int(fd_)
++                    if (0 <= fd <= 2) or fd == but:
++                        continue
++                    try:
++                        os.close(fd)
++                    except Exception: # e.g. not KeyboardInterrupt nor SystemExit
++                        pass
++
++            # fallback to closing all file descriptors in 3..MAXFD range
++            elif hasattr(os, 'closerange'):
+                 os.closerange(3, but)
+                 os.closerange(but + 1, MAXFD)
+             else:
+-- 
+2.30.2
+

component/replication-manager/buildout.cfg

@@ -28,5 +28,6 @@ command = set -e
 <= go-git-package
 go.importpath = github.com/signal18/replication-manager
 repository =  https://github.com/signal18/replication-manager
-branch = 2.1
-revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03
+#branch = 2.1
+branch = develop
+revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
\ No newline at end of file

component/theia/buildout.hash.cfg

@@ -19,4 +19,4 @@ md5sum = 8157c22134200bd862a07c6521ebf799
 
 [yarn.lock]
 _update_hash_filename_ = yarn.lock
-md5sum = 067d2db611b21f77885f3adfd7f81453
+md5sum = e0341b7a715cc757a671aef902e6767d

component/theia/generate_download_plugins_cfg.py

@@ -41,8 +41,7 @@ for plugin_and_version in '''\
     vscode/lua/latest
     vscode/make/latest
     vscode/markdown/latest
-# https://github.com/eclipse-theia/theia/issues/7780
-    vscode/markdown-language-features/1.39.2
+    vscode/markdown-language-features/latest
     vscode/merge-conflict/latest
     vscode/npm/latest
     ms-vscode/node-debug/latest
@@ -71,13 +70,17 @@ for plugin_and_version in '''\
     vscode/theme-solarized-dark/latest
     vscode/theme-tomorrow-night-blue/latest
     vscode/typescript/latest
-    vscode/typescript-language-features/latest
+# latest (1.62.3) does not activate because it uses a proposed API not in theia:
+# Activating extension 'TypeScript and JavaScript Language Features (built-in)' failed: r.languages.createLanguageStatusItem is not a function
+    vscode/typescript-language-features/1.54.3
     vscode/vb/latest
     vscode/vscode-theme-seti/latest
     vscode/xml/latest
     vscode/yaml/latest
     EditorConfig/EditorConfig/latest
-    dbaeumer/vscode-eslint/latest
+# latest (2.2.2) does not activate:
+# Activating extension 'ESLint' failed: Class extends value undefined is not a constructor or null
+    dbaeumer/vscode-eslint/2.1.20
     ms-vscode/references-view/latest
 # golang.Go removed because it overwrites the PATH in theia shell
 # golang/Go/0.16.2

component/trafficserver/buildout.cfg

@@ -24,8 +24,8 @@ min_version = 8
 
 [trafficserver]
 recipe = slapos.recipe.cmmi
-url = http://apache.claz.org/trafficserver/trafficserver-9.1.0.tar.bz2
-md5sum = 994b0aa879cbd95054048f34bf8ed954
+url = https://dlcdn.apache.org/trafficserver/trafficserver-9.1.1.tar.bz2
+md5sum = 666945652942cce8c3ff0ae2757cc3c6
 shared = true
 patch-options = -p1
 configure-options =

component/userhosts/buildout.cfg

 [userhosts]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://lab.nexedi.com/nexedi/userhosts/repository/${:revision}/archive.tar.gz
-revision = a05fe5a3a5cb7005351ef4ec41460089f3ce4d0a
-md5sum = 5a80b4d962d975f290a60cf790c3334d
+url = https://github.com/figiel/hosts/archive/${:revision}.tar.gz
+revision = fdb45fe219593d63453f4be55cfc2a1199c18f59
+md5sum = 9ff5407132a2a67200ef1036a02f0ee2
 configure-command = true
 make-options = PREFIX=@@LOCATION@@
-make-targets = check install
+make-targets = test install

component/wkhtmltopdf/buildout.cfg

@@ -4,7 +4,6 @@ extends =
   ../freetype/buildout.cfg
   ../fontconfig/buildout.cfg
   ../libpng/buildout.cfg
-  ../tar/buildout.cfg
   ../xorg/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
@@ -14,27 +13,39 @@ parts =
 
 [wkhtmltopdf]
 recipe = slapos.recipe.build
-
-# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
-base = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic
-i386-linux-gnu   = ${:base}-i386.tar.xz  ce1a2c0b2cf786ccc5d5828c42c99ddd
-x86_64-linux-gnu = ${:base}-amd64.tar.xz 96b7306cebb9e65355f69f7ab63df68b
-
+url = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_${:_url}.tar.xz
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s
 install =
-  import os,shutil, sys, tempfile
-  path = self.download(*options[multiarch()].split())
-  extract_dir = tempfile.mkdtemp(self.name)
-  self.cleanup_dir_list.append(extract_dir)
-  self.logger.debug('Created working directory %s', extract_dir)
-  env = os.environ.copy()
-  env["PATH"] = "${tar:location}/bin:${xz-utils:location}/bin" + (":" + env["PATH"] if env.get("PATH") else "")
-  env["LD_LIBRARY_PATH"] = "${xz-utils:location}/lib" + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "")
-  call(["tar", "xJf", path, "-C", extract_dir], env=env)
-  shutil.move(os.path.join(extract_dir, "wkhtmltox"), location)
+  import os
+  self.copyTree(guessworkdir(self.extract(self.download())), location)
   with open(os.path.join(location, "wkhtmltopdf"), 'w') as wrapper:
-    wrapper.write("""#!/bin/sh
-  export LD_LIBRARY_PATH=%(location)s/lib:${fontconfig:location}/lib:${freetype:location}/lib:${libX11:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libpng12:location}/lib:${zlib:location}/lib
-  export PATH=${fontconfig:location}/bin:$PATH
-  exec %(location)s/bin/wkhtmltopdf "$@"
-  """ % options)
+    wrapper.write("""#!/bin/sh -e
+  d=%s
+  export LD_LIBRARY_PATH=$d/lib:%s
+  export PATH=%s:$PATH
+  exec $d/bin/wkhtmltopdf "$@"
+  """ % (
+    location,
+    ':'.join(options['library'].split()),
+    ':'.join(options['path'].split()),
+  ))
     os.fchmod(wrapper.fileno(), 0o755)
+library =
+  ${fontconfig:location}/lib
+  ${freetype:location}/lib
+  ${libX11:location}/lib
+  ${libXext:location}/lib
+  ${libXrender:location}/lib
+  ${libpng12:location}/lib
+  ${zlib:location}/lib
+path =
+  ${fontconfig:location}/bin
+
+[wkhtmltopdf:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux-generic-i386
+md5sum = ce1a2c0b2cf786ccc5d5828c42c99ddd
+
+[wkhtmltopdf:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux-generic-amd64
+md5sum = 96b7306cebb9e65355f69f7ab63df68b

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.220'
+version = '1.0.226'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/generic_mysql/mysql.py

+from __future__ import print_function
+
 import os
 import subprocess
 import time
@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
   while True:
     while True:
       mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
-        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        universal_newlines=True)
       result = mysql_upgrade.communicate()[0]
       if mysql_upgrade.returncode:
-        print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result)
+        print("Command %r failed with result:\n%s" % (mysql_upgrade_binary, result))
         break
-      print "MySQL database upgraded with result:\n%s" % result
+      print("MySQL database upgraded with result:\n%s" % result)
       mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
-          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+          universal_newlines=True)
       result = mysql.communicate(mysql_script)[0]
       if mysql.returncode:
-        print 'Command %r failed with:\n%s' % (mysql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_list, result))
         break
       # import timezone database
       mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
-                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                               universal_newlines=True)
       timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
       if mysql_tzinfo_to_sql.returncode != 0:
-        print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result))
         break
       mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
-                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                               universal_newlines=True)
       result = mysql.communicate(timezone_sql)[0]
       if mysql.returncode:
-        print 'Command %r failed with:\n%s' % (mysql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_list, result))
         break
-      print 'SlapOS initialisation script succesfully applied on database.'
+      print('SlapOS initialisation script succesfully applied on database.')
       return
     sleep = min(sleep+1, 30)
-    print 'Sleeping for %ss and retrying' % sleep
+    print('Sleeping for %ss and retrying' % sleep)
     sys.stdout.flush()
     sys.stderr.flush()
     time.sleep(sleep)

software/erp5/test/test/__init__.py

@@ -49,17 +49,20 @@ def setUpModule():
 class ERP5InstanceTestCase(SlapOSInstanceTestCase):
   """ERP5 base test case
   """
-  def getRootPartitionConnectionParameterDict(self):
+  @classmethod
+  def getRootPartitionConnectionParameterDict(cls):
     """Return the output paramters from the root partition"""
     return json.loads(
-        self.computer_partition.getConnectionParameterDict()['_'])
+        cls.computer_partition.getConnectionParameterDict()['_'])
 
-  def getComputerPartition(self, partition_reference):
-    for computer_partition in self.slap.computer.getComputerPartitionList():
+  @classmethod
+  def getComputerPartition(cls, partition_reference):
+    for computer_partition in cls.slap.computer.getComputerPartitionList():
       if partition_reference == computer_partition.getInstanceParameter(
           'instance_title'):
         return computer_partition
 
-  def getComputerPartitionPath(self, partition_reference):
-    partition_id = self.getComputerPartition(partition_reference).getId()
-    return os.path.join(self.slap._instance_root, partition_id)
+  @classmethod
+  def getComputerPartitionPath(cls, partition_reference):
+    partition_id = cls.getComputerPartition(partition_reference).getId()
+    return os.path.join(cls.slap._instance_root, partition_id)

software/erp5/upgrade_test/test.py

@@ -39,13 +39,15 @@ from slapos.testing.testcase import installSoftwareUrlList
 from slapos.testing.testcase import SlapOSNodeCommandError
 from slapos.grid.utils import md5digest
 
-old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.4/software/erp5/software.cfg'
+old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.5/software/erp5/software.cfg'
 new_software_release_url = os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
 
 _, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
-    old_software_release_url, software_id="upgrade_erp5")
-
+    old_software_release_url,
+    software_id="upgrade_erp5",
+    skip_software_check=True,
+)
 
 def setUpModule():
   installSoftwareUrlList(

software/fluentd/test/test.py

@@ -35,6 +35,7 @@ import subprocess
 import tempfile
 import time
 import six
+import sys
 
 from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
 from six.moves.socketserver import StreamRequestHandler, TCPServer
@@ -171,7 +172,7 @@ class SensorConfTestCase(WendelinTutorialTestCase):
 <source>
   @type exec
   tag tag.name
-  command python %s
+  command %s %s
   run_interval %ss
   <parse>
     keys pressure, humidity, temperature
@@ -186,7 +187,7 @@ class SensorConfTestCase(WendelinTutorialTestCase):
   <buffer>
     flush_mode immediate
   </buffer>
-</match>''' % (script_path, FLUSH_INTERVAL, cls._ipv6_address)
+</match>''' % (sys.executable, script_path, FLUSH_INTERVAL, cls._ipv6_address)
 
   @classmethod
   def sensor_script(cls, measurementList):
@@ -280,4 +281,4 @@ class GatewayConfTestCase(WendelinTutorialTestCase):
       data,
       msgpack.unpackb(
         self.serve(self._wendelin_port, WendelinHTTPRequestHandler)),
-    )
\ No newline at end of file
+    )

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 087f226ba90928dcc5a722d7008c867a
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = b260fce887535fc69a259d1fd31af1b2
+md5sum = d949e6100fd4e01819cc2867a65c6e65
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in

software/kvm/instance-kvm.cfg.jinja2

@@ -1029,15 +1029,12 @@ config-command = ${ansible-vm-bin:rendered}
 
 {% if bootstrap_script_url -%}
 [download-bootstrap-script]
-recipe = slapos.recipe.build
-location = ${directory:public}/vm-bootstrap
-install =
-  from zc.buildout.download import check_md5sum, urlretrieve, ChecksumError
-  url, md5sum = {{ repr(bootstrap_script_url) }}.split('#')
-  urlretrieve(url, location)
-  if not check_md5sum(location, md5sum):
-    os.remove(location)
-    raise ChecksumError
+recipe = slapos.recipe.build:download
+offline = false
+destination = ${directory:public}/vm-bootstrap
+{% set bootstrap_script_url = bootstrap_script_url.split('#') -%}
+url = {{ bootstrap_script_url[0] }}
+md5sum = {{ bootstrap_script_url[1] }}
 {% endif -%}
 
 [logrotate-vm-bootstrap]

software/ors-amarisoft/buildout.hash.cfg

@@ -16,27 +16,27 @@
 
 [template]
 filename = instance.cfg
-md5sum = 96a76300b2f714b6c47157920fe79a53
+md5sum = 6ea4fa210a91c15278c847a809de5991
 
 [template-lte-enb-epc]
 _update_hash_filename_ = instance-enb-epc.jinja2.cfg
-md5sum = 762d55291e75e8b61e35f9f28d29915a
+md5sum = 25fae79db2b30be0f365ad967b278c3c
 
 [template-lte-enb]
 _update_hash_filename_ = instance-enb.jinja2.cfg
-md5sum = 83c37b43d7b70584bf71aed9289ea13c
+md5sum = fe249168a3f50b0efe6aeae39afb03ae
 
 [template-lte-gnb-epc]
 _update_hash_filename_ = instance-gnb-epc.jinja2.cfg
-md5sum = e43a726dd3023a4bbaa474bb2d7a6ebe
+md5sum = 5ec4508b02f6e7fbdbe1f37c65b9d897
 
 [template-lte-gnb]
 _update_hash_filename_ = instance-gnb.jinja2.cfg
-md5sum = fc59d15a7f7f942951f9e38d7a1cca2c
+md5sum = 6deb38b3de44f6e0a43b49fd13f0f072
 
 [template-lte-epc]
 _update_hash_filename_ = instance-epc.jinja2.cfg
-md5sum = 0585b4f8fd42538595e6abb61d07fd93
+md5sum = 089f62e736cdc620eafc2c47b050fe13
 
 [ue_db.jinja2.cfg]
 filename = config/ue_db.jinja2.cfg
@@ -44,11 +44,11 @@ md5sum = d33163012d6c98efc59161974c649557
 
 [enb.jinja2.cfg]
 filename = config/enb.jinja2.cfg
-md5sum = d841debc51d9f12555a47d1556a6a3c1
+md5sum = 97d9cdd07704cae36c5e4234c87025e8
 
 [gnb.jinja2.cfg]
 filename = config/gnb.jinja2.cfg
-md5sum = da64ea9c5003f40987a8bba3f18e8839
+md5sum = b0df7f3b679b25d5296dd67e074364b4
 
 [ltelogs.jinja2.sh]
 filename = ltelogs.jinja2.sh
@@ -61,3 +61,11 @@ md5sum = 518c71ce57204304b703b977c665a164
 [ims.jinja2.cfg]
 filename = config/ims.jinja2.cfg
 md5sum = e561ec26a70943c61557def1781cf65f
+
+[sdr-busy-promise]
+_update_hash_filename_ = promise/check_sdr_busy.py
+md5sum = 9e867282d7dd80c4255e87a82a47a19d
+
+[interface-up-promise]
+_update_hash_filename_ = promise/check_interface_up.py
+md5sum = 505efcbe04e717088924f2267b10c2b9

software/ors-amarisoft/config/enb.jinja2.cfg

@@ -56,15 +56,15 @@
     { 
       /* address of MME for S1AP connection. Must be modified if the MME
          runs on a different host. */
-      mme_addr: "127.0.1.100"
+      mme_addr: "{{ slapparameter_dict.get('mme_addr', '127.0.1.100') }}"
     },
   ],
   /* GTP bind address (=address of the ethernet interface connected to
      the MME). Must be modified if the MME runs on a different host. */
-  gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
+  gtp_addr: "{{ gtp_addr }}",
 
   /* high 20 bits of SIB1.cellIdentifier */
-  enb_id: 0x1A2D0,
+  enb_id: {{ slapparameter_dict.get('enb_id', '0x1A2D0') }},
 
   /* list of cells */
   cell_list: [

software/ors-amarisoft/config/gnb.jinja2.cfg

@@ -37,15 +37,15 @@
   amf_list: [
     {
       /* address of AMF for NGAP connection. Must be modified if the AMF runs on a different host. */
-      amf_addr: "127.0.1.100",
+      amf_addr: "{{ slapparameter_dict.get('amf_addr', '127.0.1.100') }}",
     },
   ],
   /* GTP bind address (=address of the ethernet interface connected to
      the AMF). Must be modified if the AMF runs on a different host. */
-  gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
+  gtp_addr: "{{ gtp_addr }}",
 
   gnb_id_bits: 28,
-  gnb_id: 0x12345,
+  gnb_id: {{ slapparameter_dict.get('gnb_id', '0x12345') }},
 
   nr_support: true,
 

software/ors-amarisoft/instance-enb-epc.jinja2.cfg

@@ -12,7 +12,7 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
@@ -34,7 +34,7 @@ promise = ${:etc}/promise
 log = ${:var}/log
 
 [request-common-base]
-recipe = slapos.cookbook:request.serialised
+recipe = slapos.cookbook:request
 software-url = {{ slap_connection['software-release-url'] }}
 server-url = {{ slap_connection['server-url'] }}
 computer-id = {{ slap_connection['computer-id'] }}
@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
 key-file = {{ slap_connection['key-file'] }}
 cert-file = {{ slap_connection['cert-file'] }}
 
+sla-computer_guid = {{ slap_connection['computer-id'] }}
+
 config-monitor-password = ${monitor-htpasswd:passwd}
 return = monitor-base-url
 

software/ors-amarisoft/instance-enb-input-schema.jinja2.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": {{ default_lte_n_rb_dl }}
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }

software/ors-amarisoft/instance-enb.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-enb-config
   lte-enb-service
-# Temporarily extend monitor-base until promises are added
+  sdr-busy-promise
   monitor-base
   publish-connection-information
 
@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
 key = {{ slap_connection['key-file'] }}
 cert = {{ slap_connection['cert-file'] }}
 
-configuration.network_name = VIFIB
-configuration.domain = vifib.com
+configuration.network_name = RAPIDSPACE
+configuration.domain = rapid.space
 configuration.mme_ws_port = 9000
 configuration.enb_ws_port = 9002
 configuration.ims_ws_port = 9003
@@ -60,17 +60,27 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+[lte-enb-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  sudo /opt/amarisoft/rm-tmp-lte | true;
+  {{ enb }}/lteenb ${directory:etc}/enb.cfg >> ${directory:log}/enb-output.cfg 2>> ${directory:log}/enb-output.cfg
+
 ### eNodeB (enb)
 [lte-enb-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
-command-line = {{ enb }}/lteenb ${directory:etc}/enb.cfg
+command-line = ${lte-enb-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-enb
 mode = 0775
 reserve-cpu = True
 pidfile = ${directory:run}/enb.pid
 hash-files =
   ${lte-enb-config:rendered}
+  ${lte-enb-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -83,6 +93,7 @@ context =
   section directory directory
   section slap_configuration slap-configuration
   key slapparameter_dict slap-configuration:configuration
+  raw gtp_addr {{ local_ipv4 }}
   import  netaddr netaddr
 
 [lte-enb-config]
@@ -91,9 +102,21 @@ template = {{ enb_template }}
 rendered = ${directory:etc}/enb.cfg
 
 [publish-connection-information]
-recipe = slapos.cookbook:publish.serialised
-monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
+<= monitor-publish
+recipe = slapos.cookbook:publish
+ipv4 = {{ local_ipv4 }}
 
 [monitor-instance-parameter]
+{% if slapparameter_dict.get("name", None) %}
 monitor-title = {{ slapparameter_dict['name'] | string }}
+{% endif %}
+{% if slapparameter_dict.get("monitor-password", None) %}
 password = {{ slapparameter_dict['monitor-password'] | string }}
+{% endif %}
+
+# Add custom promise to check if /dev/sdr0 is busy
+[sdr-busy-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ sdr_busy_promise }}
+output = ${directory:plugins}/check-sdr-busy.py

software/ors-amarisoft/instance-epc.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-mme-config
   lte-mme-service
-# Temporarily extend monitor-base until promises are added
+  tun-up-promise
   monitor-base
   publish-connection-information
 
@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
 key = {{ slap_connection['key-file'] }}
 cert = {{ slap_connection['cert-file'] }}
 
-configuration.network_name = VIFIB
-configuration.domain = vifib.com
+configuration.network_name = RAPIDSPACE
+configuration.domain = rapid.space
 configuration.log_size = 50M
 configuration.mme_ws_port = 9000
 configuration.enb_ws_port = 9002
@@ -62,7 +62,7 @@ context =
 [lte-ims-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/ims.log; sleep 1
-command-line = rm -f ${directory:var}/lte_ue.db; {{ mme }}/lteims ${directory:etc}/ims.cfg
+command-line = {{ mme }}/lteims ${directory:etc}/ims.cfg
 wrapper-path = ${directory:service}/lte-ims
 mode = 0775
 pidfile = ${directory:run}/ims.pid
@@ -71,17 +71,29 @@ hash-files =
   {{ ue_db_path }}
 environment = AMARISOFT_PATH=/opt/amarisoft/.amarisoft
 
+[lte-mme-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  rm -f ${directory:var}/lte_ue.db;
+  {{ mme }}/ltemme ${directory:etc}/mme.cfg >> ${directory:log}/mme-output.cfg 2>> ${directory:log}/mme-output.cfg
+
 ### MME
 [lte-mme-service]
 recipe = slapos.cookbook:wrapper
+# When the machine shutdowns abruptly, lte_ue is not cleaned up which causes
+# amarisoft ltemme to fail. TODO: find a cleaner way to handle this
 init = ${ltelogs:rendered} ${directory:log}/mme.log
-command-line = {{ mme }}/ltemme ${directory:etc}/mme.cfg
+command-line = ${lte-mme-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-mme
 mode = 0775
 pidfile = ${directory:run}/mme.pid
 hash-files =
   ${lte-mme-config:rendered}
   {{ ue_db_path }}
+  ${lte-mme-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib:{{ nghttp2_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -119,17 +131,18 @@ context =
   import  netaddr netaddr
   key ifup_empty lte-mme-ifup-empty:wrapper-path
 
-[publish-connection-information]
-recipe = slapos.cookbook:publish.serialised
-
-{% if slapparameter_dict.get("monitor-password", None) %}
-monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
-{% else %}
-monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
-{% endif %}
-
-{% if slapparameter_dict.get("monitor-password", None) %}
 [monitor-instance-parameter]
 monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-password'] }}
-{% endif %}
+
+[publish-connection-information]
+<= monitor-publish
+recipe = slapos.cookbook:publish
+
+# Add custom promise to check if /dev/sdr0 is busy
+[tun-up-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ interface_up_promise }}
+output = ${directory:plugins}/check-tun-up.py
+config-ifname = ${slap-configuration:tun-name}

software/ors-amarisoft/instance-gnb-epc.jinja2.cfg

@@ -12,7 +12,7 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
@@ -34,7 +34,7 @@ promise = ${:etc}/promise
 log = ${:var}/log
 
 [request-common-base]
-recipe = slapos.cookbook:request.serialised
+recipe = slapos.cookbook:request
 software-url = {{ slap_connection['software-release-url'] }}
 server-url = {{ slap_connection['server-url'] }}
 computer-id = {{ slap_connection['computer-id'] }}
@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
 key-file = {{ slap_connection['key-file'] }}
 cert-file = {{ slap_connection['cert-file'] }}
 
+sla-computer_guid = {{ slap_connection['computer-id'] }}
+
 config-monitor-password = ${monitor-htpasswd:passwd}
 return = monitor-base-url
 

software/ors-amarisoft/instance-gnb-input-schema.jinja2.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": {{ default_nr_bandwidth }}
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }

software/ors-amarisoft/instance-gnb.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-gnb-config
   lte-enb-service
-# Temporarily extend monitor-base until promises are added
+  sdr-busy-promise
   monitor-base
   publish-connection-information
 
@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
 key = {{ slap_connection['key-file'] }}
 cert = {{ slap_connection['cert-file'] }}
 
-configuration.network_name = VIFIB
-configuration.domain = vifib.com
+configuration.network_name = RAPIDSPACE
+configuration.domain = rapid.space
 configuration.mme_ws_port = 9000
 configuration.enb_ws_port = 9002
 configuration.ims_ws_port = 9003
@@ -60,17 +60,27 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+[lte-enb-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  sudo /opt/amarisoft/rm-tmp-lte | true;
+  {{ enb }}/lteenb ${directory:etc}/gnb.cfg >> ${directory:log}/gnb-output.cfg 2>> ${directory:log}/gnb-output.cfg
+
 ### eNodeB (enb)
 [lte-enb-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
-command-line = {{ enb }}/lteenb ${directory:etc}/gnb.cfg
+command-line = ${lte-enb-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-enb
 mode = 0775
 reserve-cpu = True
 pidfile = ${directory:run}/enb.pid
 hash-files =
   ${lte-gnb-config:rendered}
+  ${lte-enb-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -83,6 +93,7 @@ context =
   section directory directory
   section slap_configuration slap-configuration
   key slapparameter_dict slap-configuration:configuration
+  raw gtp_addr {{ local_ipv4 }}
   import  netaddr netaddr
 
 [lte-gnb-config]
@@ -91,9 +102,21 @@ template = {{ gnb_template }}
 rendered = ${directory:etc}/gnb.cfg
 
 [publish-connection-information]
-recipe = slapos.cookbook:publish.serialised
-monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
+<= monitor-publish
+recipe = slapos.cookbook:publish
+ipv4 = {{ local_ipv4 }}
 
 [monitor-instance-parameter]
+{% if slapparameter_dict.get("name", None) %}
 monitor-title = {{ slapparameter_dict['name'] | string }}
+{% endif %}
+{% if slapparameter_dict.get("monitor-password", None) %}
 password = {{ slapparameter_dict['monitor-password'] | string }}
+{% endif %}
+
+# Add custom promise to check if /dev/sdr0 is busy
+[sdr-busy-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ sdr_busy_promise }}
+output = ${directory:plugins}/check-sdr-busy.py

software/ors-amarisoft/instance-tdd1900-enb-epc-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd1900-enb-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd1900-gnb-epc-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd1900-gnb-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd2600-enb-epc-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd2600-enb-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd2600-gnb-epc-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd2600-gnb-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3500-enb-epc-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3500-enb-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3500-gnb-epc-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3500-gnb-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3700-enb-epc-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3700-enb-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3700-gnb-epc-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3700-gnb-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance.cfg

@@ -7,7 +7,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = $${slap-connection:computer-id}
 partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
@@ -36,6 +36,20 @@ gnb = dynamic-template-lte-gnb:rendered
 epc = dynamic-template-lte-epc:rendered
 RootSoftwareInstance = $${:enb-epc}
 
+[local-ipv4-address]
+recipe = slapos.recipe.build
+init = 
+  import socket
+  s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+  try:
+      s.connect(('1.1.1.1', 1))
+      ip = s.getsockname()[0]
+  except Exception:
+      ip = '127.0.0.1'
+  finally:
+      s.close()
+  options['local-ipv4'] = ip
+
 [dynamic-template-lte-enb-epc]
 < = jinja2-template-base
 template = ${template-lte-enb-epc:target}
@@ -43,6 +57,7 @@ filename = instance-lte-enb-epc.cfg
 extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:rendered}
+    key local_ipv4 local-ipv4-address:local-ipv4
  
 [dynamic-template-lte-gnb-epc]
 < = jinja2-template-base
@@ -51,6 +66,7 @@ filename = instance-lte-gnb-epc.cfg
 extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:rendered}
+    key local_ipv4 local-ipv4-address:local-ipv4
  
 [dynamic-template-lte-enb]
 < = jinja2-template-base
@@ -62,6 +78,7 @@ extra-context =
     raw enb   ${enb:destination}
     raw enb_template   ${enb.jinja2.cfg:target}
     raw ltelogs_template   ${ltelogs.jinja2.sh:target}
+    raw sdr_busy_promise   ${sdr-busy-promise:target}
     raw openssl_location ${openssl:location}
     raw default_dl_earfcn ${enb:default-dl-earfcn}
     raw default_lte_dl_freq ${enb:default-lte-dl-freq}
@@ -71,6 +88,7 @@ extra-context =
     raw default_lte_rx_gain ${enb:default-lte-rx-gain}
     raw min_frequency ${enb:min-frequency}
     raw max_frequency ${enb:max-frequency}
+    key local_ipv4 local-ipv4-address:local-ipv4
 
 [dynamic-template-lte-gnb]
 < = jinja2-template-base
@@ -82,6 +100,7 @@ extra-context =
     raw enb   ${enb:destination}
     raw gnb_template   ${gnb.jinja2.cfg:target}
     raw ltelogs_template   ${ltelogs.jinja2.sh:target}
+    raw sdr_busy_promise   ${sdr-busy-promise:target}
     raw openssl_location ${openssl:location}
     raw default_dl_nr_arfcn ${enb:default-dl-nr-arfcn}
     raw default_nr_band ${enb:default-nr-band}
@@ -91,6 +110,7 @@ extra-context =
     raw default_nr_rx_gain ${enb:default-nr-rx-gain}
     raw min_frequency ${enb:min-frequency}
     raw max_frequency ${enb:max-frequency}
+    key local_ipv4 local-ipv4-address:local-ipv4
 
 [dynamic-template-lte-epc]
 < = jinja2-template-base
@@ -99,6 +119,7 @@ filename = instance-lte-epc.cfg
 extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:rendered}
+    raw interface_up_promise   ${interface-up-promise:target}
     raw mme   ${mme:destination}
     raw mme_template   ${mme.jinja2.cfg:target}
     raw ims_template   ${ims.jinja2.cfg:target}

software/ors-amarisoft/promise/check_interface_up.py

+import socket
+import errno
+
+from zope.interface import implementer
+from slapos.grid.promise import interface
+from slapos.grid.promise.generic import GenericPromise
+
+
+
+
+@implementer(interface.IPromise)
+class RunPromise(GenericPromise):
+
+  def __init__(self, config):
+    """
+      Called when initialising the promise before testing.
+      Sets the configuration and the periodicity.
+    """
+    super(RunPromise, self).__init__(config)
+    self.setPeriodicity(minute=2)
+
+
+  def sense(self):
+    """
+      Called every time the promise is tested.
+      Signals a positive or negative result.
+
+      In this case, check whether the file exists.
+    """
+    ifname = self.getConfig('ifname')
+
+    f = open('/sys/class/net/%s/operstate' % ifname, 'r')
+    if f.read() == 'up\n':
+      self.logger.info("%s is up", ifname)
+    else:
+      self.logger.error("%s is down", ifname)
+    f.close()
+
+  def test(self):
+    """
+      Called after sense() if the instance is still converging.
+      Returns success or failure based on sense results.
+
+      In this case, fail if the previous sensor result is negative.
+    """
+    return self._test(result_count=1, failure_amount=1)
+
+
+  def anomaly(self):
+    """
+      Called after sense() if the instance has finished converging.
+      Returns success or failure based on sense results.
+      Failure signals the instance has diverged.
+
+      In this case, fail if two out of the last three results are negative.
+    """
+    return self._anomaly(result_count=3, failure_amount=2)

software/ors-amarisoft/promise/check_sdr_busy.py

+import os
+import errno
+
+from zope.interface import implementer
+from slapos.grid.promise import interface
+from slapos.grid.promise.generic import GenericPromise
+
+
+
+
+@implementer(interface.IPromise)
+class RunPromise(GenericPromise):
+
+  def __init__(self, config):
+    """
+      Called when initialising the promise before testing.
+      Sets the configuration and the periodicity.
+    """
+    super(RunPromise, self).__init__(config)
+    self.setPeriodicity(minute=2)
+
+
+  def sense(self):
+    """
+      Called every time the promise is tested.
+      Signals a positive or negative result.
+
+      In this case, check whether the file exists.
+    """
+    sdr_dev = '/dev/sdr0'
+    try:
+      open(sdr_dev, 'w').close()
+      self.logger.error("eNB is not using %s", sdr_dev)
+    except IOError as e:
+      if e.errno == errno.EBUSY:
+        self.logger.info("eNB is using %s", sdr_dev)
+
+  def test(self):
+    """
+      Called after sense() if the instance is still converging.
+      Returns success or failure based on sense results.
+
+      In this case, fail if the previous sensor result is negative.
+    """
+    return self._test(result_count=1, failure_amount=1)
+
+
+  def anomaly(self):
+    """
+      Called after sense() if the instance has finished converging.
+      Returns success or failure based on sense results.
+      Failure signals the instance has diverged.
+
+      In this case, fail if two out of the last three results are negative.
+    """
+    return self._anomaly(result_count=3, failure_amount=2)

software/ors-amarisoft/software.cfg

@@ -49,6 +49,12 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
 [template-lte-epc]
 <= download-base
 
+[sdr-busy-promise]
+<= download-base
+
+[interface-up-promise]
+<= download-base
+
 [amarisoft]
 recipe = slapos.recipe.build
 path = /opt/amarisoft/lte

software/repman/buildout.hash.cfg

@@ -14,19 +14,19 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 8a08be95a04f1a47098c4fdef80bdfed
+md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 697a1b546c883da45c14dbcd2d73b2b9
+md5sum = 7782adde357c9444fed51f2dcc230a52
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
-md5sum = 2ee2c4bc5f985c11c2167b819d261256
+md5sum = 0a4ce036ea524373a85fac10e81edb96
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
-md5sum = c78c6be9537d3dffbb526b0e819bacb5
+md5sum = 8cdeb63a2acc294801266f24599f6d32
 
 [nginx.conf.in]
 _update_hash_filename_ = templates/nginx.conf.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = a5c204cac552754520aee0570d379723
+md5sum = 938f1e8087395757f5ae861d20500658
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = df44c2d6fb1971df582345daedad280c
+md5sum = 47b76144e1b116580c8acf08274af976
+
+[template-publish-slave-information]
+_update_hash_filename_ = publish-database-slave-parameters.cfg.in
+md5sum = 9616477ff993c55c2b43bf8797db6994
 
 [mariadb-init-root-sql]
 _update_hash_filename_ = templates/mariadb_init_root.sql.in
-md5sum = d8a0a7d72b02f911dafda655941b805d
+md5sum = 6e5c02542568f250aacd84694249efc0
 
 [init-root-wrapper-in]
 _update_hash_filename_ = templates/init_root_wrapper.in
-md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef
+md5sum = c203f40a58386310a433b58fd345a341
 
 [repman-manager-sh.in]
 _update_hash_filename_ = templates/repman-manager.sh.in
@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
-md5sum = d2ebd2ec55bf8489789a52c808729925
+md5sum = d5c19d866c5a76d3bc2f434a91b1553a
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in
-md5sum = e9bcee5dc1318fe3acda2663472214f5
+md5sum = 8f14054b3761b167cf8a86f4f084a574
 
 [proxy-need-start-stop.sh.in]
 _update_hash_filename_ = templates/proxy-need-start-stop.sh.in
 md5sum = 455aaf369bf5141758dc57f2c0e67b08
+
+[slave-db-manage.in]
+_update_hash_filename_ = templates/slave-db-manage.in
+md5sum = b45313ae5fb06972cc7fc945e34e434a

software/repman/instance-mariadb.cfg.jinja2.in

@@ -17,17 +17,11 @@
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
--extends = publish-early
 database-host = {{ host }}:{{ port }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 partition-path = ${buildout:directory}
 receiver-port = ${dbjob-parameter:socat-port}
-
-[publish-early]
-recipe = slapos.cookbook:publish-early
--init =
-
-database-list = {{ dumps(database_list) }}
+csr-id = ${caucase-csr-id:csr-id}
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -37,6 +31,22 @@ mode = 644
 < = jinja2-template-base
 mode = 755
 
+[jinja2-template-script-base]
+< = jinja2-template-executable
+context =
+  key tmp_dir            directory:tmp
+  key partition_dir      buildout:directory
+  raw username           {{ slapparameter_dict['repman-user'] }}
+  raw password           {{ slapparameter_dict['repman-passwd'] }}
+  raw cluster_name       {{ slapparameter_dict['cluster'] }}
+  raw secure_url         {{ slapparameter_dict['repman-secure-url'] }}
+  raw mysql_conf         ${directory:etc}/mysql/my.cnf
+  raw dash_bin           {{ dash }}
+  raw jq_bin             {{ jq_bin }}
+  raw curl_bin           {{ curl_bin }}
+  raw mysql_bin          {{ parameter_dict['mariadb-location'] }}/bin/mysql
+  ${:extra-context}
+
 [simplefile]
 < = jinja2-template-base
 template = inline:{{ '{{ content }}' }}
@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
+{% import "caucase" as caucase with context %}
+{{ caucase.updater(
+     prefix='caucase-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-updater',
+     url=slapparameter_dict['caucase-url'],
+     data_dir='${mysql-directory:ssl}',
+     crt_path='${mysql-directory:ssl}/server-cert.pem',
+     ca_path='${mysql-directory:ssl}/ca-cert.pem',
+     crl_path='${mysql-directory:ssl}/crl.pem',
+     key_path='${mysql-directory:ssl}/server-key.pem',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+)}}
+{% do part_list.append('caucase-updater') -%}
+{% do part_list.append('caucase-updater-promise') -%}
+
+[get-csr-id]
+recipe = plone.recipe.command
+output = ${directory:tmp}/csr_id
+command = 
+  if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
+    RESULT="None";
+  else
+    if [ -f "${caucase-updater-csr:csr}" ]; then
+      RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
+      if [ ! $? -eq 0 ]; then
+        RESULT="None";
+      fi
+    fi
+  fi
+  cat <<EOF > ${:output}
+  [caucase]
+  csr-id = $(echo $RESULT)
+  EOF
+update-command = ${:command}
+
+[caucase-csr-id]
+recipe = slapos.cookbook:zero-knowledge.read
+file-path = ${get-csr-id:output}
+csr-id =
+
 [my-cnf-parameters]
 socket = ${directory:run}/mariadb.sock
 ip = {{ ip }}
-data-directory = ${directory:srv}/mariadb
+data-directory = ${mysql-directory:data}
 pid-file = ${directory:run}/mariadb.pid
 plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
 groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
 innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
 innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
 relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
-ssl-crt = ${directory:mariadb-ssl}/crt.pem
-ssl-key = ${directory:mariadb-ssl}/key.pem
-ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
 
 [my-cnf]
 < = jinja2-template-base
@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }}
 mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
 root-user = repman
 heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
+require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
 
 [init-script]
 < = jinja2-template-executable
+extensions = jinja2.ext.do
 # XXX: is there a better location ?
 rendered = ${directory:etc}/mariadb_initial_setup.sql
 template = {{ parameter_dict['template-mariadb-initial-setup'] }}
@@ -106,20 +159,26 @@ init-script = ${init-script:rendered}
 mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
 
 [{{ section('update-mysql-script') }}]
-< = jinja2-template-executable
+< = jinja2-template-script-base
 rendered = ${directory:scripts}/mariadb_update
 init-password = ${directory:etc}/.init-passwd.done
 upgrade-done = ${directory:lib}/mariadb-update-done
-context =
+extra-context =
   key init_password_done :init-password
   key upgrade_done       :upgrade-done
   key init_root_sql      init-root-sql:rendered
   key mysql_update       update-mysql:output
-  raw mysql_conf         ${directory:etc}/mysql/my.cnf
-  raw dash_bin           {{ dash }}
-  raw mysql_bin          {{ parameter_dict['mariadb-location'] }}/bin/mysql
+  key init_database_sql  init-script:rendered
 template = {{ parameter_dict['template-init-root-wrapper'] }}
 
+[{{ section('mysql-slave-db-cleanup') }}]
+< = jinja2-template-script-base
+rendered = ${directory:scripts}/manage-slave-db
+db-name = {{ slapparameter_dict['database-name'] }}
+extra-context =
+  key database_name      :db-name
+template = {{ parameter_dict['template-manage-db'] }}
+
 [mysqld]
 < = jinja2-template-executable
 rendered = ${directory:bin}/mysqld
@@ -136,19 +195,19 @@ environ =
   {{ variable }}
   {%- endfor %}
 
-[ca-mysqld]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-key-file = ${my-cnf-parameters:ssl-key}
-cert-file = ${my-cnf-parameters:ssl-crt}
-executable = ${mysqld:rendered}
-wrapper = ${directory:controller}/mariadb
+[mysqld-launcher]
+recipe = slapos.cookbook:wrapper
+command-line = ${mysqld:rendered}
+wrapper-path = ${directory:controller}/mariadb
+wait-for-files =
+  ${mysql-directory:mysql}/my.cnf
+  ${mysql-directory:ssl}/server-cert.pem
 
 {% import "supervisord_lib" as supervisord_lib with context %}
 {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
 {% do part_list.append("supervisord-mariadb-ctl") -%}
-{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}",
-  "stopwaitsecs": 86400, "environment": [],
+{% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
+  "stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
   "stdout_logfile": "${directory:log}/mariadb_stdout.log",
   "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
 
@@ -178,13 +237,18 @@ target-directory = ${directory:bin}
 link-binary = {{ dumps(parameter_dict['link-binary']) }}
 
 [binary-wrap-base]
-recipe = slapos.cookbook:wrapper
 # Note: --defaults-file must be the first argument, otherwise wrapped binary
 # will reject it.
-command-line = 
-  "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
-  --defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
-wrapper-path = ${directory:bin}/${:command}
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:command}
+mode = 755
+template =
+  inline:#!/bin/sh
+  exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
+  --defaults-file="${directory:etc}/mysql/my.cnf" \
+    --protocol=socket \
+  ${:extra-args} "$@"
+wrapper-path = ${:rendered}
 extra-args =
 
 [binary-wrap-mysql]
@@ -208,6 +272,11 @@ command = mysqladmin
 <= binary-wrap-base
 command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
 command = mysql_tzinfo_to_sql
+template =
+  inline:#!/bin/sh
+  exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
+  --defaults-file="${directory:etc}/mysql/my.cnf" \
+  --skip-write-binlog "$@"
 
 [binary-wrap-pt-digest]
 <= binary-wrap-base
@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin
 srv = ${buildout:directory}/srv
 tmp = ${buildout:directory}/tmp
 backup = ${:srv}/backup
-mariadb-backup-full = ${:backup}/mariadb-full
-mariadb-backup-incremental = ${:backup}/mariadb-incremental
-mariadb-ssl = ${:etc}/mariadb-ssl
 var = ${buildout:directory}/var
 lib = ${:var}/lib
-mysql = ${:lib}/mysql
 log = ${:var}/log
 run = ${:var}/run
 config-tmp = ${:tmp}/config
-custom = ${directory:etc}/mysql/custom
+
+[mysql-directory]
+recipe = slapos.cookbook:mkdirectory
+mysql = ${directory:etc}/mysql
+ssl = ${:mysql}/ssl
+data = ${directory:lib}/mysql
+custom = ${:mysql}/custom.d
 
 [dbjob-parameter]
 bash-bin = {{ bash_bin }}
 db-user = ${init-script-parameters:root-user}
 db-password = ${init-script-parameters:password}
-mysql-dir = ${directory:mysql}
-dbjob-cnf = ${directory:etc}/mysql/my.cnf
-log-dir = ${directory:mysql}/.system/logs
+mysql-dir = ${mysql-directory:data}
+dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
+log-dir = ${mysql-directory:data}/.system/logs
 tmp-dir = ${directory:tmp}
 mysqld-socket  = ${my-cnf-parameters:socket}
+mysql-client = ${binary-wrap-mysql:wrapper-path}
 socat-port = {{ int(port) + 9 }}
 restart-script = ${mysqld-restart-script:rendered}
 socat-location = {{ parameter_dict['socat-location'] }}
@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }}
   cd ${directory:config-tmp} &&
   {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
   tar -xzf config.tar.gz
-  cp -r data/.system ${directory:mysql}
-  rm -rf ${directory:etc}/mysql
-  cp -r etc/mysql ${directory:etc}
-  ln -sf ${directory:mysql}/.system ${directory:var}/system
-  ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf
+  rm -r etc/mysql/ssl  # we don't need generated ssl files
+  cp -r data/.system ${mysql-directory:data}
+  cp -r etc/mysql/* ${mysql-directory:mysql}
+  ln -sf ${mysql-directory:data}/.system ${directory:var}/system
+  ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
 
 [{{ section('install-mysql-config') }}]
 recipe = plone.recipe.command

software/repman/instance-repman-input-schema.json

@@ -89,33 +89,6 @@
       "type": "string",
       "default": ""
     },
-    "tags": {
-      "title": "Provisioning db tag list",
-      "description": "Provisioning db tags. Set one tag per line.",
-      "type": "array",
-      "items": {
-        "type": "string"
-      },
-      "default": [
-        "gtidstrict",
-        "bind",
-        "pkg",
-        "innodb",
-        "noquerycache",
-        "slow",
-        "pfs",
-        "linux",
-        "readonly",
-        "diskmonitor",
-        "sqlerror",
-        "compressbinlog",
-        "bm4ci",
-        "mroonga",
-        "utctime",
-        "readcommitted",
-        "nohandshake"
-      ]
-    },
     "http-session-lifetime": {
       "title": "Web Session life time in s",
       "description": "Web interface Session life time in seconds. Default 86400",
@@ -128,17 +101,11 @@
       "type": "integer",
       "default": 4
     },
-    "autorejoin": {
-      "title": "Automatic rejoin a failed master",
-      "description": "Automatic rejoin a failed master (default true)",
+    "require-ssl": {
+      "title": "Enable REQUIRE SSL for repman database users",
+      "description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
       "type": "boolean",
-      "default": true
-    },
-    "autoseed": {
-      "title": "Automatic join a standalone node",
-      "description": "Automatic join a standalone node",
-      "type": "boolean",
-      "default": true
+      "default": false
     },
     "repman-cluster-dict": {
       "title": "Replication Manager clusters definition",
@@ -189,6 +156,34 @@
               "type": "string",
               "default": ""
             },
+            "tags": {
+              "title": "Provisioning db tag list",
+              "description": "Provisioning db tags. Set one tag per line.",
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "default": [
+                "gtidstrict",
+                "bind",
+                "pkg",
+                "innodb",
+                "noquerycache",
+                "slow",
+                "pfs",
+                "linux",
+                "readonly",
+                "diskmonitor",
+                "sqlerror",
+                "compressbinlog",
+                "bm4ci",
+                "mroonga",
+                "utctime",
+                "readcommitted",
+                "nohandshake",
+                "ssl"
+              ]
+            },
             "proxysql-user": {
               "title": "Proxysql username",
               "description": "Proxysql external user, default is 'external'.",
@@ -197,7 +192,7 @@
             },
             "proxy-tags": {
               "title": "Proxy tag list",
-              "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]",
+              "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
               "type": "array",
               "items": {
                 "type": "string"
@@ -206,7 +201,8 @@
                 "pkg",
                 "masterslave",
                 "linux",
-                "noreadwritesplit"
+                "noreadwritesplit",
+                "ssl"
               ]
             },
             "logical-backup-cron": {
@@ -302,6 +298,18 @@
               "type": "boolean",
               "default": true
             },
+            "autorejoin": {
+              "title": "Automatic rejoin a failed master db",
+              "description": "Automatic rejoin a failed master (default: true).",
+              "type": "boolean",
+              "default": true
+            },
+            "autoseed": {
+              "title": "Automatic join a standalone node",
+              "description": "Automatic join a standalone mariadb node to the cluster.",
+              "type": "boolean",
+              "default": true
+            },
             "failover-mode": {
               "title": "Failover mode",
               "description": "Failover is manual or automatic (default \"manual\").",

software/repman/instance-repman-slave-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "properties": {
+    "db_user": {
+      "description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
+      "title": "Database User",
+      "type": "string",
+      "default": "user"
+    },
+    "db_password": {
+      "description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
+      "title": "Initial database password",
+      "type": "string",
+      "default": ""
+    },
+    "db_charset": {
+      "description": "Database Character set information. This parameter is set only if database is not created yet.",
+      "title": "Database Character set information",
+      "type": "string",
+      "enum": [
+        "latin1",
+        "latin2",
+        "latin5",
+        "latin7",
+        "ascii",
+        "latin5",
+        "utf8",
+        "utf8mb4",
+        "utf16",
+        "utf16le",
+        "utf32",
+        "binary",
+        "utf8_general_ci",
+        "utf8_unicode_ci",
+        "utf8_unicode_ci",
+        "utf8mb4_bin",
+        "utf8mb4_general_ci",
+        "utf8mb4_unicode_ci",
+        "latin1_general_ci",
+        "latin1_bin",
+        "ascii_general_ci",
+        "ascii_bin"
+      ],
+      "default": ""
+    }
+  },
+  "title": "Input Parameters",
+  "type": "object"
+}

software/repman/instance.cfg.in

@@ -28,8 +28,10 @@ mode    = 0644
 extensions = jinja2.ext.do
 rendered= ${buildout:directory}/${:_buildout_section_name_}
 supervisord-lib = {{ supervisord_lib }}
+causace-lib = {{ caucase_library }}
 import-list =
   file supervisord_lib :supervisord-lib
+  file caucase         :causace-lib
 context =
     key slapparameter_dict      slap-configuration:configuration
     key computer_id             slap-configuration:computer
@@ -48,6 +50,8 @@ context =
     raw bash_bin                {{ bash_location }}/bin/bash
     raw jq_bin                  {{ jq_location }}/bin/jq
     raw curl_bin                {{ curl_location }}/bin/curl
+    raw openssl_bin             {{ openssl_location }}/bin/openssl
+    raw caucase_bin_client      {{ caucase_bin_client }}
 
     ${:extra-context}
 extra-context =
@@ -56,6 +60,7 @@ extra-context =
 <= jinja2-template-base
 template= {{ template_repman_cfg }}
 extra-context =
+    key slave_instance_list         slap-configuration:slave-instance-list
     raw gowork_bin                  {{ gowork_bin }}
     raw haproxy_location            {{ haproxy_location }}
     raw nginx_bin                   {{ nginx_location }}/sbin/nginx
@@ -71,6 +76,7 @@ extra-context =
     raw proxysql_location          {{ proxysql_location }}
     raw template_repman_manager_sh {{ template_repman_manager_sh }}
     raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
+    raw template_publish_slave_information {{ template_publish_slave }}
 
 [template-mariadb-parameters]
 bash = {{ bash_location }}
@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }}
 template-init-root-wrapper = {{ template_init_root_wrapper }}
 template-mysqld-wrapper = {{ template_mysqld_wrapper }}
 template-mysqld-need-start = {{ mysqld_start_template }}
+template-manage-db = {{ template_slave_db_manage }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
 bin-directory = {{ bin_directory }}

software/repman/publish-database-slave-parameters.cfg.in

+{% set part_list = [] -%}
+
+[slap_connection]
+# Kept for backward compatibility
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}
+
+
+{% for _, database in slave_dict.items() -%}
+{% set slave_reference = database['slave_reference'] -%}
+{% do part_list.append('publish-slave-' ~ slave_reference) -%}
+
+[publish-slave-{{ slave_reference }}]
+recipe = slapos.cookbook:publish
+-slave-reference = {{ slave_reference }}
+{%   if database.get('user') -%}
+url_v4 = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":" ~ proxy_port ~ '/' ~ database['name'] }}
+url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv6 ~ "]:" ~ proxy_port ~ '/' ~ database['name'] }}
+{%   else -%}
+url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
+url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
+{%   endif -%}
+{% endfor %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+parts =
+{% for part in part_list -%}
+{{ '  %s' % part }}
+{% endfor -%}

software/repman/software.cfg

@@ -18,9 +18,10 @@ extends =
   ../../component/socat/buildout.cfg
   ../../component/rsync/buildout.cfg
   ../../component/jq/buildout.cfg
+  ../../component/openssl/buildout.cfg
   ../../stack/supervisord/buildout.cfg
   ../../stack/monitor/buildout.cfg
-  ../neoppod/software-common.cfg
+  ../../stack/caucase/buildout.cfg
 
 parts =
   slapos-cookbook
@@ -28,14 +29,43 @@ parts =
   instance.cfg
   template-mariadb.cfg
   template-mysqld-wrapper
+  caucase-eggs
   gowork
-  
+
+[mariadb]
+location = ${mariadb-10.4:location}
+
 [python]
 part = python3
 
-[gowork]
-# replication-manager does not build on golang 1.17
-golang = ${golang1.16:location}
+[template-mysqld-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
+mode = 644
+template =
+  inline:{% raw %}#!/bin/sh -e
+  basedir='${mariadb:location}'
+  datadir='{{datadir}}'
+  marker=$datadir/.slapos_initializing
+  system=$datadir/.system
+  [ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
+    find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
+    touch "$marker"
+    "$basedir/scripts/mysql_install_db" \
+      --defaults-file='{{defaults_file}}' \
+      --skip-name-resolve \
+      --auth-root-authentication-method=normal \
+      --basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
+      --datadir="$datadir"
+    rm "$marker"
+  }
+  {%- if environ is defined %}
+  {%-   for variable in environ.splitlines() %}
+  {{ variable }} \
+  {%-   endfor %}
+  {%- endif %}
+  exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
+  {% endraw %}
 
 [instance.cfg]
 recipe = slapos.recipe.template:jinja2
@@ -45,6 +75,7 @@ mode = 0644
 context =
     key bash_location bash:location
     key bin_directory buildout:bin-directory
+    key caucase_library caucase-jinja2-library:target
     key config_toml_in config-toml.in:target
     key config_cluster_toml_in config-cluster-toml.in:target
     key coreutils_location coreutils:location
@@ -70,6 +101,7 @@ context =
     key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
     key nginx_conf_in nginx.conf.in:target
     key nginx_location nginx:location
+    key openssl_location openssl:location
     key percona_toolkit_location percona-toolkit:location
     key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
     key repman_src_location git.signal18.io_signal18_repman:location
@@ -87,10 +119,12 @@ context =
     key template_init_root_sql mariadb-init-root-sql:target
     key template_init_root_wrapper init-root-wrapper-in:target
     key template_repman_cfg instance-repman.cfg:target
+    key template_publish_slave template-publish-slave-information:target
+    key template_slave_db_manage slave-db-manage.in:target
     key unixodbc_location unixodbc:location
     key sysbench_location sysbench:location
     key proxysql_location proxysql:location
-
+    raw caucase_bin_client ${buildout:bin-directory}/caucase
 
 [download-file]
 recipe  = slapos.recipe.build:download
@@ -126,6 +160,9 @@ link-binary =
 [template-mariadb-initial-setup]
 <= download-file
 
+[template-publish-slave-information]
+<= download-file
+
 [template-my-cnf]
 <= download-file
 
@@ -146,3 +183,6 @@ link-binary =
 
 [proxy-need-start-stop.sh.in]
 <= download-file
+
+[slave-db-manage.in]
+<= download-file

software/repman/software.cfg.json

@@ -10,6 +10,15 @@
       "request": "instance-repman-input-schema.json",
       "response": "instance-repman-output-schema.json",
       "index": 0
+    },
+    "default-slave": {
+      "title": "Replication Manager Database",
+      "description": "Replication Manager Database",
+      "software-type": "default",
+      "request": "instance-repman-slave-input-schema.json",
+      "response": "instance-repman-output-schema.json",
+      "shared": true,
+      "index": 1
     }
   }
 }

software/repman/templates/cluster-config.toml.in

@@ -12,6 +12,11 @@ monitoring-write-heartbeat-credential="{{ parameter_dict['heartbeat-user'] }}:{{
 db-servers-connect-timeout = 1
 slapos-db-partitions = "{{ parameter_dict['partition-list'] }}"
 slapos-proxysql-partitions = "{{ parameter_dict['proxysql-partition'] }}"
+autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
+autoseed = {{ setbool(parameter_dict['autoseed']) }}
+{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
+autorejoin-logical-backup = true
+{% endif -%}
 
 proxysql = true
 proxysql-port = {{ parameter_dict['proxy-port'] }}
@@ -40,6 +45,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
 backup-physical-type = "mariabackup"
 backup-logical-type = "mysqldump"
 scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
+#scheduler-db-servers-receiver-use-ssl   ## set to true when ssl enabled on socat
 
 prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
 prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
@@ -49,6 +55,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
 prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
 prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
 
+replication-use-ssl = true
 test-inject-traffic = true
 # failover
 failover-mode = "{{ parameter_dict['failover-mode'] }}"

software/repman/templates/config.toml.in

-{% macro setbool(value) -%}
-{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
-{% endmacro -%}
 [Default]
 api-bind = "{{ parameter_dict['ipv4'] }}"
 http-bind-address = "{{ parameter_dict['ipv4'] }}"
@@ -11,17 +8,14 @@ monitoring-save-config = false
 api-https-bind = true
 api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
 include = "{{ parameter_dict['cluster-d'] }}"
-autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
-autoseed = {{ setbool(parameter_dict['autoseed']) }}
-{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
-autorejoin-logical-backup = true
-{% endif -%}
 db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
 # Database list of hosts to ignore in election
 #db-servers-ignored-hosts =
 # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
 monitoring-address = "{{ parameter_dict['ipv4'] }}"
-monitoring-wait-retry = 40
+monitoring-wait-retry = 1200
+monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
+monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
 
 #haproxy = true
 #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
@@ -48,6 +42,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
 backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
 backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
 backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
+backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
 
 # Mail configuration
 # Alert email sender (default "mrm@localhost")

software/repman/templates/dbjobs.in

 #!/bin/bash
 
-USER={{ parameter_dict['db-user'] }}
+
+DBUSER={{ parameter_dict['db-user'] }}
 PASSWORD={{ parameter_dict['db-password'] }}
 ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
 SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
 BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
 DATADIR={{ parameter_dict['mysql-dir'] }}/
+MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
 {% if parameter_dict['use-ipv6'] == True -%}
 {% set listen = "TCP6-LISTEN" -%}
 {% else -%}
@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
 
 doneJob()
 {
-  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
+  $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
 }
 
 pauseJob()
 {
-  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
+  $MYSQL_CLIENT  -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
 }
 
 partialRestore()
 {
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
+ $MYSQL_CLIENT  -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
  for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
+ $MYSQL_CLIENT  -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
 
 
   for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo  engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo  engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
     mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
     mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
     mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
     mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
   done
   for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
-    mysql --defaults-file=/etc/mysql/dbjob.cnf  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
   done
   for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
   done
  done
  for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
  done
- cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}   -e"flush privileges;start slave;"
+ cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
+ $MYSQL_CLIENT -e "flush privileges;start slave;"
 }
 
 kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
 for job in "${JOBS[@]}"
 do
 
- TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
+ TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
 
  ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
  ID=($(echo $TASK | awk -F@ '{ print $1 }'))
  #purge de past
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
+ $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
 
   if [ "$ADDRESS" == "" ]; then
     echo "No $job needed"
@@ -77,14 +79,14 @@ do
       reseedmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
-        mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT -e 'start slave;'
       ;;
       flashbackmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
-       mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT  -e 'start slave;'
       ;;
       reseedmariabackup)
        rm -rf $BACKUPDIR
@@ -107,7 +109,7 @@ do
       ;;
       mariabackup)
        cd {{ parameter_dict['tmp-dir'] }}
-       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       error)
        cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
@@ -116,10 +118,10 @@ do
        cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       optimize)
-       mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       restart)
-        {{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
+        {{ parameter_dict['restart-script'] }} &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
   esac
   doneJob

software/repman/templates/init_root_wrapper.in

@@ -2,7 +2,7 @@
 
 run_mysql () {
   {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
-  --protocol=socket -uroot -hlocalhost $@
+  --protocol=socket -hlocalhost "$@"
 }
 
 if [ ! -f "{{ init_password_done }}" ]; then
@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
   fi
   echo "Setting mariabdb root password...";
   run_mysql  < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
+  echo "remove unwanted users...";
+  run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
+  run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
   echo "done"
 fi
 
@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then
   if [ $? -eq 0 ]; then
     touch {{ upgrade_done }};
   fi
+else
+  # only create databases now
+  run_mysql  < {{ init_database_sql }} || exit 1;
 fi

software/repman/templates/mariadb_init_root.sql.in

@@ -2,16 +2,18 @@
 --  or products like mysql-fabric won't work
 SET @@SESSION.SQL_LOG_BIN=0;
 
+# {{ parameter_dict["require-ssl"] }}
+{% set ssl = parameter_dict["require-ssl"] -%}
 CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
-CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
 CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
 
 CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
-CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
 
 DROP DATABASE IF EXISTS test ;

software/repman/templates/mariadb_initial_setup.sql.in

 SET @@SESSION.SQL_LOG_BIN=0;
 
 USE mysql;
-{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
-{% if mroonga %}
-SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
-{% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 
-{% macro database(name, user, password) -%}
-CREATE DATABASE IF NOT EXISTS `{{ name }}`;
+CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
+
+{% macro database(name, user, password, ssl, charset="") -%}
+{% set charset_collate = "" -%}
+{% set charset_dict = {'utf8_general_ci': 'utf8',
+                       'utf8_unicode_ci': 'utf8',
+                       'utf8mb4_bin': 'utf8mb4',
+                       'utf8mb4_general_ci': 'utf8mb4',
+                       'utf8mb4_unicode_ci': 'utf8mb4',
+                       'latin1_general_ci': 'latin1',
+                       'latin1_bin': 'latin1',
+                       'ascii_general_ci': 'ascii',
+                       'ascii_bin': 'ascii'} -%}
+{% if charset %}
+{%   set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
+{%   if charset_dict.get(charset) -%}
+{%     set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
+{%   endif -%}
+{% endif -%}
+CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
 {% if user -%}
-GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
+GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
 {%- endif %}
 {% endmacro -%}
 
 {% for entry in parameter_dict['database-list'] -%}
-{{   database(entry['name'], entry.get('user'), entry.get('password')) }}
+{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
 {% endfor -%}

software/repman/templates/mysqld-need-start.sh.in

@@ -4,13 +4,11 @@ curl () {
   {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
 }
 
-# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
-
 # Checking if mariadb start is needed
-#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
 CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
 
 if [ $CODE -eq 200 ]; then
+  sleep 5
   echo "$CODE: Updating mysql configuration..."
   # update mysql configuration
   {{ update_config }}
@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
   sleep 5
   # check again if the service is still up...
   {{ mariadb_controller }} status mariadb
+else
+  echo "CODE: $CODE. No need start!"
 fi

software/repman/templates/slave-db-manage.in

+#!/bin/bash
+
+set -e
+
+curl () {
+  {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
+}
+
+get_token () {
+  curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
+}
+
+run_mysql () {
+  {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
+}
+
+TOKEN=$(get_token | {{ jq_bin }} -r '.token')
+DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \   {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }}  -r '.slaposDatadir')
+
+# Only write or delete on master database else, we break replication.
+if [ "$DATADIR" = "{{ partition_dir }}" ]; then
+
+  cat << EOF > {{ tmp_dir }}/.script.sql
+use repman_slave_definition;
+
+CREATE TABLE IF NOT EXISTS \`slave\` (
+  \`name\` varchar(80) NOT NULL,
+  \`state\` tinyint(1) DEFAULT NULL,
+  PRIMARY KEY (\`name\`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+UPDATE \`slave\` set \`state\`=false;
+{% for name in database_name.split(' ') -%}
+{%  if name -%}
+REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
+{%  endif -%}
+{% endfor -%}
+
+EOF
+
+  # Update requested slaves database
+  run_mysql < {{ tmp_dir }}/.script.sql
+  rm -f {{ tmp_dir }}/.script.sql
+
+  DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
+  RET=$?
+
+  if [ ! "$RET" = "0" ]; then
+    echo "Mysql command failed: $DBNAME"
+    exit $RET
+  fi
+  if [ -z "$DBNAME" ]; then
+    echo "No database for slave to remove.";
+  fi
+
+  for NAME in $DBNAME; do
+    if [ ! -z "$NAME" ]; then
+      echo "Deleting database $NAME..."
+      run_mysql -e "DROP DATABASE IF EXISTS $NAME";
+      run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
+      echo "Done."
+    fi
+  done
+fi

software/repman/test/test.py

@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase):
 
     token = resp.json()['token']
     headers = {"authorization": "Bearer " + token}
-    resp = requests.get(
-        urljoin(self.url, '/api/monitor'),
-        headers=headers,
-        verify=False,
-    )
-    self.assertEqual(resp.status_code, requests.codes.ok)
 
     for i in range(20):
       resp = requests.get(
@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase):
     self.assertTrue(cluster['isProvision'])
     self.assertTrue(cluster['isFailable'])
     self.assertFalse(cluster['isDown'])
+
+    resp = requests.get(
+        urljoin(self.url, '/api/clusters/cluster1/status'),
+        headers=headers,
+        verify=False,
+    )
+    self.assertEqual(resp.status_code, requests.codes.ok)
+    self.assertEqual(resp.json(), {"alive": "running"})

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 597c29546519aabe7259e416d0b92095
+md5sum = ba6c572678687081ce111d3805be3cee

software/slapos-testing/instance.cfg

@@ -35,10 +35,10 @@ repository = ${caucase-repository:location}
 [caucase-test-runner]
 recipe = slapos.recipe.template:jinja2
 template = inline:#!/bin/sh
-  export HOSTS="$(mktemp)"
-  trap 'rm "$HOSTS"' EXIT
-  printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS"
-  export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/userhosts.so:$LD_PRELOAD
+  export HOSTS_FILE="$(mktemp)"
+  trap 'rm "$HOSTS_FILE"' EXIT
+  printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS_FILE"
+  export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/libuserhosts.so:$LD_PRELOAD
   exec python -m unittest discover -v
 rendered = $${caucase:location}/host_setting.sh
 

software/theia/buildout.hash.cfg

@@ -47,7 +47,7 @@ md5sum = 1a668d6203d42b4d46d56e24c7606cb2
 
 [python-language-server-requirements.txt]
 _update_hash_filename_ = python-language-server-requirements.txt
-md5sum = 9f478fd1b03b7738f3de549cb899bf54
+md5sum = e16f1414f2657fa5ffc949839207f41b
 
 [slapos.css.in]
 _update_hash_filename_ = slapos.css.in

software/theia/python-language-server-requirements.txt

 appdirs==1.4.4
-astroid==2.5.2
-attrs==20.3.0
+astroid==2.6.2
+attrs==21.2.0
 black==20.8b1
-certifi==2020.12.5
+cachetools==4.2.4
+certifi==2021.5.30
 chardet==4.0.0
 click==7.1.2
 future==0.18.2
 idna==2.10
 importlib-metadata==3.10.0
-isort==5.8.0
+iniconfig==1.1.1
+isort==5.9.1
 jedi==0.18.0
-lazy-object-proxy==1.4.3
+lazy-object-proxy==1.6.0
 mccabe==0.6.1
-mypy==0.812
 mypy-extensions==0.4.3
+mypy==0.930
+packaging==21.0
 parso==0.8.2
 pathspec==0.8.1
 pluggy==0.13.1
-pydantic==1.8.1
+pydantic==1.8.2
 pydocstyle==6.0.0
 pyflags==0.1
 pyflakes==2.1.0
-pygls==0.10.2
-pylint==2.7.4
+pygls==0.11.1
+pylint==2.9.3
+pyparsing==2.4.7
 regex==2021.4.4
 requests==2.25.1
 rope==0.11.0
-six==1.12.0
+six==1.16.0
 snowballstemmer==1.2.1
-toml==0.10.2
+tomli==1.2.3
 typed-ast==1.4.1
-typeguard==2.12.0
-typing-extensions==3.7.4.3
-urllib3==1.26.4
-wrapt==1.11.2
-yapf==0.29.0
+typeguard==2.12.1
+typing-extensions==4.0.1
+urllib3==1.26.6
+wrapt==1.12.1
+yapf==0.31.0
+zc.buildout.languageserver==0.6.2
 zc.buildout==2.13.4
-zc.buildout.languageserver==0.5.0
-zipp==3.4.1
+zipp==3.4.1
\ No newline at end of file

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = c82ea00c4514b72fb97a6fa7ac36ec52
+md5sum = f553aa7d6596dcf98e7e61bfb6bd81c7
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -46,7 +46,7 @@ md5sum = 1de449e8c0c4a85c5ce2b447785b7654
 
 [template-mariadb-initial-setup]
 filename = mariadb_initial_setup.sql.in
-md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
+md5sum = f928b9dc99f7f970caadfe7dd6f95d34
 
 [template-postfix]
 filename = instance-postfix.cfg.in
@@ -70,7 +70,7 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
 
 [template]
 filename = instance.cfg.in
-md5sum = bbef65b4edeb342f08309604ca3717d5
+md5sum = 0444d646607f8643e987bb62d1cbbc1d
 
 [template-erp5]
 filename = instance-erp5.cfg.in
@@ -86,11 +86,11 @@ md5sum = bc821f9f9696953b10a03ad7b59a1936
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = f3121380ab4d31ba5f4984aec74d0a2f
+md5sum = 09895deebca3206425fada3caf1d92cf
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = d6166515fda7b09df754672536b131be
+md5sum = cbcbe2daf3a51dfab50446a18c0be76e
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -189,7 +189,7 @@ cert = ${haproxy-conf-ssl:certificate}
 ca-cert = ${haproxy-conf-ssl:ca-cert}
 crl = ${haproxy-conf-ssl:crl}
 {% endif %}
-stats-socket = ${directory:run}/haproxy.sock
+stats-socket = ${directory:run}/ha.sock
 path-routing-list = {{ dumps(slapparameter_dict['path-routing-list']) }}
 family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict']) }}
 pidfile = ${directory:run}/haproxy.pid

stack/erp5/instance-mariadb.cfg.in

 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
-{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
+{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure', 'with-process-privilege': True}]) -%}
 {% set test_database_list = [] %}
 {% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
 {%   do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}

stack/erp5/instance-zope.cfg.in

@@ -185,7 +185,7 @@ context = key host_dict hosts-parameter:host-dict
 recipe = slapos.recipe.template:jinja2
 mode = 755
 template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
-  export HOSTS=${hosts:rendered}
+  export HOSTS_FILE=${hosts:rendered}
   export LD_PRELOAD={{ parameter_dict['userhosts'] }}:$LD_PRELOAD
   exec "$@"
 rendered = ${directory:bin}/userhosts

stack/erp5/instance.cfg.in

@@ -110,7 +110,7 @@ link-binary = {{ dumps(zope_link_binary) }}
 fonts = {{ dumps(zope_fonts) }}
 fontconfig-includes = {{ dumps(zope_fontconfig_includes) }}
 template-fonts-conf = {{ dumps(template_fonts_conf) }}
-userhosts = {{ userhosts_location }}/lib/userhosts.so
+userhosts = {{ userhosts_location }}/lib/libuserhosts.so
 site-zcml = {{ site_zcml }}
 extra-path-list = {{ dumps(extra_path_list) }}
 matplotlibrc = {{ matplotlibrc_location }}

stack/erp5/mariadb_initial_setup.sql.in

@@ -4,16 +4,19 @@ USE mysql;
 SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
 {% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
-#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 
-{% macro database(name, user, password) -%}
+{% macro database(name, user, password, with_process_privilege) -%}
 CREATE DATABASE IF NOT EXISTS `{{ name }}`;
 {% if user -%}
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
+{%   if with_process_privilege %}
+GRANT PROCESS ON *.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
+GRANT PROCESS ON *.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
+{%-   endif %}
 {%- endif %}
 {% endmacro -%}
 
 {% for entry in parameter_dict['database-list'] -%}
-{{   database(entry['name'], entry.get('user'), entry.get('password')) }}
+{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('with-process-privilege')) }}
 {% endfor -%}

stack/slapos.cfg

@@ -132,7 +132,7 @@ eggs =
 [versions]
 setuptools = 44.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos014
+zc.buildout = 2.7.1+slapos015
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 
@@ -167,7 +167,7 @@ importlib-metadata = 1.7.0:whl
 inotify-simple = 1.1.1
 itsdangerous = 0.24
 lock-file = 2.0
-lxml = 4.6.3
+lxml = 4.6.5
 meld3 = 1.0.2
 mock = 3.0.5
 more-itertools = 5.0.0
@@ -189,14 +189,14 @@ setproctitle = 1.1.10
 setuptools-dso = 1.7
 rubygemsrecipe  = 0.4.3
 six = 1.12.0
-slapos.cookbook = 1.0.220
+slapos.cookbook = 1.0.226
 slapos.core = 1.7.2
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.22
 slapos.rebootstrap = 4.5
-slapos.recipe.build = 0.52
-slapos.recipe.cmmi = 0.18
+slapos.recipe.build = 0.53
+slapos.recipe.cmmi = 0.19
 slapos.recipe.template = 4.6
 slapos.toolbox = 0.126
 stevedore = 1.21.0:whl