Update Release Candidate

.gitignore

@@ -14,3 +14,4 @@ slapos.cookbook.egg-info
 .eggs/
 *.egg/
 TEST_KNOWN_HOSTS
+node_modules

component/apache/buildout.cfg

@@ -38,9 +38,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
 shared = false
-version = 2.4.33
+version = 2.4.37
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = 6ef469d3f16fffeb688bc6e0346823e5
+md5sum = 6a36e742180ee74bff97b28eee90c3f7
 pre-configure =
   cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
   cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util

component/chromedriver/buildout.cfg

+# ChromeDriver - Webdriver for Chrome
+# http://chromedriver.chromium.org/
+# This is a binary download with wrapper scripts.
+
+[buildout]
+extends =
+  ../chromium/buildout.cfg
+  ../nss/buildout.cfg
+  ../nspr/buildout.cfg
+
+parts =
+  chromedriver-wrapper
+
+
+
+[chromedriver-wrapper-2.41]
+<= chromedriver-wrapper
+wrapper-name = chromedriver-2.41
+part = ${chromedriver-2.41:location}
+
+[chromedriver-wrapper]
+# generate a wrapper named ${:wrapper-name} setting $LD_LIBRARY_PATH
+# to run chromedriver installed from ${:part}
+wrapper-name = chromedriver
+part = ${chromedriver:location}
+
+recipe = slapos.recipe.build
+location = ${buildout:bin-directory}/${:wrapper-name}
+script =
+  chromedriver = self.options['part']
+  part = self.buildout[os.path.basename(chromedriver)]
+  with open(%(location)r, 'w') as f:
+    f.write("""#!/bin/sh -e
+  cd {}
+  export LD_LIBRARY_PATH=$PWD:{}
+  export PATH={}:$PATH
+  exec ./chromedriver "$@"
+  """.format(
+    chromedriver,
+    ':'.join(part['library'].split()),
+    ':'.join(part['path'].split()),
+  ))
+    os.fchmod(f.fileno(), 0o755)
+
+
+[chromedriver]
+<= chromedriver-2.41
+
+[chromedriver-2.41]
+<= chromedriver-download
+version = 2.41
+# Supports Chrome v67-69
+md5sum-x86_64 = fbd8b9561575054e0e7e9cc53b680a70
+
+
+[chromedriver-download]
+# Installs chromedriver ${version}.
+# This chromedriver is not usable directly, it needs a wrapper.
+version =
+md5sum-x86_64 =
+
+recipe = slapos.recipe.build
+x86-64 = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_linux64.zip ${:md5sum-x86_64}
+library =
+  ${nss:location}/lib
+  ${nspr:location}/lib
+path =
+
+script =
+  url, md5sum = self.options[guessPlatform()].split()
+  extract_dir = self.extract(self.download(url, md5sum))
+  os.mkdir(%(location)r)
+  shutil.copy(extract_dir + '/chromedriver', %(location)r)
+

component/chromium/buildout.cfg

+# Chromium binary build, from "Downloading old builds of Chrome / Chromium" of
+# https://www.chromium.org/getting-involved/download-chromium
+# Currently, only Linux_x64 is supported.
+
+
 [buildout]
 # chromium binary is linking to libudev, that can be provided by
 # component/systemd.  but current component/systemd can be built only
 # with Linux kernel >= 3.7. so libudev is not included in
 # LD_LIBRARY_PATH below intentionally and libudev should be installed
 # in the system.
-
 extends =
+  ../alsa/buildout.cfg
   ../coreutils/buildout.cfg
   ../cups/buildout.cfg
   ../dbus/buildout.cfg
@@ -14,6 +19,7 @@ extends =
   ../gettext/buildout.cfg
   ../glib/buildout.cfg
   ../gtk-2/buildout.cfg
+  ../gtk-3/buildout.cfg
   ../libexpat/buildout.cfg
   ../libffi/buildout.cfg
   ../libpng/buildout.cfg
@@ -27,77 +33,133 @@ extends =
   ../zlib/buildout.cfg
 
 parts =
-  chromium
+  chromium-wrapper
+
+
+[chromium-wrapper]
+# Install a chromium wrapper named ${:wrapper-name} in buildout's bin-directory,
+# wrapping chromium installed in ${:part}
+# This [chromium-wrapper] installs the default chromium with name `chromium` and
+# can also be used as a macro to install under a different name.
+wrapper-name = chromium
+part = ${chromium:location}
+
+recipe = slapos.recipe.build
+location = ${buildout:bin-directory}/${:wrapper-name}
+script =
+  chromeium = self.options['part']
+  part = self.buildout[os.path.basename(chromeium)]
+  with open(%(location)r, 'w') as f:
+    f.write("""#!/bin/sh -e
+  cd {}
+  # this also needs a $FONTCONFIG_FILE set, otherwise system fonts will be used and if
+  # no system fonts are available, chrome will refuse to start with this error:
+  # FATAL:platform_font_linux.cc(83)] Check failed: InitDefaultFont(). Could not find the default font
+  export LD_LIBRARY_PATH=$PWD:{}
+  export PATH={}:$PATH
+  exec ./chrome --disable-setuid-sandbox --no-sandbox --disable-gpu $@
+  """.format(
+    chromeium,
+    ':'.join(part['library'].split()),
+    ':'.join(part['path'].split()),
+  ))
+    os.fchmod(f.fileno(), 0o755)
+
+[chromium-wrapper-69]
+<= chromium-wrapper
+wrapper-name = chromium-69
+part = ${chromium-69:location}
+
+
+
 
 [chromium]
+<= chromium-69
+
+
+[chromium-69]
+<= chromium-download
+version = 69.0.3497.0
+
+revision_x86-64 = 576753
+md5sum-x86_64 = 08ac27fd40ace4ca8dfbd1db403deccb
+generation-x86_64 = 1532051976706023
+
+
+[chromium-download]
+# macro to download a binary build of chromium and generate a
+# wrapper as chrome-slapos in the part directory
+
+# the full version tag
+version =
+
+# needs the following URL parts:
+# revision is from the "base position"
+revision_x86-64 =
+# generation is in the final download URL
+generation-x86_64 =
+# this is the md5sum of the downloaded archive
+md5sum-x86_64 =
+
+
 recipe = slapos.recipe.build
 slapos_promise =
   file:chrome
-  file:chrome-wrapper
-  file:chrome-slapos
 
-# How to get the revision :
-# stable : https://www.googleapis.com/download/storage/v1/b/chromium-browser-continuous/o/Linux_x64%2FLAST_CHANGE?alt=media
-# snapshot : https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2FLAST_CHANGE?alt=media
-revision_x86 = 382014
-revision_x86-64 = 382014
-revision_mac = 381909
 
-linux_x86 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-continuous/o/Linux%2F${:revision_x86}%2Fchrome-linux.zip?alt=media
-linux_x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-continuous/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?alt=media ef2c476b1f059e9aa026bbac1872368d
-mac_x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-continuous/o/Mac%2F${:revision_mac}%2Fchrome-linux.zip?alt=media
+x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
+
+
+library =
+  ${atk:location}/lib
+  ${at-spi2-atk:location}/lib
+  ${alsa:location}/lib
+  ${cairo:location}/lib
+  ${cups:location}/lib
+  ${cups:location}/lib64
+  ${dbus:location}/lib/
+  ${dbus-glib:location}/lib/
+  ${fontconfig:location}/lib/
+  ${gdk-pixbuf:location}/lib
+  ${gettext:location}/lib
+  ${glib:location}/lib
+  ${gtk-3:location}/lib
+  ${harfbuzz:location}/lib
+  ${libX11:location}/lib
+  ${libXau:location}/lib
+  ${libXcomposite:location}/lib
+  ${libXcursor:location}/lib
+  ${libXext:location}/lib
+  ${libXi:location}/lib
+  ${libXrender:location}/lib/
+  ${libXtst:location}/lib
+  ${libXScrnSaver:location}/lib
+  ${libXrandr:location}/lib
+  ${libexpat:location}/lib
+  ${libffi:location}/lib
+  ${libpng:location}/lib
+  ${libpng12:location}/lib
+  ${libxcb:location}/lib
+  ${libxml2:location}/lib
+  ${mesa:location}/lib
+  ${nspr:location}/lib
+  ${nss:location}/lib
+  ${pango:location}/lib
+  ${pcre:location}/lib
+  ${pixman:location}/lib
+  ${sqlite3:location}/lib
+  ${xdamage:location}/lib
+  ${xfixes:location}/lib
+  ${zlib:location}/lib
+path =
+  ${fontconfig:location}/bin
 
 script =
-  #If part directory already exist, will just throw an error.
+  url, md5sum = self.options[guessPlatform()].split()
+  extract_dir = self.extract(self.download(url, md5sum))
+  self.copyTree(guessworkdir(extract_dir), %(location)r)
+  # XXX adjust some permissions
   import os
-  platform = '%%s_%%s' %% (guessOperatingSystem(), guessPlatform())
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[platform].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
-  workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, "%(location)s")
-  wrapper_location = os.path.join("%(location)s", "chrome-slapos")
-  wrapper = open(wrapper_location, 'w')
-  wrapper.write("""#!/bin/sh
-  export LD_LIBRARY_PATH="%(location)s"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${atk:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${cairo:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${cups:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${dbus:location}/lib/"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${dbus-glib:location}/lib/"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${fontconfig:location}/lib/"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${gdk-pixbuf:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${gettext:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${glib:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${gtk-2:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${harfbuzz:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libX11:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libXau:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libXcomposite:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libXcursor:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libXext:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libXi:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libXrender:location}/lib/"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libXtst:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libexpat:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libffi:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libpng:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libpng12:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libxcb:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libxml2:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${mesa:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${nspr:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${nss:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${pango:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${pcre:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${pixman:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${sqlite3:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${xdamage:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${xfixes:location}/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${zlib:location}/lib"
-  exec %(location)s/chrome --disable-setuid-sandbox --disable-gpu $*""")
-  wrapper.flush()
-  wrapper.close()
   os.system('"${findutils:location}/bin/find" "%(location)s" -type d -exec "${coreutils:location}/bin/chmod" a+rx {} \;')
   os.system('"${findutils:location}/bin/find" "%(location)s" -type f -executable -exec "${coreutils:location}/bin/chmod" a+rx {} \;')
   os.system('"${findutils:location}/bin/find" "%(location)s" -type f -exec "${coreutils:location}/bin/chmod" a+r {} \;')
-  os.chmod(wrapper_location, 0755)

component/firefox/buildout.cfg

@@ -31,7 +31,6 @@ part = ${firefox:location}
 
 recipe = slapos.recipe.build
 location = ${buildout:bin-directory}/${:wrapper-name}
-part = ${firefox:location}
 script =
   firefox = self.options['part']
   part = self.buildout[os.path.basename(firefox)]
@@ -48,6 +47,11 @@ script =
   ))
     os.fchmod(f.fileno(), 0o755)
 
+[firefox-wrapper-60]
+<= firefox-wrapper
+wrapper-name = firefox-60
+part = ${firefox-60:location}
+
 [firefox-wrapper-52]
 <= firefox-wrapper
 wrapper-name = firefox-52
@@ -67,6 +71,12 @@ part = ${firefox-51:location}
 # would not be created.
 <= firefox-52
 
+[firefox-60]
+<= firefox-download
+version = 60.0.2esr
+i686-md5sum = ce7c80716036dfb5c2fb1ca2538556ff
+x86_64-md5sum = 6fe25d9a3fcc82670320242c9047d1da
+
 [firefox-52]
 <= firefox-download
 version = 52.9.0esr
@@ -142,15 +152,38 @@ script =
 
 
 [geckodriver]
-# Current geckodriver installed as buildout:bin-directory/geckodriver
+# Current geckodriver installed as ${buildout:bin-directory}/geckodriver
 <= geckodriver-0.17.0
 
+[geckodriver-0.22.0]
+<= geckodriver-base
+version = 0.22.0
+i686-md5sum = 6de7544753fda56fbaa8382dcac99aaa
+x86_64-md5sum = 81746200ce5841e00cabf3b8ea7db542
+
+[geckodriver-0.21.0]
+<= geckodriver-base
+version = 0.21.0
+i686-md5sum = 9fc1657dd1b94272d0cdb3b29ca80f79
+x86_64-md5sum = 6bc36d4fd4975e296cdb3fa3c5e26a41
+
+[geckodriver-0.19.0]
+<= geckodriver-base
+version = 0.19.0
+i686-md5sum = 07cd383c8aef8ea5ef194a506141afd6
+x86_64-md5sum = ca6935a72fd0527d15a78a17a35e56e8
+
 [geckodriver-0.17.0]
 <= geckodriver-base
 version = 0.17.0
 i686-md5sum = 79b1a158f96d29942a111c0905f1c807
 x86_64-md5sum = be18faeea6e7db9db6990d8667e2298f
 
+[geckodriver-0.16.1]
+<= geckodriver-base
+version = 0.16.1
+i686-md5sum = not not on github
+x86_64-md5sum = 57dfd55d4759d9878eb75b4c0123d00c
 
 [geckodriver-0.14.0]
 <= geckodriver-base

component/fonts/buildout.cfg

 [buildout]
 extends =
   ../xz-utils/buildout.cfg
+  ../p7zip/buildout.cfg
 
 parts =
   liberation-fonts

component/gcc/buildout.cfg

 # GNU C Compiler
-# Mostly required to support languages different than C or C++
+# Required to use a "known good version" of the compiler or to support languages different than C or C++
 [buildout]
 extends =
   ../gettext/buildout.cfg
@@ -50,3 +50,23 @@ configure-options =
 environment =
   LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
   PATH=${perl:location}/bin:${tar:location}/bin:%(PATH)s
+
+
+[gcc-8.2]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://ftp.gnu.org/gnu/gcc/gcc-8.2.0/gcc-8.2.0.tar.gz
+md5sum = ee04f0c22a941f5f17d93809387f2729
+configure-options =
+  --disable-bootstrap
+  --disable-multilib
+  --with-gmp=${gmp:location}
+  --with-mpfr=${mpfr:location}
+  --with-mpc=${mpc:location}
+  --enable-languages="c,c++,fortran"
+  --with-isl=${isl:location}
+  --with-ld=${binutils:location}/bin/ld
+  --with-as=${binutils:location}/bin/as
+environment =
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
+  PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s

component/geoip2/buildout.cfg

@@ -17,5 +17,5 @@ rpath = ${:library-dirs}
 [geolite2-country]
 recipe = slapos.recipe.build:download-unpacked
 url = http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz#${:md5sum}
-md5sum = 4d4bf41db15f99b227a1009043963102
+md5sum = 3ae76aa88be4401923acc3acf34e0faf
 strip-top-level-dir = true

component/golang/buildout.cfg

@@ -2,6 +2,7 @@
 [buildout]
 extends =
   ../findutils/buildout.cfg
+  ../gcc/buildout.cfg
   ../git/buildout.cfg
   ../pkgconfig/buildout.cfg
 
@@ -18,6 +19,7 @@ make-targets= cd src && ./all.bash && cp -alf .. ${:location}
 # some testdata files have an issue with slapos.extension.strip.
 post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true
 environment =
+  PATH=${gcc-8.2:location}/bin:%(PATH)s
   GOROOT_FINAL=${:location}
   ${:environment-extra}
 
@@ -61,8 +63,8 @@ environment-extra =
 #
 #   [gowork]
 #   install =
-#       lab.nexedi.com/kirr/neo/go/...  \
-#       github.com/pkg/profile          \
+#       lab.nexedi.com/kirr/neo/go/...
+#       github.com/pkg/profile
 #       golang.org/x/perf/cmd/benchstat
 #
 # For Cgo support pkg-config is made pre-available by gowork, and users
@@ -120,7 +122,7 @@ stop-on-error = true
 # clients should put package list to install to gowork:install ("..." requests installing everything)
 [gowork.goinstall]
 recipe  = plone.recipe.command
-command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v ${gowork:install}"
+command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
 update-command = ${:command}
 stop-on-error = true
 

component/gtk-2/buildout.cfg

@@ -16,6 +16,7 @@ extends =
   ../perl/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../xorg/buildout.cfg
+  ../icu/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
 
@@ -49,13 +50,13 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-0.9.35.tar.bz2
 md5sum = 531ee8650626ecddcd90b2a4637e31d4
-pkg_config_depends = ${cairo:location}/lib/pkgconfig:${cairo:pkg_config_depends}
+pkg_config_depends = ${cairo:location}/lib/pkgconfig:${cairo:pkg_config_depends}:${icu4c:location}/lib/pkgconfig
 configure-options =
   --disable-static
   --disable-gtk-doc-html
   --with-cairo
   --with-freetype
-  --without-icu
+  --with-icu
 environment =
   PATH=${glib:location}/bin:${freetype:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
@@ -126,5 +127,5 @@ environment =
   PATH=${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
 # not taken from pkg-config result...
-  CPPFLAGS=-I${libX11:location}/include/ -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${render:location}/include -I${libXext:location}/include
+  CPPFLAGS=-I${libX11:location}/include/ -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${renderext:location}/include -I${libXext:location}/include
   LDFLAGS=-L${libX11:location}/lib -L${libXext:location}/lib -L${libXrender:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/gtk-3/buildout.cfg

@@ -35,19 +35,24 @@ environment =
 recipe = slapos.recipe.cmmi
 url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/gtk+-3.22.3.tar.xz
 md5sum = f0e6492896a2ca244501142319adaa95
-pkg_config_depends = ${at-spi2-atk:location}/lib/pkgconfig:${at-spi2-core:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${libepoxy:location}/lib/pkgconfig:${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${libXi:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
+pkg_config_depends = ${at-spi2-atk:location}/lib/pkgconfig:${at-spi2-core:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${libepoxy:location}/lib/pkgconfig:${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${libXi:pkg_config_depends}:${libXi:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libX11:pkg_config_depends}:${libX11:location}/lib/pkgconfig:${xorgproto:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${xcbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXinerama:location}/lib/pkgconfig:${libXrandr:pkg_config_depends}:${libXrandr:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${libXcomposite:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${compositeproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig:${damageproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig
 configure-options =
   --disable-static
   --disable-glibtest
   --disable-cups
   --disable-papi
   --enable-explicit-deps
-  --disable-xinerama
   --disable-gtk-doc-html
   --disable-man
+  --enable-x11-backend
+  --enable-xrandr
+  --enable-xfixes
+  --enable-xcomposite
+  --enable-xdamage
 environment =
   PATH=${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
 # not taken from pkg-config result...
-  CPPFLAGS=-I${cairo:location}/include -I${inputproto:location}/include -I${libX11:location}/include -I${libXi:location}/include -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${render:location}/include -I${libXext:location}/include
-  LDFLAGS=-L${libX11:location}/lib -L${libXi:location}/lib -L${libXext:location}/lib -L${libXrender:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  CPPFLAGS=-I${cairo:location}/include -I${inputproto:location}/include -I${libX11:location}/include -I${libXi:location}/include -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${renderext:location}/include -I${libXext:location}/include -I${libX11:location}/include  -I${libXinerama:location}/include -I${gdk-pixbuf:location}/include -I${libXrandr:location}/include -I${xfixes:location}/include -I${libXcomposite:location}/include -I${xdamage:location}/include -I${fixesproto:location}/include
+  LDFLAGS=-L${libX11:location}/lib -L${libXi:location}/lib -L${libXext:location}/lib -L${libXrender:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libX11:location}/lib -Wl,-rpath=${libX11:location}/lib -L${libXinerama:location}/lib -Wl,-rpath=${libXinerama:location}/lib -L${gdk-pixbuf:location}/lib -Wl,-rpath=${gdk-pixbuf:location}/lib -L${libXrandr:location}/lib -Wl,-rpath=${libXrandr:location}/lib -L${xfixes:location}/lib -Wl,-rpath=${xfixes:location}/lib -L${libXcomposite:location}/lib -Wl,-rpath=${libXcomposite:location}/lib -L${xdamage:location}/lib -Wl,-rpath=${xdamage:location}/lib
+

component/gtkmm/buildout.cfg

@@ -26,7 +26,7 @@ configure-options =
 environment =
   PATH=${freetype:location}/bin:${gcc:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
-  CPPFLAGS=-I${libX11:location}/include -I${libXrender:location}/include -I${render:location}/include -I${xproto:location}/include
+  CPPFLAGS=-I${libX11:location}/include -I${libXrender:location}/include -I${renderext:location}/include -I${xproto:location}/include
   LD_LIBRARY_PATH=${gcc:location}/lib:${gcc:location}/lib64
   LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -L${libX11:location}/lib -Wl,-rpath=${libX11:location}/lib -L${libXext:location}/lib -Wl,-rpath=${libXext:location}/lib -L${libXrender:location}/lib -Wl,-rpath=${libXrender:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
 

component/kumo/buildout.cfg

@@ -29,3 +29,4 @@ environment =
   LD_LIBRARY_PATH=${gcc:location}/lib:${gcc:location}/lib64
   LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${messagepack:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
   PATH=${gcc:location}/bin:${patch:location}/bin:%(PATH)s
+# KumoFS is known not to build with gcc>6, so we use our own gcc which is 5.5
\ No newline at end of file

component/nodejs/buildout.cfg

@@ -15,12 +15,24 @@ parts =
 [nodejs]
 <= nodejs-0.12
 
+[nodejs-8.9.4]
+<= nodejs-base
+recipe = slapos.recipe.cmmi
+version = v8.9.4
+md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
+
 [nodejs-8.6.0]
+<= nodejs-base
+version = v8.6.0
+md5sum = 0c95e08220667d8a18b97ecec8218ac6
+
+[nodejs-base]
 # Server-side Javascript.
+version =
+md5sum =
+
 recipe = slapos.recipe.cmmi
-version = v8.6.0
 url = https://nodejs.org/dist/${:version}/node-${:version}.tar.gz
-md5sum = 0c95e08220667d8a18b97ecec8218ac6
 configure-options =
   --shared-openssl
   --shared-openssl-includes=${openssl:location}/include

component/python-pynacl/buildout.cfg

+[buildout]
+extends =
+  ../libffi/buildout.cfg
+  ../pkgconfig/buildout.cfg
+
+parts =
+  python-pynacl
+
+[python-pynacl]
+recipe = zc.recipe.egg:custom
+egg = pynacl
+environment = python-pynacl-env
+library-dirs =
+  ${libffi:location}/lib/
+rpath =
+  ${libffi:location}/lib/
+
+[python-pynacl-env]
+PATH = ${pkgconfig:location}/bin:%(PATH)s
+PKG_CONFIG_PATH = ${libffi:location}/lib/pkgconfig

component/python3/buildout.cfg

@@ -30,7 +30,7 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/default_encoding.patch#4ad9664e622d5556b4c32b1d9cb587ff
 url =
-  http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
+  https://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =
   --enable-ipv6
   --without-ensurepip
@@ -52,3 +52,9 @@ environment =
 <= python3-common
 package_version = 3.5.3
 md5sum = 57d1f8bfbabf4f2500273fb0706e6f21
+
+[python3.6.6]
+<= python3-common
+package_version = 3.6.6
+md5sum = c3f30a0aff425dda77d19e02f420d6ba
+executable = @@LOCATION@@/bin/python3.6

component/rdiff-backup/buildout.cfg

@@ -25,7 +25,7 @@ find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1
 [rdiff-backup-build-1.3.4]
 <= rdiff-backup-build
 # use our own version
-find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd5.tar.gz
+find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd6.tar.gz
 patches =
   ${:_profile_base_location_}/rdiff-backup-1.3.4-librsync-1.0.0.patch#31fafc8bc4a00f002f52008a9f3b671f
 

component/tomcat/buildout.cfg

@@ -20,8 +20,8 @@ md5sum = 3dde098fd0b3a08d3f2867e4a95591ba
 recipe = hexagonit.recipe.download
 ignore-existing = true
 strip-top-level-dir = true
-url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.84/bin/apache-tomcat-7.0.84.tar.gz
-md5sum = 1c6f2c06a90bd7d8a19522749c219a2a
+url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.91/bin/apache-tomcat-7.0.91.tar.gz
+md5sum = 8bfbb358b51f90374067879f8db1e91c
 
 [tomcat7-output]
 # Shared binary location to ease migration

component/webp/buildout.cfg

@@ -19,6 +19,7 @@ configure-options =
   --disable-static
   --disable-gl
   --disable-wic
+  --enable-everything
   --with-jpegincludedir=${libjpeg:location}/include
   --with-jpeglibdir=${libjpeg:location}/lib
   --with-tiffincludedir=${libtiff:location}/include

component/xorg/buildout.cfg

@@ -33,14 +33,14 @@ ACLOCAL=${xorg-util-macros:location}/share/aclocal
 [xorg-util-macros]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/util-macros-1.17.tar.bz2
-md5sum = 4f41667e1bf4938bb2b24fa09d517f77
+url = https://www.x.org/releases/individual/util/util-macros-1.19.2.tar.gz
+md5sum = 5059b328fac086b733ffac6607164c41
 
 [xproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/xproto-7.0.23.tar.bz2
-md5sum = d4d241a4849167e4e694fe73371c328c
+url = https://www.x.org/releases/individual/proto/xproto-7.0.31.tar.gz
+md5sum = 04b925bf9e472c80f9212615cd684f1e
 configure-options =
   --disable-specs
   --without-xmlto
@@ -52,8 +52,8 @@ environment =
 [xextproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/xextproto-7.2.1.tar.bz2
-md5sum = eaac343af094e6b608cf15cfba0f77c5
+url = https://www.x.org/releases/individual/proto/xextproto-7.3.0.tar.gz
+md5sum = 37b700baa8c8ea7964702d948dd13821
 configure-options =
   --disable-specs
   --without-xmlto
@@ -65,8 +65,8 @@ environment =
 [xtrans]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/xtrans-1.2.7.tar.bz2
-md5sum = 84c66908cf003ad8c272b0eecbdbaee3
+url = https://www.x.org/releases/individual/lib/xtrans-1.3.5.tar.gz
+md5sum = 6e4eac1b7c6591da0753052e1eccfb58
 patches =
   ${:_profile_base_location_}/xtrans_tmp_env.patch#${xtrans_tmp_env.patch:md5sum}
 patch-options = -p1
@@ -81,8 +81,8 @@ environment =
 [libXau]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXau-1.0.7.tar.bz2
-md5sum = 2d241521df40d27034413436d1a1465c
+url = https://www.x.org/releases/individual/lib/libXau-1.0.8.tar.gz
+md5sum = a85cd601d82bc79c0daa280917572e20
 configure-options =
   --disable-static
 environment =
@@ -92,8 +92,8 @@ environment =
 [xcbproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://xcb.freedesktop.org/dist/xcb-proto-1.8.tar.bz2
-md5sum = a5de3432cc6e43cc6a27f241dbb991b1
+url = https://xcb.freedesktop.org/dist/xcb-proto-1.13.tar.bz2
+md5sum = abe9aa4886138150bbc04ae4f29b90e3
 environment =
   PATH=${libxml2:location}/bin:%(PATH)s
   PYTHON=${buildout:executable}
@@ -101,14 +101,15 @@ environment =
 [xorg-libpthread-stubs]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://xcb.freedesktop.org/dist/libpthread-stubs-0.3.tar.bz2
-md5sum = e8fa31b42e13f87e8f5a7a2b731db7ee
+url = https://www.x.org/releases/individual/xcb/libpthread-stubs-0.4.tar.gz
+md5sum = 7d2734e604a3e2f6f665c420b835ab62
 
 [libxcb]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://xcb.freedesktop.org/dist/libxcb-1.9.1.tar.bz2
-md5sum = ed632cb0dc31b6fbd7ea5c0f931cf5a4
+url = https://xorg.freedesktop.org/archive/individual/xcb/libxcb-1.13.tar.bz2
+md5sum = c2b6cf928afa16b0047c974e7aaa783f
+
 patches =
   ${:_profile_base_location_}/libxcb_tmp_env.patch#${libxcb_tmp_env.patch:md5sum}
 patch-options = -p1
@@ -146,8 +147,8 @@ configure-options =
 [inputproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/inputproto-2.2.tar.bz2
-md5sum = 13c8aedaf98a92e282b7e456c0a5bed9
+url = https://www.x.org/releases/individual/proto/inputproto-2.3.2.tar.gz
+md5sum = 6450bad6f8d5ebe354b01b734d1fd7ca
 environment =
   PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -155,8 +156,8 @@ environment =
 [kbproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/kbproto-1.0.6.tar.bz2
-md5sum = 677ea8523eec6caca86121ad2dca0b71
+url = https://www.x.org/releases/individual/proto/kbproto-1.0.7.tar.gz
+md5sum = 19acc5f02ae80381e216f443134e0bbb
 environment =
   PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -164,8 +165,8 @@ environment =
 [libX11]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.x.org/pub/individual/lib/libX11-1.6.2.tar.bz2
-md5sum = c35d6ad95b06635a524579e88622fdb5
+url = https://www.x.org/releases/individual/lib/libX11-1.6.6.tar.gz
+md5sum = 3fd4c6b9f2333dbc5d16824baa1cfb67
 pkg_config_depends = ${inputproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig:${xtrans:location}/share/pkgconfig
 configure-options =
   --disable-static
@@ -186,8 +187,8 @@ environment =
 [libXdmcp]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXdmcp-1.1.1.tar.bz2
-md5sum = b94af6cef211cf3ee256f7e81f70fcd9
+url = https://www.x.org/releases/individual/lib/libXdmcp-1.1.2.tar.gz
+md5sum = ab0d6a38f0344a05d698ec7d48cfa5a8
 environment =
   PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -232,15 +233,25 @@ environment =
   PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
 
-[randrproto]
+[presentproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/randrproto-1.3.2.tar.bz2
-md5sum = 597491c0d8055e2a66f11350c4985775
+url = https://www.x.org/releases/individual/proto/presentproto-1.1.tar.gz
+md5sum = dc689e8569eda66b8c404e355f575119
 environment =
   PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
 
+[xorgproto]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://www.x.org/releases/individual/proto/xorgproto-2018.4.tar.gz
+md5sum = 2763268f5bc742e337415bfedf06b845
+pkg_config_depends = ${xorg-util-macros:location}/share/pkgconfig
+environment =
+  PKG_CONFIG_PATH=${:pkg_config_depends}
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+
 [renderproto]
 recipe = slapos.recipe.cmmi
 shared = true
@@ -253,8 +264,8 @@ environment =
 [videoproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/videoproto-2.3.1.tar.bz2
-md5sum = c3b348c6e2031b72b11ae63fc7f805c2
+url = https://www.x.org/releases/individual/proto/videoproto-2.3.3.tar.gz
+md5sum = d984100603ee2420072f27bb491f4b7d
 environment =
   PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -262,8 +273,8 @@ environment =
 [fontsproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/fontsproto-2.1.2.tar.bz2
-md5sum = c5f4f1fb4ba7766eedbc9489e81f3be2
+url = https://www.x.org/releases/individual/proto/fontsproto-2.1.3.tar.gz
+md5sum = 0415f0360e33f3202af67c6c46782251
 configure-options =
   --disable-specs
   --without-xmlto
@@ -310,8 +321,8 @@ configure-options =
 [libfontenc]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libfontenc-1.1.1.tar.bz2
-md5sum = a2a861f142c3b4367f14fc14239fc1f7
+url = https://www.x.org/releases/individual/lib/libfontenc-1.1.3.tar.gz
+md5sum = 0ffa28542aa7d246299b1f7211cdb768
 environment =
   PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -321,8 +332,19 @@ environment =
 [libXfont]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXfont-1.4.5.tar.bz2
-md5sum = 6851da5dae0a6cf5f7c9b9e2b05dd3b4
+url = https://www.x.org/releases/individual/lib/libXfont-1.5.4.tar.gz
+md5sum = 562cc729034de3442d860f1c50508c8b
+environment =
+  PKG_CONFIG_PATH=${fontsproto:location}/lib/pkgconfig:${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libfontenc:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig:${xtrans:location}/share/pkgconfig
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+
+[libXfont2]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://www.x.org/releases/individual/lib/libXfont2-2.0.3.tar.gz
+md5sum = ba7277762e3737cd8dcb6c7fe5113a34
 environment =
   PKG_CONFIG_PATH=${fontsproto:location}/lib/pkgconfig:${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libfontenc:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig:${xtrans:location}/share/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -332,8 +354,8 @@ environment =
 [libxkbfile]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libxkbfile-1.0.8.tar.bz2
-md5sum = 19e6533ae64abba0773816a23f2b9507
+url = https://www.x.org/releases/individual/lib/libxkbfile-1.0.9.tar.gz
+md5sum = 5aab87eba67f37dd910a19be5c1129ee
 environment =
   PKG_CONFIG_PATH=${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -343,8 +365,8 @@ environment =
 [xkeyboard-config]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/xkeyboard-config-2.6.tar.bz2
-md5sum = e7e31da9adb56ba52b5c18226b8cbef3
+url = https://www.x.org/releases/individual/data/xkeyboard-config-2.5.1.tar.gz
+md5sum = 62c6583b4ec5775717e7c8b05732763c
 configure-options =
   --disable-static
   --enable-shared
@@ -357,8 +379,8 @@ environment =
 [xkbcomp]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/xkbcomp-1.2.4.tar.bz2
-md5sum = a0fc1ac3fc4fe479ade09674347c5aa0
+url = https://www.x.org/releases/individual/app/xkbcomp-1.4.2.tar.gz
+md5sum = 84b6bafb660181a8c2572981a7fff54d
 # Hardcoded location for xkeyboard-config, needed during compile time
 xkeyboard-config-location = ${libxkbfile:location}/../xkeyboard-config/share/X11/xkb
 configure-options =
@@ -368,7 +390,7 @@ environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   LDFLAGS=-L${libX11:location}/lib -Wl,-rpath=${libX11:location}/lib -L${libxkbfile:location}/lib -Wl,-rpath=${libxkbfile:location}/lib
 
-[render]
+[renderext]
 recipe = slapos.recipe.cmmi
 shared = true
 url = http://xlibs.freedesktop.org/release/renderext-0.9.tar.bz2
@@ -377,19 +399,19 @@ md5sum = d43c2afc69937655d13c02588c9ff974
 [libXrender]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXrender-0.9.7.tar.bz2
-md5sum = ee62f4c7f0f16ced4da63308963ccad2
+url = https://www.x.org/releases/individual/lib/libXrender-0.9.10.tar.gz
+md5sum = 98a14fc11aee08b4a1769426ab4b23a3
 configure-options =
   --disable-static
 environment =
-  PKG_CONFIG_PATH=${kbproto:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${render:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${kbproto:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${renderext:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
 
 [libXinerama]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXinerama-1.1.2.tar.bz2
-md5sum = cb45d6672c93a608f003b6404f1dd462
+url = https://www.x.org/releases/individual/lib/libXinerama-1.1.4.tar.gz
+md5sum = 416f5afc64b8d064187b3508081dd194
 environment =
   PKG_CONFIG_PATH=${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xineramaproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -399,8 +421,8 @@ configure-options =
 [libICE]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libICE-1.0.8.tar.bz2
-md5sum = 471b5ca9f5562ac0d6eac7a0bf650738
+url = https://www.x.org/releases/individual/lib/libICE-1.0.9.tar.gz
+md5sum = 95812d61df8139c7cacc1325a26d5e37
 configure-options =
   --disable-static
   --without-xmlto
@@ -412,8 +434,8 @@ environment =
 [libSM]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libSM-1.2.1.tar.bz2
-md5sum = 766de9d1e1ecf8bf74cebe2111d8e2bd
+url = https://www.x.org/releases/individual/lib/libSM-1.2.2.tar.gz
+md5sum = 18e5084ed9500b1b47719fd1758f0ec8
 configure-options =
   --disable-static
   --without-xmlto
@@ -426,8 +448,8 @@ environment =
 [libXt]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXt-1.1.3.tar.bz2
-md5sum = a6f137ae100e74ebe3b71eb4a38c40b3
+url = https://www.x.org/releases/individual/lib/libXt-1.1.5.tar.gz
+md5sum = 77d317fbc508dd6adefb59d57a663032
 configure-options =
   --disable-static
 environment =
@@ -437,8 +459,8 @@ environment =
 [dri2proto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/dri2proto-2.6.tar.bz2
-md5sum = 2eb74959684f47c862081099059a11ab
+url = https://www.x.org/releases/individual/proto/dri2proto-2.8.tar.gz
+md5sum = 19ea18f63d8ae8053c9fa84b60365b77
 configure-options =
   --disable-static
 environment =
@@ -449,8 +471,8 @@ environment =
 [pciaccess]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libpciaccess-0.13.1.tar.bz2
-md5sum = 399a419ac6a54f0fc07c69c9bdf452dc
+url = https://www.x.org/releases/individual/lib/libpciaccess-0.14.tar.gz
+md5sum = 344872335233111f44504d3f7cb71bb3
 configure-options =
   --disable-static
 environment =
@@ -459,8 +481,8 @@ environment =
 [makedepend]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://xorg.freedesktop.org/releases/individual/util/makedepend-1.0.3.tar.bz2
-md5sum = 4e6cb97bbecfbb34f3f644a75e513093
+url = https://www.x.org/releases/individual/util/makedepend-1.0.5.tar.gz
+md5sum = efb2d7c7e22840947863efaedc175747
 configure-options =
   --disable-static
 environment =
@@ -470,8 +492,8 @@ environment =
 [glproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://xorg.freedesktop.org/releases/individual/proto/glproto-1.4.14.tar.bz2
-md5sum = f48257daf0017f7a7667e5bf48ca3578
+url = https://www.x.org/releases/individual/proto/glproto-1.4.17.tar.gz
+md5sum = d69554c1b51a83f2c6976a640819911b
 configure-options =
   --disable-static
 environment =
@@ -481,8 +503,8 @@ environment =
 [xfixes]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://xorg.freedesktop.org/releases/individual/lib/libXfixes-4.0.5.tar.bz2
-md5sum = 1b4b8386bd5d1751b2c7177223ad4629
+url = https://www.x.org/releases/individual/lib/libXfixes-5.0.3.tar.gz
+md5sum = fd07d0d77e92b0a72ca1740a72322837
 configure-options =
   --disable-static
 environment =
@@ -492,8 +514,8 @@ environment =
 [xdamage]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXdamage-1.1.3.tar.bz2
-md5sum = 44774e1a065158b52f1a0da5100cebec
+url = https://www.x.org/releases/individual/lib/libXdamage-1.1.4.tar.gz
+md5sum = 95867778da012623815214769007c0d7
 configure-options =
   --disable-static
 environment =
@@ -503,8 +525,8 @@ environment =
 [libxmu]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXmu-1.1.1.tar.bz2
-md5sum = a4efff8de85bd45dd3da124285d10c00
+url = https://www.x.org/releases/individual/lib/libXmu-1.1.2.tar.gz
+md5sum = d5be323b02e6851607205c8e941b4e61
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${xextproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXt:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libICE:location}/lib/pkgconfig:${libSM:location}/lib/pkgconfig
@@ -518,8 +540,8 @@ md5sum = 98482f65ba1e74a08bf5b056a4031ef0
 [libXcomposite]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXcomposite-0.4.3.tar.bz2
-md5sum = a60e0b5c276d0aa9e2d3b982c98f61c8
+url = https://www.x.org/releases/individual/lib/libXcomposite-0.4.4.tar.gz
+md5sum = af860b1554a423735d831e6f29ac1ef5
 configure-options =
   --disable-static
 environment =
@@ -529,19 +551,19 @@ environment =
 [libXcursor]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXcursor-1.1.13.tar.bz2
-md5sum = 52efa81b7f26c8eda13510a2fba98eea
+url = https://www.x.org/releases/individual/lib/libXcursor-1.1.15.tar.gz
+md5sum = 837cd0d40afa6ecdafaf6f7b574a0899
 configure-options =
   --disable-static
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${render:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${renderext:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig
 
 [xwd]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/xwd-1.0.5.tar.bz2
-md5sum = 2113126f9ac9c02bb8547c112c5d037e
+url = https://www.x.org/releases/individual/app/xwd-1.0.7.tar.gz
+md5sum = 3ebd74f7a1980305e5e19ec8ff7aa794
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libxkbfile:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig
@@ -552,14 +574,13 @@ environment =
 # Adds Xvfb functionnality
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/xorg-server-1.12.2.tar.bz2
-md5sum = 791f0323b886abb7954de7f042bb7dc6
+url = https://www.x.org/releases/individual/xserver/xorg-server-1.20.1.tar.gz
+md5sum = f5ba68452b1710306aabc32308c6ac59
 patches =
   ${:_profile_base_location_}/xorg-server_tmp_env.patch#${xorg-server_tmp_env.patch:md5sum}
 patch-options = -p1
 configure-options =
   --enable-xvfb
-  --disable-aiglx
   --disable-composite
   --disable-screensaver
   --disable-glx
@@ -572,7 +593,7 @@ configure-options =
   --with-xkb-path=${xkeyboard-config:location}/share/X11/xkb
   --with-sha1=libgcrypt
 environment =
-  PKG_CONFIG_PATH=${pixman:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${fixesproto:location}/lib/pkgconfig:${damageproto:location}/lib/pkgconfig:${xcmiscproto:location}/lib/pkgconfig:${xtrans:location}/share/pkgconfig:${bigreqsproto:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${inputproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${fontsproto:location}/lib/pkgconfig:${videoproto:location}/lib/pkgconfig:${recordproto:location}/lib/pkgconfig:${resourceproto:location}/lib/pkgconfig:${xineramaproto:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxkbfile:location}/lib/pkgconfig:${libXfont:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libfontenc:location}/lib/pkgconfig:${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${pixman:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${fixesproto:location}/lib/pkgconfig:${damageproto:location}/lib/pkgconfig:${xcmiscproto:location}/lib/pkgconfig:${xtrans:location}/share/pkgconfig:${bigreqsproto:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${xorgproto:pkg_config_depends}:${xorgproto:location}/share/pkgconfig:${renderproto:location}/lib/pkgconfig:${presentproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${inputproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${fontsproto:location}/lib/pkgconfig:${videoproto:location}/lib/pkgconfig:${recordproto:location}/lib/pkgconfig:${resourceproto:location}/lib/pkgconfig:${xineramaproto:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxkbfile:location}/lib/pkgconfig:${libXfont:location}/lib/pkgconfig:${libXfont2:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libfontenc:location}/lib/pkgconfig:${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
   CPPFLAGS=-I${bzip2:location}/include -I${libgcrypt:location}/include -I${libgpg-error:location}/include -I${zlib:location}/include
   PATH=${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${libgcrypt:location}/lib -Wl,-rpath=${libgcrypt:location}/lib -L${libgpg-error:location}/lib -Wl,-rpath=${libgpg-error:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
@@ -580,9 +601,9 @@ environment =
 [libXi]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXi-1.6.1.tar.bz2
-md5sum = 78ee882e1ff3b192cf54070bdb19938e
-pkg_config_depends = ${inputproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
+url = https://www.x.org/releases/individual/lib/libXi-1.7.9.tar.gz
+md5sum = 7f0483d3fa110092b75378b3c926566f
+pkg_config_depends = ${inputproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig
 environment =
   PKG_CONFIG_PATH=${:pkg_config_depends}
   PATH=${pkgconfig:location}/bin:%(PATH)s
@@ -596,8 +617,8 @@ configure-options =
 [libXtst]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.x.org/releases/X11R7.7/src/everything/libXtst-1.2.1.tar.bz2
-md5sum = e8abc5c00c666f551cf26aa53819d592
+url = https://www.x.org/releases/individual/lib/libXtst-1.2.3.tar.gz
+md5sum = 2534e6015a52e0bb7b6f9148ca180028
 configure-options =
   --disable-specs
   --disable-static
@@ -606,7 +627,7 @@ configure-options =
   --without-xsltproc
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libXi:location}/lib/pkgconfig:${recordproto:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libXi:location}/lib/pkgconfig:${libXi:pkg_config_depends}:${recordproto:location}/lib/pkgconfig
 
 [scrnsaverproto]
 recipe = slapos.recipe.cmmi
@@ -620,9 +641,18 @@ environment =
 [libXScrnSaver]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.x.org/releases/X11R7.7/src/everything/libXScrnSaver-1.2.2.tar.bz2
-md5sum = 7a773b16165e39e938650bcc9027c1d5
+url = https://www.x.org/releases/individual/lib/libXScrnSaver-1.2.3.tar.gz
+md5sum = 6ae51eb64351e11cea281f3a331ac461
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${libX11:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${scrnsaverproto:location}/lib/pkgconfig
 
+[libXrandr]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://www.x.org/releases/individual/lib/libXrandr-1.5.1.tar.gz
+md5sum = 59e90a544ee8cf706cf11e3027339f60
+pkg_config_depends = ${libX11:location}/lib/pkgconfig:${xorgproto:pkg_config_depends}:${xorgproto:location}/share/pkgconfig:${libXext:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${xcbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig
+environment =
+  PKG_CONFIG_PATH=${:pkg_config_depends}
+  PATH=${pkgconfig:location}/bin:%(PATH)s

component/xorg/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [libxcb_tmp_env.patch]
 filename = libxcb_tmp_env.patch
-md5sum = 03bc7279f2a37bac7bf426af213123b2
+md5sum = 5a2875e62216d2f41a75e92d1bf0ae67
 
 [xorg-server_tmp_env.patch]
 filename = xorg-server_tmp_env.patch

component/xorg/libxcb_tmp_env.patch

-Store Xorg lock files and sockets in different directories
+From 8469ef2a978e19145a56adb54925cf5936ab2d49 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
+Date: Mon, 17 Sep 2018 16:07:02 +0200
+Subject: [PATCH] SlapOS: Store Xorg lock files and sockets in different
+ directories
 
 see xorg-server_tmp_env.patch
+---
+ src/xcb_util.c | 26 ++++++++++++++++++++++++--
+ 1 file changed, 24 insertions(+), 2 deletions(-)
 
---- libxcb-1.7/src/xcb_util.c.back	2012-04-03 13:30:36.000000000 +0200
-+++ libxcb-1.7/src/xcb_util.c	2012-04-03 14:47:00.000000000 +0200
-@@ -147,11 +147,31 @@
+diff --git a/src/xcb_util.c b/src/xcb_util.c
+index a16270c..d05f8a1 100644
+--- a/src/xcb_util.c
++++ b/src/xcb_util.c
+@@ -223,15 +223,37 @@ static int _xcb_open_unix(char *protocol, const char *file);
  static int _xcb_open_abstract(char *protocol, const char *file, size_t filelen);
  #endif
  
@@ -27,14 +36,23 @@ see xorg-server_tmp_env.patch
 +  }
 +  return strncat(path_list[0], suffix, PATH_MAX);
 +}
++
 +
  static int _xcb_open(const char *host, char *protocol, const int display)
  {
      int fd;
+ #ifdef __hpux
+     static const char unix_base[] = "/usr/spool/sockets/X11/";
++    #error "SlapOS: no patch for hpux"
+ #else
 -    static const char unix_base[] = "/tmp/.X11-unix/X";
--    const char *base = unix_base;
 +    char *base;
 +    base = _xcb_getandappendcompatibleenv("XORG_LOCK_DIR", "/tmp", "/.X11-unix/X");
+ #endif
+-    const char *base = unix_base;
      size_t filelen;
      char *file = NULL;
      int actual_filelen;
+-- 
+2.11.0
+

format-json

 #!/usr/bin/env python
 r"""Command-line tool to format software release JSON for slapos.
 
-Inspired by json.tool from python
+Inspired by json.tool from python, but enforcing 2 spaces and non-sorted keys.
+The files are modified in-place.
 
 Usage::
 
-    format-json infile outfile
+    format-json file1.json [file2.json]
 
 """
 
@@ -16,19 +17,16 @@ import collections
 
 
 def main():
-  if len(sys.argv) != 3:
-    raise SystemExit(sys.argv[0] + " infile outfile")
-
-  with open(sys.argv[1], 'rb') as infile:
+  for f in sys.argv[1:]:
+    with open(f, 'rb') as infile:
       try:
         obj = json.load(infile, object_pairs_hook=collections.OrderedDict)
-    except ValueError, e:
+      except ValueError as e:
         raise SystemExit(e)
 
-  with open(sys.argv[2], 'wb') as outfile:
+    with open(f, 'wb') as outfile:
       json.dump(obj, outfile, sort_keys=False, indent=2, separators=(',', ': '))
       outfile.write('\n')
 
-
 if __name__ == '__main__':
   main()

package-lock.json

+{
+  "requires": true,
+  "lockfileVersion": 1,
+  "dependencies": {
+    "@samverschueren/stream-to-observable": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.0.tgz",
+      "integrity": "sha512-MI4Xx6LHs4Webyvi6EbspgyAb4D2Q2VtnCQ1blOJcoLS6mVa8lNN2rkIy1CVxfTUpoyIbCTkXES1rLXztFD1lg==",
+      "requires": {
+        "any-observable": "^0.3.0"
+      }
+    },
+    "ansi-escapes": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.4.0.tgz",
+      "integrity": "sha1-06ioOzGapneTZisT52HHkRQiMG4="
+    },
+    "ansi-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+      "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="
+    },
+    "ansi-styles": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+      "requires": {
+        "color-convert": "^1.9.0"
+      }
+    },
+    "any-observable": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/any-observable/-/any-observable-0.3.0.tgz",
+      "integrity": "sha512-/FQM1EDkTsf63Ub2C6O7GuYFDsSXUwsaZDurV0np41ocwq0jthUAYCmhBX9f+KwlaCgIuWyr/4WlUQUBfKfZog=="
+    },
+    "argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "requires": {
+        "sprintf-js": "~1.0.2"
+      }
+    },
+    "arr-diff": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
+      "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA="
+    },
+    "arr-flatten": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz",
+      "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg=="
+    },
+    "arr-union": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
+      "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ="
+    },
+    "array-unique": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
+      "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
+    },
+    "assign-symbols": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
+      "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c="
+    },
+    "atob": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
+      "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
+    },
+    "base": {
+      "version": "0.11.2",
+      "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz",
+      "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==",
+      "requires": {
+        "cache-base": "^1.0.1",
+        "class-utils": "^0.3.5",
+        "component-emitter": "^1.2.1",
+        "define-property": "^1.0.0",
+        "isobject": "^3.0.1",
+        "mixin-deep": "^1.2.0",
+        "pascalcase": "^0.1.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+          "requires": {
+            "is-descriptor": "^1.0.0"
+          }
+        },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        }
+      }
+    },
+    "braces": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+      "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
+      "requires": {
+        "arr-flatten": "^1.1.0",
+        "array-unique": "^0.3.2",
+        "extend-shallow": "^2.0.1",
+        "fill-range": "^4.0.0",
+        "isobject": "^3.0.1",
+        "repeat-element": "^1.1.2",
+        "snapdragon": "^0.8.1",
+        "snapdragon-node": "^2.0.1",
+        "split-string": "^3.0.2",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "builtin-modules": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
+      "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8="
+    },
+    "cache-base": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz",
+      "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==",
+      "requires": {
+        "collection-visit": "^1.0.0",
+        "component-emitter": "^1.2.1",
+        "get-value": "^2.0.6",
+        "has-value": "^1.0.0",
+        "isobject": "^3.0.1",
+        "set-value": "^2.0.0",
+        "to-object-path": "^0.3.0",
+        "union-value": "^1.0.0",
+        "unset-value": "^1.0.0"
+      }
+    },
+    "chalk": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+      "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+      "requires": {
+        "ansi-styles": "^3.2.1",
+        "escape-string-regexp": "^1.0.5",
+        "supports-color": "^5.3.0"
+      }
+    },
+    "ci-info": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz",
+      "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A=="
+    },
+    "class-utils": {
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
+      "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==",
+      "requires": {
+        "arr-union": "^3.1.0",
+        "define-property": "^0.2.5",
+        "isobject": "^3.0.0",
+        "static-extend": "^0.1.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        }
+      }
+    },
+    "cli-cursor": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz",
+      "integrity": "sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc=",
+      "requires": {
+        "restore-cursor": "^1.0.1"
+      }
+    },
+    "cli-truncate": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-0.2.1.tgz",
+      "integrity": "sha1-nxXPuwcFAFNpIWxiasfQWrkN1XQ=",
+      "requires": {
+        "slice-ansi": "0.0.4",
+        "string-width": "^1.0.1"
+      }
+    },
+    "code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
+    },
+    "collection-visit": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
+      "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=",
+      "requires": {
+        "map-visit": "^1.0.0",
+        "object-visit": "^1.0.0"
+      }
+    },
+    "color-convert": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+      "requires": {
+        "color-name": "1.1.3"
+      }
+    },
+    "color-name": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+      "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
+    },
+    "commander": {
+      "version": "2.19.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz",
+      "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg=="
+    },
+    "component-emitter": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
+      "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY="
+    },
+    "copy-descriptor": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
+      "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40="
+    },
+    "cosmiconfig": {
+      "version": "5.0.6",
+      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-5.0.6.tgz",
+      "integrity": "sha512-6DWfizHriCrFWURP1/qyhsiFvYdlJzbCzmtFWh744+KyWsJo5+kPzUZZaMRSSItoYc0pxFX7gEO7ZC1/gN/7AQ==",
+      "requires": {
+        "is-directory": "^0.3.1",
+        "js-yaml": "^3.9.0",
+        "parse-json": "^4.0.0"
+      }
+    },
+    "cross-spawn": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
+      "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=",
+      "requires": {
+        "lru-cache": "^4.0.1",
+        "shebang-command": "^1.2.0",
+        "which": "^1.2.9"
+      }
+    },
+    "date-fns": {
+      "version": "1.29.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz",
+      "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw=="
+    },
+    "debug": {
+      "version": "3.2.6",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+      "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+      "requires": {
+        "ms": "^2.1.1"
+      }
+    },
+    "decode-uri-component": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
+      "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU="
+    },
+    "dedent": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz",
+      "integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw="
+    },
+    "define-property": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
+      "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
+      "requires": {
+        "is-descriptor": "^1.0.2",
+        "isobject": "^3.0.1"
+      },
+      "dependencies": {
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        }
+      }
+    },
+    "elegant-spinner": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/elegant-spinner/-/elegant-spinner-1.0.1.tgz",
+      "integrity": "sha1-2wQ1IcldfjA/2PNFvtwzSc+wcp4="
+    },
+    "error-ex": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
+      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
+      "requires": {
+        "is-arrayish": "^0.2.1"
+      }
+    },
+    "escape-string-regexp": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+      "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
+    },
+    "esprima": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="
+    },
+    "execa": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-0.9.0.tgz",
+      "integrity": "sha512-BbUMBiX4hqiHZUA5+JujIjNb6TyAlp2D5KLheMjMluwOuzcnylDL4AxZYLLn1n2AGB49eSWwyKvvEQoRpnAtmA==",
+      "requires": {
+        "cross-spawn": "^5.0.1",
+        "get-stream": "^3.0.0",
+        "is-stream": "^1.1.0",
+        "npm-run-path": "^2.0.0",
+        "p-finally": "^1.0.0",
+        "signal-exit": "^3.0.0",
+        "strip-eof": "^1.0.0"
+      }
+    },
+    "exit-hook": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-1.1.1.tgz",
+      "integrity": "sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g="
+    },
+    "expand-brackets": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
+      "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
+      "requires": {
+        "debug": "^2.3.3",
+        "define-property": "^0.2.5",
+        "extend-shallow": "^2.0.1",
+        "posix-character-classes": "^0.1.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        },
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        }
+      }
+    },
+    "extend-shallow": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
+      "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
+      "requires": {
+        "assign-symbols": "^1.0.0",
+        "is-extendable": "^1.0.1"
+      },
+      "dependencies": {
+        "is-extendable": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
+          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
+          "requires": {
+            "is-plain-object": "^2.0.4"
+          }
+        }
+      }
+    },
+    "extglob": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
+      "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
+      "requires": {
+        "array-unique": "^0.3.2",
+        "define-property": "^1.0.0",
+        "expand-brackets": "^2.1.4",
+        "extend-shallow": "^2.0.1",
+        "fragment-cache": "^0.2.1",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+          "requires": {
+            "is-descriptor": "^1.0.0"
+          }
+        },
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        }
+      }
+    },
+    "figures": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz",
+      "integrity": "sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4=",
+      "requires": {
+        "escape-string-regexp": "^1.0.5",
+        "object-assign": "^4.1.0"
+      }
+    },
+    "fill-range": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
+      "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
+      "requires": {
+        "extend-shallow": "^2.0.1",
+        "is-number": "^3.0.0",
+        "repeat-string": "^1.6.1",
+        "to-regex-range": "^2.1.0"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "find-parent-dir": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/find-parent-dir/-/find-parent-dir-0.3.0.tgz",
+      "integrity": "sha1-M8RLQpqysvBkYpnF+fcY83b/jVQ="
+    },
+    "find-up": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
+      "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
+      "requires": {
+        "locate-path": "^3.0.0"
+      }
+    },
+    "for-in": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
+      "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA="
+    },
+    "fragment-cache": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
+      "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=",
+      "requires": {
+        "map-cache": "^0.2.2"
+      }
+    },
+    "get-own-enumerable-property-symbols": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.0.tgz",
+      "integrity": "sha512-CIJYJC4GGF06TakLg8z4GQKvDsx9EMspVxOYih7LerEL/WosUnFIww45CGfxfeKHqlg3twgUrYRT1O3WQqjGCg=="
+    },
+    "get-stdin": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz",
+      "integrity": "sha512-jp4tHawyV7+fkkSKyvjuLZswblUtz+SQKzSWnBbii16BuZksJlU1wuBYXY75r+duh/llF1ur6oNwi+2ZzjKZ7g=="
+    },
+    "get-stream": {
+      "version": "3.0.0",
+      "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+      "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ="
+    },
+    "get-value": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz",
+      "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg="
+    },
+    "has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+          "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
+        }
+      }
+    },
+    "has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
+    },
+    "has-value": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
+      "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=",
+      "requires": {
+        "get-value": "^2.0.6",
+        "has-values": "^1.0.0",
+        "isobject": "^3.0.0"
+      }
+    },
+    "has-values": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz",
+      "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=",
+      "requires": {
+        "is-number": "^3.0.0",
+        "kind-of": "^4.0.0"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz",
+          "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=",
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
+      }
+    },
+    "hosted-git-info": {
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz",
+      "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w=="
+    },
+    "husky": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-1.1.2.tgz",
+      "integrity": "sha512-9TdkUpBeEOjz0AnFdUN4i3w8kEbOsVs9/WSeJqWLq2OO6bcKQhVW64Zi+pVd/AMRLpN3QTINb6ZXiELczvdmqQ==",
+      "requires": {
+        "cosmiconfig": "^5.0.6",
+        "execa": "^0.9.0",
+        "find-up": "^3.0.0",
+        "get-stdin": "^6.0.0",
+        "is-ci": "^1.2.1",
+        "pkg-dir": "^3.0.0",
+        "please-upgrade-node": "^3.1.1",
+        "read-pkg": "^4.0.1",
+        "run-node": "^1.0.0",
+        "slash": "^2.0.0"
+      }
+    },
+    "indent-string": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz",
+      "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok="
+    },
+    "is-accessor-descriptor": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
+      "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+      "requires": {
+        "kind-of": "^3.0.2"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
+      }
+    },
+    "is-arrayish": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
+      "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0="
+    },
+    "is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
+    "is-builtin-module": {
+      "version": "1.0.0",
+      "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
+      "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=",
+      "requires": {
+        "builtin-modules": "^1.0.0"
+      }
+    },
+    "is-ci": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz",
+      "integrity": "sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==",
+      "requires": {
+        "ci-info": "^1.5.0"
+      }
+    },
+    "is-data-descriptor": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
+      "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+      "requires": {
+        "kind-of": "^3.0.2"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
+      }
+    },
+    "is-descriptor": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
+      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
+      "requires": {
+        "is-accessor-descriptor": "^0.1.6",
+        "is-data-descriptor": "^0.1.4",
+        "kind-of": "^5.0.0"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
+          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
+        }
+      }
+    },
+    "is-directory": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz",
+      "integrity": "sha1-YTObbyR1/Hcv2cnYP1yFddwVSuE="
+    },
+    "is-extendable": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
+      "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik="
+    },
+    "is-extglob": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+      "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI="
+    },
+    "is-fullwidth-code-point": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
+      "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+      "requires": {
+        "number-is-nan": "^1.0.0"
+      }
+    },
+    "is-glob": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
+      "integrity": "sha1-lSHHaEXMJhCoUgPd8ICpWML/q8A=",
+      "requires": {
+        "is-extglob": "^2.1.1"
+      }
+    },
+    "is-number": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
+      "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
+      "requires": {
+        "kind-of": "^3.0.2"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
+      }
+    },
+    "is-obj": {
+      "version": "1.0.1",
+      "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
+      "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8="
+    },
+    "is-observable": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-observable/-/is-observable-1.1.0.tgz",
+      "integrity": "sha512-NqCa4Sa2d+u7BWc6CukaObG3Fh+CU9bvixbpcXYhy2VvYS7vVGIdAgnIS5Ks3A/cqk4rebLJ9s8zBstT2aKnIA==",
+      "requires": {
+        "symbol-observable": "^1.1.0"
+      }
+    },
+    "is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "requires": {
+        "isobject": "^3.0.1"
+      }
+    },
+    "is-promise": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
+      "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o="
+    },
+    "is-regexp": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz",
+      "integrity": "sha1-/S2INUXEa6xaYz57mgnof6LLUGk="
+    },
+    "is-stream": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
+      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
+    },
+    "is-windows": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
+      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA=="
+    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+    },
+    "isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
+    },
+    "isobject": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
+      "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
+    },
+    "jest-get-type": {
+      "version": "22.4.3",
+      "resolved": "http://registry.npmjs.org/jest-get-type/-/jest-get-type-22.4.3.tgz",
+      "integrity": "sha512-/jsz0Y+V29w1chdXVygEKSz2nBoHoYqNShPe+QgxSNjAuP1i8+k4LbQNrfoliKej0P45sivkSCh7yiD6ubHS3w=="
+    },
+    "jest-validate": {
+      "version": "23.6.0",
+      "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-23.6.0.tgz",
+      "integrity": "sha512-OFKapYxe72yz7agrDAWi8v2WL8GIfVqcbKRCLbRG9PAxtzF9b1SEDdTpytNDN12z2fJynoBwpMpvj2R39plI2A==",
+      "requires": {
+        "chalk": "^2.0.1",
+        "jest-get-type": "^22.1.0",
+        "leven": "^2.1.0",
+        "pretty-format": "^23.6.0"
+      }
+    },
+    "js-yaml": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz",
+      "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==",
+      "requires": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      }
+    },
+    "json-parse-better-errors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz",
+      "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw=="
+    },
+    "kind-of": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+      "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
+    },
+    "leven": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz",
+      "integrity": "sha1-wuep93IJTe6dNCAq6KzORoeHVYA="
+    },
+    "lint-staged": {
+      "version": "7.3.0",
+      "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-7.3.0.tgz",
+      "integrity": "sha512-AXk40M9DAiPi7f4tdJggwuKIViUplYtVj1os1MVEteW7qOkU50EOehayCfO9TsoGK24o/EsWb41yrEgfJDDjCw==",
+      "requires": {
+        "chalk": "^2.3.1",
+        "commander": "^2.14.1",
+        "cosmiconfig": "^5.0.2",
+        "debug": "^3.1.0",
+        "dedent": "^0.7.0",
+        "execa": "^0.9.0",
+        "find-parent-dir": "^0.3.0",
+        "is-glob": "^4.0.0",
+        "is-windows": "^1.0.2",
+        "jest-validate": "^23.5.0",
+        "listr": "^0.14.1",
+        "lodash": "^4.17.5",
+        "log-symbols": "^2.2.0",
+        "micromatch": "^3.1.8",
+        "npm-which": "^3.0.1",
+        "p-map": "^1.1.1",
+        "path-is-inside": "^1.0.2",
+        "pify": "^3.0.0",
+        "please-upgrade-node": "^3.0.2",
+        "staged-git-files": "1.1.1",
+        "string-argv": "^0.0.2",
+        "stringify-object": "^3.2.2"
+      }
+    },
+    "listr": {
+      "version": "0.14.2",
+      "resolved": "https://registry.npmjs.org/listr/-/listr-0.14.2.tgz",
+      "integrity": "sha512-vmaNJ1KlGuGWShHI35X/F8r9xxS0VTHh9GejVXwSN20fG5xpq3Jh4bJbnumoT6q5EDM/8/YP1z3YMtQbFmhuXw==",
+      "requires": {
+        "@samverschueren/stream-to-observable": "^0.3.0",
+        "is-observable": "^1.1.0",
+        "is-promise": "^2.1.0",
+        "is-stream": "^1.1.0",
+        "listr-silent-renderer": "^1.1.1",
+        "listr-update-renderer": "^0.4.0",
+        "listr-verbose-renderer": "^0.4.0",
+        "p-map": "^1.1.1",
+        "rxjs": "^6.1.0"
+      }
+    },
+    "listr-silent-renderer": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz",
+      "integrity": "sha1-kktaN1cVN3C/Go4/v3S4u/P5JC4="
+    },
+    "listr-update-renderer": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/listr-update-renderer/-/listr-update-renderer-0.4.0.tgz",
+      "integrity": "sha1-NE2YDaLKLosUW6MFkI8yrj9MyKc=",
+      "requires": {
+        "chalk": "^1.1.3",
+        "cli-truncate": "^0.2.1",
+        "elegant-spinner": "^1.0.1",
+        "figures": "^1.7.0",
+        "indent-string": "^3.0.0",
+        "log-symbols": "^1.0.2",
+        "log-update": "^1.0.2",
+        "strip-ansi": "^3.0.1"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "2.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+          "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
+        },
+        "chalk": {
+          "version": "1.1.3",
+          "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+          "requires": {
+            "ansi-styles": "^2.2.1",
+            "escape-string-regexp": "^1.0.2",
+            "has-ansi": "^2.0.0",
+            "strip-ansi": "^3.0.0",
+            "supports-color": "^2.0.0"
+          }
+        },
+        "log-symbols": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-1.0.2.tgz",
+          "integrity": "sha1-N2/3tY6jCGoPCfrMdGF+ylAeGhg=",
+          "requires": {
+            "chalk": "^1.0.0"
+          }
+        },
+        "supports-color": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+          "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
+        }
+      }
+    },
+    "listr-verbose-renderer": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/listr-verbose-renderer/-/listr-verbose-renderer-0.4.1.tgz",
+      "integrity": "sha1-ggb0z21S3cWCfl/RSYng6WWTOjU=",
+      "requires": {
+        "chalk": "^1.1.3",
+        "cli-cursor": "^1.0.2",
+        "date-fns": "^1.27.2",
+        "figures": "^1.7.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "2.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+          "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
+        },
+        "chalk": {
+          "version": "1.1.3",
+          "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+          "requires": {
+            "ansi-styles": "^2.2.1",
+            "escape-string-regexp": "^1.0.2",
+            "has-ansi": "^2.0.0",
+            "strip-ansi": "^3.0.0",
+            "supports-color": "^2.0.0"
+          }
+        },
+        "supports-color": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+          "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
+        }
+      }
+    },
+    "locate-path": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
+      "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+      "requires": {
+        "p-locate": "^3.0.0",
+        "path-exists": "^3.0.0"
+      }
+    },
+    "lodash": {
+      "version": "4.17.11",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
+      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
+    },
+    "log-symbols": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
+      "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==",
+      "requires": {
+        "chalk": "^2.0.1"
+      }
+    },
+    "log-update": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/log-update/-/log-update-1.0.2.tgz",
+      "integrity": "sha1-GZKfZMQJPS0ucHWh2tivWcKWuNE=",
+      "requires": {
+        "ansi-escapes": "^1.0.0",
+        "cli-cursor": "^1.0.2"
+      }
+    },
+    "lru-cache": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz",
+      "integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==",
+      "requires": {
+        "pseudomap": "^1.0.2",
+        "yallist": "^2.1.2"
+      }
+    },
+    "map-cache": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz",
+      "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8="
+    },
+    "map-visit": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz",
+      "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=",
+      "requires": {
+        "object-visit": "^1.0.0"
+      }
+    },
+    "micromatch": {
+      "version": "3.1.10",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
+      "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==",
+      "requires": {
+        "arr-diff": "^4.0.0",
+        "array-unique": "^0.3.2",
+        "braces": "^2.3.1",
+        "define-property": "^2.0.2",
+        "extend-shallow": "^3.0.2",
+        "extglob": "^2.0.4",
+        "fragment-cache": "^0.2.1",
+        "kind-of": "^6.0.2",
+        "nanomatch": "^1.2.9",
+        "object.pick": "^1.3.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.2"
+      }
+    },
+    "mixin-deep": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
+      "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
+      "requires": {
+        "for-in": "^1.0.2",
+        "is-extendable": "^1.0.1"
+      },
+      "dependencies": {
+        "is-extendable": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
+          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
+          "requires": {
+            "is-plain-object": "^2.0.4"
+          }
+        }
+      }
+    },
+    "ms": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+      "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
+    },
+    "nanomatch": {
+      "version": "1.2.13",
+      "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
+      "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==",
+      "requires": {
+        "arr-diff": "^4.0.0",
+        "array-unique": "^0.3.2",
+        "define-property": "^2.0.2",
+        "extend-shallow": "^3.0.2",
+        "fragment-cache": "^0.2.1",
+        "is-windows": "^1.0.2",
+        "kind-of": "^6.0.2",
+        "object.pick": "^1.3.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      }
+    },
+    "normalize-package-data": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
+      "integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==",
+      "requires": {
+        "hosted-git-info": "^2.1.4",
+        "is-builtin-module": "^1.0.0",
+        "semver": "2 || 3 || 4 || 5",
+        "validate-npm-package-license": "^3.0.1"
+      }
+    },
+    "npm-path": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/npm-path/-/npm-path-2.0.4.tgz",
+      "integrity": "sha512-IFsj0R9C7ZdR5cP+ET342q77uSRdtWOlWpih5eC+lu29tIDbNEgDbzgVJ5UFvYHWhxDZ5TFkJafFioO0pPQjCw==",
+      "requires": {
+        "which": "^1.2.10"
+      }
+    },
+    "npm-run-path": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
+      "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=",
+      "requires": {
+        "path-key": "^2.0.0"
+      }
+    },
+    "npm-which": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/npm-which/-/npm-which-3.0.1.tgz",
+      "integrity": "sha1-kiXybsOihcIJyuZ8OxGmtKtxQKo=",
+      "requires": {
+        "commander": "^2.9.0",
+        "npm-path": "^2.0.2",
+        "which": "^1.2.10"
+      }
+    },
+    "number-is-nan": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
+      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
+    },
+    "object-assign": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
+    },
+    "object-copy": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
+      "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=",
+      "requires": {
+        "copy-descriptor": "^0.1.0",
+        "define-property": "^0.2.5",
+        "kind-of": "^3.0.3"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        },
+        "kind-of": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
+      }
+    },
+    "object-visit": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
+      "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=",
+      "requires": {
+        "isobject": "^3.0.0"
+      }
+    },
+    "object.pick": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz",
+      "integrity": "sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=",
+      "requires": {
+        "isobject": "^3.0.1"
+      }
+    },
+    "onetime": {
+      "version": "1.1.0",
+      "resolved": "http://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz",
+      "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k="
+    },
+    "p-finally": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
+      "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
+    },
+    "p-limit": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz",
+      "integrity": "sha512-fl5s52lI5ahKCernzzIyAP0QAZbGIovtVHGwpcu1Jr/EpzLVDI2myISHwGqK7m8uQFugVWSrbxH7XnhGtvEc+A==",
+      "requires": {
+        "p-try": "^2.0.0"
+      }
+    },
+    "p-locate": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
+      "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+      "requires": {
+        "p-limit": "^2.0.0"
+      }
+    },
+    "p-map": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/p-map/-/p-map-1.2.0.tgz",
+      "integrity": "sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA=="
+    },
+    "p-try": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz",
+      "integrity": "sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ=="
+    },
+    "parse-json": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+      "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
+      "requires": {
+        "error-ex": "^1.3.1",
+        "json-parse-better-errors": "^1.0.1"
+      }
+    },
+    "pascalcase": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz",
+      "integrity": "sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ="
+    },
+    "path-exists": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+      "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU="
+    },
+    "path-is-inside": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz",
+      "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM="
+    },
+    "path-key": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
+      "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A="
+    },
+    "pify": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
+      "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
+    },
+    "pkg-dir": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz",
+      "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==",
+      "requires": {
+        "find-up": "^3.0.0"
+      }
+    },
+    "please-upgrade-node": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/please-upgrade-node/-/please-upgrade-node-3.1.1.tgz",
+      "integrity": "sha512-KY1uHnQ2NlQHqIJQpnh/i54rKkuxCEBx+voJIS/Mvb+L2iYd2NMotwduhKTMjfC1uKoX3VXOxLjIYG66dfJTVQ==",
+      "requires": {
+        "semver-compare": "^1.0.0"
+      }
+    },
+    "posix-character-classes": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz",
+      "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs="
+    },
+    "pretty-format": {
+      "version": "23.6.0",
+      "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-23.6.0.tgz",
+      "integrity": "sha512-zf9NV1NSlDLDjycnwm6hpFATCGl/K1lt0R/GdkAK2O5LN/rwJoB+Mh93gGJjut4YbmecbfgLWVGSTCr0Ewvvbw==",
+      "requires": {
+        "ansi-regex": "^3.0.0",
+        "ansi-styles": "^3.2.0"
+      }
+    },
+    "pseudomap": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
+      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
+    },
+    "read-pkg": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-4.0.1.tgz",
+      "integrity": "sha1-ljYlN48+HE1IyFhytabsfV0JMjc=",
+      "requires": {
+        "normalize-package-data": "^2.3.2",
+        "parse-json": "^4.0.0",
+        "pify": "^3.0.0"
+      }
+    },
+    "regex-not": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
+      "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==",
+      "requires": {
+        "extend-shallow": "^3.0.2",
+        "safe-regex": "^1.1.0"
+      }
+    },
+    "repeat-element": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.3.tgz",
+      "integrity": "sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g=="
+    },
+    "repeat-string": {
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
+      "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc="
+    },
+    "resolve-url": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz",
+      "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo="
+    },
+    "restore-cursor": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz",
+      "integrity": "sha1-NGYfRohjJ/7SmRR5FSJS35LapUE=",
+      "requires": {
+        "exit-hook": "^1.0.0",
+        "onetime": "^1.0.0"
+      }
+    },
+    "ret": {
+      "version": "0.1.15",
+      "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz",
+      "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg=="
+    },
+    "run-node": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/run-node/-/run-node-1.0.0.tgz",
+      "integrity": "sha512-kc120TBlQ3mih1LSzdAJXo4xn/GWS2ec0l3S+syHDXP9uRr0JAT8Qd3mdMuyjqCzeZktgP3try92cEgf9Nks8A=="
+    },
+    "rxjs": {
+      "version": "6.3.3",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.3.3.tgz",
+      "integrity": "sha512-JTWmoY9tWCs7zvIk/CvRjhjGaOd+OVBM987mxFo+OW66cGpdKjZcpmc74ES1sB//7Kl/PAe8+wEakuhG4pcgOw==",
+      "requires": {
+        "tslib": "^1.9.0"
+      }
+    },
+    "safe-regex": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz",
+      "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=",
+      "requires": {
+        "ret": "~0.1.10"
+      }
+    },
+    "semver": {
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
+      "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg=="
+    },
+    "semver-compare": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
+      "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w="
+    },
+    "set-value": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
+      "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
+      "requires": {
+        "extend-shallow": "^2.0.1",
+        "is-extendable": "^0.1.1",
+        "is-plain-object": "^2.0.3",
+        "split-string": "^3.0.1"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "shebang-command": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
+      "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
+      "requires": {
+        "shebang-regex": "^1.0.0"
+      }
+    },
+    "shebang-regex": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
+      "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
+    },
+    "signal-exit": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
+      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
+    },
+    "slash": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
+      "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A=="
+    },
+    "slice-ansi": {
+      "version": "0.0.4",
+      "resolved": "http://registry.npmjs.org/slice-ansi/-/slice-ansi-0.0.4.tgz",
+      "integrity": "sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU="
+    },
+    "snapdragon": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz",
+      "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==",
+      "requires": {
+        "base": "^0.11.1",
+        "debug": "^2.2.0",
+        "define-property": "^0.2.5",
+        "extend-shallow": "^2.0.1",
+        "map-cache": "^0.2.2",
+        "source-map": "^0.5.6",
+        "source-map-resolve": "^0.5.0",
+        "use": "^3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        },
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        }
+      }
+    },
+    "snapdragon-node": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz",
+      "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==",
+      "requires": {
+        "define-property": "^1.0.0",
+        "isobject": "^3.0.0",
+        "snapdragon-util": "^3.0.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+          "requires": {
+            "is-descriptor": "^1.0.0"
+          }
+        },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        }
+      }
+    },
+    "snapdragon-util": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz",
+      "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==",
+      "requires": {
+        "kind-of": "^3.2.0"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
+      }
+    },
+    "source-map": {
+      "version": "0.5.7",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
+      "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w="
+    },
+    "source-map-resolve": {
+      "version": "0.5.2",
+      "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz",
+      "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==",
+      "requires": {
+        "atob": "^2.1.1",
+        "decode-uri-component": "^0.2.0",
+        "resolve-url": "^0.2.1",
+        "source-map-url": "^0.4.0",
+        "urix": "^0.1.0"
+      }
+    },
+    "source-map-url": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz",
+      "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM="
+    },
+    "spdx-correct": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.0.2.tgz",
+      "integrity": "sha512-q9hedtzyXHr5S0A1vEPoK/7l8NpfkFYTq6iCY+Pno2ZbdZR6WexZFtqeVGkGxW3TEJMN914Z55EnAGMmenlIQQ==",
+      "requires": {
+        "spdx-expression-parse": "^3.0.0",
+        "spdx-license-ids": "^3.0.0"
+      }
+    },
+    "spdx-exceptions": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz",
+      "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA=="
+    },
+    "spdx-expression-parse": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz",
+      "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==",
+      "requires": {
+        "spdx-exceptions": "^2.1.0",
+        "spdx-license-ids": "^3.0.0"
+      }
+    },
+    "spdx-license-ids": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.1.tgz",
+      "integrity": "sha512-TfOfPcYGBB5sDuPn3deByxPhmfegAhpDYKSOXZQN81Oyrrif8ZCodOLzK3AesELnCx03kikhyDwh0pfvvQvF8w=="
+    },
+    "split-string": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz",
+      "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==",
+      "requires": {
+        "extend-shallow": "^3.0.0"
+      }
+    },
+    "sprintf-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+      "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw="
+    },
+    "staged-git-files": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/staged-git-files/-/staged-git-files-1.1.1.tgz",
+      "integrity": "sha512-H89UNKr1rQJvI1c/PIR3kiAMBV23yvR7LItZiV74HWZwzt7f3YHuujJ9nJZlt58WlFox7XQsOahexwk7nTe69A=="
+    },
+    "static-extend": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
+      "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=",
+      "requires": {
+        "define-property": "^0.2.5",
+        "object-copy": "^0.1.0"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        }
+      }
+    },
+    "string-argv": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.0.2.tgz",
+      "integrity": "sha1-2sMECGkMIfPDYwo/86BYd73L1zY="
+    },
+    "string-width": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+      "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
+      "requires": {
+        "code-point-at": "^1.0.0",
+        "is-fullwidth-code-point": "^1.0.0",
+        "strip-ansi": "^3.0.0"
+      }
+    },
+    "stringify-object": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz",
+      "integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==",
+      "requires": {
+        "get-own-enumerable-property-symbols": "^3.0.0",
+        "is-obj": "^1.0.1",
+        "is-regexp": "^1.0.0"
+      }
+    },
+    "strip-ansi": {
+      "version": "3.0.1",
+      "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+      "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+          "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
+        }
+      }
+    },
+    "strip-eof": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
+      "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="
+    },
+    "supports-color": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+      "requires": {
+        "has-flag": "^3.0.0"
+      }
+    },
+    "symbol-observable": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz",
+      "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ=="
+    },
+    "to-object-path": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz",
+      "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=",
+      "requires": {
+        "kind-of": "^3.0.2"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
+      }
+    },
+    "to-regex": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz",
+      "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==",
+      "requires": {
+        "define-property": "^2.0.2",
+        "extend-shallow": "^3.0.2",
+        "regex-not": "^1.0.2",
+        "safe-regex": "^1.1.0"
+      }
+    },
+    "to-regex-range": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz",
+      "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=",
+      "requires": {
+        "is-number": "^3.0.0",
+        "repeat-string": "^1.6.1"
+      }
+    },
+    "tslib": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz",
+      "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ=="
+    },
+    "union-value": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
+      "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
+      "requires": {
+        "arr-union": "^3.1.0",
+        "get-value": "^2.0.6",
+        "is-extendable": "^0.1.1",
+        "set-value": "^0.4.3"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        },
+        "set-value": {
+          "version": "0.4.3",
+          "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
+          "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
+          "requires": {
+            "extend-shallow": "^2.0.1",
+            "is-extendable": "^0.1.1",
+            "is-plain-object": "^2.0.1",
+            "to-object-path": "^0.3.0"
+          }
+        }
+      }
+    },
+    "unset-value": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz",
+      "integrity": "sha1-g3aHP30jNRef+x5vw6jtDfyKtVk=",
+      "requires": {
+        "has-value": "^0.3.1",
+        "isobject": "^3.0.0"
+      },
+      "dependencies": {
+        "has-value": {
+          "version": "0.3.1",
+          "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz",
+          "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=",
+          "requires": {
+            "get-value": "^2.0.3",
+            "has-values": "^0.1.4",
+            "isobject": "^2.0.0"
+          },
+          "dependencies": {
+            "isobject": {
+              "version": "2.1.0",
+              "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz",
+              "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=",
+              "requires": {
+                "isarray": "1.0.0"
+              }
+            }
+          }
+        },
+        "has-values": {
+          "version": "0.1.4",
+          "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz",
+          "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E="
+        }
+      }
+    },
+    "urix": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz",
+      "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI="
+    },
+    "use": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz",
+      "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ=="
+    },
+    "validate-npm-package-license": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz",
+      "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==",
+      "requires": {
+        "spdx-correct": "^3.0.0",
+        "spdx-expression-parse": "^3.0.0"
+      }
+    },
+    "which": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+      "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+      "requires": {
+        "isexe": "^2.0.0"
+      }
+    },
+    "yallist": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+      "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
+    }
+  }
+}

package.json

+{
+  "dependencies": {
+    "husky": "^1.1.2",
+    "lint-staged": "^7.3.0"
+  },
+  "lint-staged": {
+    "*.json": [
+      "python2 ./format-json",
+      "git add"
+    ],
+    "{components,software,stack}/**": [
+      "python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"python2\", \"./update-hash\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'",
+      "python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"git\", \"add\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'"
+    ]
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged"
+    }
+  }
+}

slapos/recipe/README.generic_cloudooo.rst

@@ -4,12 +4,12 @@ generic_cloudooo
 The generic_cloudooo recipe helps you to deploy cloudooo services with their configuration files.
 
 
-How to use?
------------
+How to use generic_cloudooo?
+----------------------------
 
 Here is an example of a section to add in your software.cfg :
 
-.. code-block::
+.. code-block:: ini
 
   [cloudooo-configuration]
   recipe = slapos.cookbook:generic_cloudooo
@@ -42,7 +42,7 @@ Where :
   order to make the global mimetype at the bottom of the list.
   (i.e. `* * ooo` > `text/* * ooo`)
 
-    .. code-block::
+    .. code-block:: ini
 
         mimetype_entry_addition =
           <input_format> <output_format> <handler>
@@ -57,7 +57,7 @@ Where :
 Default mimetype registry
 -------------------------
 
-.. code-block::
+.. code-block:: ini
 
   application/vnd.oasis.opendocument* * ooo
   application/vnd.sun.xml* * ooo

slapos/recipe/README.generic_varnish.rst

@@ -4,12 +4,12 @@ generic_varnish
 This recipe creates a varnish instance dedicated for ERP5 with a web checker[1]
 set up.
 
-How to Use
-==========
+How to Use generic_varnish ?
+============================
 
-On slap console, you can instanciate varnish like this:
+On slap console, you can instanciate varnish like this::
   
-instance = request(
+  instance = request(
     software_type='varnish',
     partition_parameter_kw={
        'backend-url':'https://[your_backend_address]:your_backend_port',
@@ -17,7 +17,7 @@ instance = request(
        'web-checker-mail-address':'web-checker-result@example.com',
        'web-checker-smtp-host':'mail.example.com',
     }
-)
+  )
 
 backend-url is the backend url that varnish will cache.
 
@@ -33,11 +33,6 @@ result.
 [Note]
 When web-checker-* parameters are not given, web_checker will be disabled.
 
-TODO
-====
-
-We need to merge this and apache_frontend recipe.
-
 References
 ==========
 

slapos/recipe/README.kvm.rst

 kvm
 ===
 
-Introduction
-------------
+kvm: Introduction
+-----------------
 
 The erp5.recipe.kvm aims to integrate KVM setups and buildout. This recipe is 
 able to download one remote image and setup a KVM environment to use it. 

slapos/recipe/README.kvm_frontend.rst

 kvm_frontend
 =============
 
-Introduction
-------------
+kvm_frontend: Introduction
+---------------------------
 
 The ``slapos.recipe.kvm_frontend`` aims to provide proxy server to KVM instances.
 
@@ -13,44 +13,40 @@ It works following the master/slave instances system. A master instance is
 created, containing all what is needed to run the proxy. Slave instances
 are later created, adding one line in the master instance's proxy configuration
 that specify the IP/port to proxy to the KVM.
-
 The slave instance (kvm) is then accessible from
 http://[masterinstanceIPorhostname]/[randomgeneratednumber]
 
 
 Instance parameters
-------------
+-------------------
 
 Incoming master instance parameters
-+++++++
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``port``                - Port of server, optional, defaults to 4443.
-``domain``              - domain name to use, optional, default to
-                          "host.vifib.net".
-``redirect_plain_http`` - if value is one of ['y', 'yes', '1', 'true'], instance
-                          will try to create a simple http server on port 80
-                          redirecting to the proxy. Optional.
+ * ``port``                - Port of server, optional, defaults to 4443.
+ * ``domain``              - domain name to use, optional, default to ``host.vifib.net``.
+ * ``redirect_plain_http`` - if value is one of ``['y', 'yes', '1', 'true']``,
+   will try to create a simple http server on port 80 redirecting to the proxy. Optional.
 
 Incoming slave instance parameters
-+++++++
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``host``    - KVM instance IP or hostname. Mandatory.
-``port``    - KVM instance port, Mandatory.
-``https``   - if value is one of ['n', 'no', '0', 'false'], will try to connect
-              to target in plain http. Optional.
+ * ``host``    - KVM instance IP or hostname. Mandatory.
+ * ``port``    - KVM instance port, Mandatory.
+ * ``https``   - if value is one of ``['n', 'no', '0', 'false']``, will try to connect to target in plain http. Optional.
 
 Connection parameters
--------------
+---------------------
 
 Outgoing master connection parameters
-+++++++
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``domain_ipv6_address``  - Proxy IP
-``site_url``             - Proxy URL
+ * ``domain_ipv6_address``  - Proxy IP
+ * ``site_url``             - Proxy URL
 
-Outgoing slave connection parameters are :
-+++++++
+Outgoing slave connection parameters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``site_url``             - URL of instance
-``domain_name``          - Domain name of proxy
-``port``                 - Port of proxy
+ * ``site_url``             - URL of instance
+ * ``domain_name``          - Domain name of proxy
+ * ``port``                 - Port of proxy

slapos/recipe/README.lamp.rst

@@ -6,12 +6,12 @@ able to setup mariadb, apache and apache-php for your php application, and is al
 configuring your software during installation to ensure a full compatibility.
 
 
-How to use?
------------
+How to use lamp ?
+-----------------
 
 just add this part in your software.cfg to use the lamp.simple module
 
-.. code-block::
+.. code-block:: ini
 
   [instance-recipe]
   egg = slapos.cookbook
@@ -19,7 +19,7 @@ just add this part in your software.cfg to use the lamp.simple module
 
 you also need to extend lamp.cfg
 
-.. code-block::
+.. code-block:: ini
 
   extends =
     https://lab.nexedi.com/nexedi/slapos/raw/slapos-0.50/stack/lamp.cfg
@@ -35,13 +35,13 @@ in this case you need to write a python script and lamp recipe must run it when
 
 
 
-How to use?
------------
+How to use lamp.runner ?
+------------------------
 
 this part of lamp recipe work with slapos.toolbox, Therefore you must add it to your recipe.
 in software.cfg, replace instance-recipe-egg part by
 
-.. code-block::
+.. code-block:: ini
 
   [instance-recipe-egg]
   recipe = zc.recipe.egg
@@ -52,7 +52,7 @@ in software.cfg, replace instance-recipe-egg part by
 
 and add into your instance.cfg
 
-.. code-block::
+.. code-block:: ini
 
   lampconfigure_directory = ${buildout:bin-directory}/lampconfigure
 
@@ -71,10 +71,10 @@ you can also use database to check condition. add ::
   table_name = name_of_table
   constraint = sql_where_condition
 
-name_of_table is the full or partial name(in some cases we can not know the prefix used to create tables) of table
-into mariadb databse for example table_name = admin. if you use
-name_of_table = **, the action will begin when database is ready. 
-constraint is the sql_condition to use when search entry into name_of_table for example constraint = `admin_id`=1
+``name_of_table`` is the full or partial name(in some cases we can not know the prefix used to create tables) of table
+into mariadb databse for example ``table_name = admin``. if you use
+``name_of_table = **``, the action will begin when database is ready.
+constraint is the sql_condition to use when search entry into name_of_table for example constraint = ``admin_id=1``
 
 you can't use file_token and table_name at the same time, otherwise file_token will be used in priority. Beware of conditions that will never be satisfied.
 
@@ -115,7 +115,7 @@ mode = mode_to_apply (ex= 0644)
 use script = ${configure-script:location}/${configure-script:filename} into instance.cfg, add part configure-script
 into software.cfg
 
-.. code-block::
+.. code-block:: ini
 
   parts = configure-script
 

slapos/recipe/pbs.py

@@ -179,26 +179,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
             $RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force $BACKUP_DIR
         fi
 
-        SUCCEEDED=true
-
-        if [ -e %(backup_signature)s ]; then
-          cd $BACKUP_DIR
-          find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -0 sha256sum  | LC_ALL=C sort -k 66 > ../proof.signature
-          cmp backup.signature ../proof.signature || SUCCEEDED=false
-          diff -ruw backup.signature ../proof.signature > ../backup.diff
-          # XXX If there is a difference on the backup, we should publish the
-          # failure and ask the equeue, re-run this script again,
-          # instead do a push it to the clone.
-        fi
-
-        $SUCCEEDED || find $BACKUP_DIR -name rdiff-backup.tmp.* -exec rm -rf {} \;
-
         """)
 
     template_dict = {
       'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
       'rdiff_backup_data': shlex.quote(os.path.join(local_dir, 'rdiff-backup-data')),
-      'backup_signature': shlex.quote(os.path.join(local_dir, 'backup.signature')),
       'remote_schema': shlex.quote(remote_schema),
       'remote_dir': shlex.quote(remote_dir),
       'local_dir': shlex.quote(local_dir),

software/agent/software.cfg

@@ -51,29 +51,5 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.81
 smmap = 0.9.0
 
-# Required by:
-# slapos.toolbox==0.71
-GitPython = 2.0.8
-
-# Required by:
-# slapos.toolbox==0.71
-atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.71
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.71
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.71
-paramiko = 2.0.1
-
-# Required by:
-# slapos.toolbox==0.71
-rpdb = 0.1.5

software/apache-frontend/software.cfg

@@ -11,49 +11,9 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.81
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3
 
 websockify = 0.8.0
 
-# Required by:
-# slapos.toolbox==0.71
-dnspython = 1.15.0
-
-# Required by:
-# slapos.toolbox==0.71
-erp5.util = 0.4.51
-
-# Required by:
-# slapos.toolbox==0.71
-passlib = 1.6.5
-
-# Required by:
-# slapos.toolbox==0.71
-GitPython = 2.0.8
-
-# Required by:
-# slapos.toolbox==0.71
-atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.71
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.71
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.71
-paramiko = 2.0.1
-
-# Required by:
-# slapos.toolbox==0.71
-pycurl = 7.43.0
-
-# Required by:
-# slapos.toolbox==0.71
-rpdb = 0.1.5

software/backupserver/software.cfg

@@ -121,9 +121,9 @@ eggs =
   ${rdiff-backup-build-1.3.4:egg}
 
 [versions]
-# 1.3.4nxd5 is invalid version string, thus pached version string is not '1.3.4nxd5+SlapOSPatched001'
-# but '1.3.4nxd5-SlapOSPatched001'.
-rdiff-backup = 1.3.4nxd5-SlapOSPatched001
+# 1.3.4nxdX is invalid version string, thus pached version string is not '1.3.4nxdX+SlapOSPatched001'
+# but '1.3.4nxdX-SlapOSPatched001'.
+rdiff-backup = 1.3.4nxd6-SlapOSPatched001
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.4.2

software/caddy-frontend/TODO.rst

 Generally things to be done with ``caddy-frontend``:
 
+ * return warning on not implemented keys (from ``apache-frontend`` perspective) in master and slave request
  * tests: add assertion with results of promises in etc/promise for each partition
  * check the whole frontend slave snippet with ``caddy -validate`` during buildout run, and reject if does not pass validation
  * ``apache-ca-certificate`` shall be merged with ``apache-certificate``
@@ -11,7 +12,6 @@ Generally things to be done with ``caddy-frontend``:
  * ``type:eventsource``:
 
    * **Jérome Perrin**: *For event source, if I understand https://github.com/mholt/caddy/issues/1355 correctly, we could use caddy as a proxy in front of nginx-push-stream . If we have a "central shared" caddy instance, can it handle keeping connections opens for many clients ?*
- * ``ssl_ca_crt``
  * ``ssl_proxy_ca_crt`` for ``ssl_proxy_verify``, this is related to bug `#1550 <https://github.com/mholt/caddy/issues/1550>`_, proposed solution `just adding your CA to the system's trust store`
  * ``check-error-on-caddy-log`` like ``check-error-on-apache-log``
  * cover test suite like resilient tests for KVM and prove it works the same way as Caddy

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = bce721468b4c16294404cac8b88356c0
+md5sum = d43a1631bcd0f4307507268a06f0fac2
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,15 +22,15 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 750e2b1c922bf14511a3bc8a42468b1b
+md5sum = ab1795f92e32655d05c662c965d2b1f5
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 1576859772052bcb85ff2b5a7b786410
+md5sum = 44d50bf8391b5a73b2ab72923efe6437
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = ed1c086f0548a908661b294e845dc008
+md5sum = 69992ad1dffe1c23237f3ef97193c95c
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -46,7 +46,7 @@ md5sum = 7c987ad75fcce6f5b925c7696ff41971
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = ed1c086f0548a908661b294e845dc008
+md5sum = 69992ad1dffe1c23237f3ef97193c95c
 
 [caddy-backend-url-validator]
 filename = templates/caddy-backend-url-validator.in
@@ -62,7 +62,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = 9e00b6d981b9f93a486ef06a47345ebd
+md5sum = 73bc8abae6aadc3ce55662c3d821b363
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in

software/caddy-frontend/instance-apache-frontend.cfg.in

+{%- if slap_software_type == software_type -%}
 {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
 [buildout]
 extends =
@@ -11,7 +12,9 @@ parts =
   cron
   cron-entry-logrotate
   ca-frontend
+  ca-frontend-service
   certificate-authority
+  certificate-authority-service
   logrotate-entry-caddy
   logrotate-entry-nginx
   caddy-frontend
@@ -240,6 +243,7 @@ wrapper-path = ${directory:service}/frontend_caddy
 wait-for-files =
 	       ${ca-frontend:cert-file}
 	       ${ca-frontend:key-file}
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 [not-found-html]
 recipe = slapos.cookbook:symbolic.link
@@ -281,12 +285,18 @@ recipe = slapos.cookbook:certificate_authority
 openssl-binary = {{ parameter_dict['openssl'] }}/bin/openssl
 ca-dir = ${directory:ca-dir}
 requests-directory = ${cadirectory:requests}
-wrapper = ${directory:service}/certificate_authority
+wrapper = ${directory:bin}/certificate_authority
 ca-private = ${cadirectory:private}
 ca-certs = ${cadirectory:certs}
 ca-newcerts = ${cadirectory:newcerts}
 ca-crl = ${cadirectory:crl}
 
+[certificate-authority-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${certificate-authority:wrapper}
+wrapper-path = ${directory:service}/certificate_authority
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
 requests = ${directory:ca-dir}/requests/
@@ -301,12 +311,18 @@ recipe = slapos.cookbook:certificate_authority.request
 key-file = ${cadirectory:certs}/frontend.key
 cert-file = ${cadirectory:certs}/frontend.crt
 executable = ${directory:service}/frontend_caddy
-wrapper = ${directory:service}/frontend_caddy
+wrapper = ${directory:bin}/frontend_caddy
 key-content = ${configuration:apache-key}
 cert-content = ${configuration:apache-certificate}
 # Put domain name
 name = ${configuration:domain}
 
+[ca-frontend-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${ca-frontend:wrapper}
+wrapper-path = ${directory:service}/frontend_caddy
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
 [ca-custom-frontend]
 < = jinja2-template-base
 template = {{ parameter_dict['template_empty'] }}
@@ -411,6 +427,7 @@ recipe = slapos.cookbook:wrapper
 command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_cop
 wrapper-path = ${trafficserver-variable:wrapper-path}
 environment = TS_ROOT=${buildout:directory}
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 [trafficserver-reload]
 recipe = slapos.cookbook:wrapper
@@ -644,6 +661,7 @@ wrapper-path = ${directory:bin}/nginx-wrapper
 recipe = slapos.cookbook:wrapper
 command-line = ${nginx-wrapper:wrapper-path} -pidfile ${nginx-configuration:pid-file}
 wrapper-path = ${directory:service}/frontend_nginx
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 [dynamic-nginx-frontend-template]
 < = jinja2-template-base
@@ -738,3 +756,4 @@ extra-context =
 {{ key.replace('configuration.', '') }} = {{ dumps(value) }}
 {%-   endif -%}
 {%- endfor -%}
+{%- endif -%} {# if slap_software_type == software_type #}

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -64,13 +64,14 @@ context =
                                   }) %}
 {% endfor %}
 
-{% set authorized_slave_string = slapparameter_dict.pop('-frontend-authorized-slave-string', '') %}
+{% set authorized_slave_string_list = slapparameter_dict.pop('-frontend-authorized-slave-string', '').split() %}
 {% set authorized_slave_list = [] %}
 {% set rejected_slave_dict = {} %}
 {% set used_host_list = [] %}
-{% set unauthorised_message = 'slave not authorised' %}
+{% set unauthorized_message = 'slave not authorized' %}
 {% for slave in slave_instance_list %}
 {%   set slave_error_list = [] %}
+{%   set slave_server_alias_unclashed = [] %}
 {#   BBB: apache_custom_https AND apache_custom_http #}
 {%   set custom_domain = slave.get('custom_domain') %}
 {%   if custom_domain and custom_domain in used_host_list %}
@@ -80,22 +81,31 @@ context =
 {%   endif %}
 {%   if slave.get('server-alias') %}
 {%     for slave_alias in slave['server-alias'].split() %}
-{%       if not validators.domain(slave_alias) %}
+{%       if slave_alias.startswith('*.') %}
+{%         set clean_slave_alias = slave_alias[2:] %}
+{%       else %}
+{%         set clean_slave_alias = slave_alias %}
+{%       endif %}
+{%       if not validators.domain(clean_slave_alias) %}
 {%         do slave_error_list.append('server-alias %r not valid' % (slave_alias,)) %}
 {%       else %}
-{%         if slave_alias in used_host_list %}
+{%         if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
+{#           optionally do something about reporting back that server-alias has been unclashed #}
+{%         elif slave_alias in used_host_list %}
 {%           do slave_error_list.append('server-alias %r clashes' % (slave_alias,)) %}
 {%         else %}
+{%           do slave_server_alias_unclashed.append(slave_alias) %}
 {%           do used_host_list.append(slave_alias) %}
 {%         endif %}
 {%       endif %}
 {%     endfor %}
+{%     do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
 {%   endif %}
 {%   for key in ['caddy_custom_http', 'caddy_custom_https', 'apache_custom_http', 'apache_custom_https'] %}
 {%     if slave.get(key) %}
-{%       if not slave.get('slave_reference') in authorized_slave_string %}
-{%         if not unauthorised_message in slave_error_list %}
-{%           do slave_error_list.append(unauthorised_message) %}
+{%       if not slave.get('slave_reference') in authorized_slave_string_list %}
+{%         if not unauthorized_message in slave_error_list %}
+{%           do slave_error_list.append(unauthorized_message) %}
 {%         endif %}
 {%       elif subprocess_module.call([caddy_custom_http_validator, slave[key]]) == 1 %}
 {%         do slave_error_list.append('slave %s configuration invalid' % (key,)) %}
@@ -122,7 +132,12 @@ context =
 {%     endif %}
 {%   endif %}
 {%   if slave.get('custom_domain') %}
-{%     if not validators.domain(slave['custom_domain']) %}
+{%     if slave['custom_domain'].startswith('*.') %}
+{%       set clean_custom_domain = slave['custom_domain'][2:] %}
+{%     else %}
+{%       set clean_custom_domain = slave['custom_domain'] %}
+{%     endif %}
+{%     if not validators.domain(clean_custom_domain) %}
 {%       do slave_error_list.append('custom_domain %r invalid' % (slave['custom_domain'],)) %}
 {%     endif %}
 {%   endif %}

software/caddy-frontend/instance-caddy-input-schema.json

@@ -12,7 +12,7 @@
       "type": "integer"
     },
     "apache-ca-certificate": {
-      "description": "[NOT IMPLEMENTED] SSL CA Certificate used by the server.",
+      "description": "[NOT IMPLEMENTED] SSL CA Certificate used by the server. You can append it to 'apache-certificate'.",
       "textarea": true,
       "title": "[NOT IMPLEMENTED] SSL CA Certificate",
       "type": "string"

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -141,9 +141,9 @@
     },
     "ssl_ca_crt": {
       "default": "",
-      "description": "[NOT Implemented] Content of the CA certificate file",
+      "description": "Content of the CA certificate file",
       "textarea": true,
-      "title": "[NOT Implemented] SSL Certificate Authority's Certificate",
+      "title": "SSL Certificate Authority's Certificate",
       "type": "string"
     },
     "ssl_crt": {

software/caddy-frontend/instance.cfg.in

@@ -41,6 +41,7 @@ filename = instance-caddy-frontend.cfg
 extensions = jinja2.ext.do
 extra-context =
   section parameter_dict dynamic-template-caddy-frontend-parameters
+  raw software_type single-custom-personal
 
 [dynamic-template-caddy-replicate]
 < = jinja2-template-base

software/caddy-frontend/software.cfg

@@ -12,49 +12,8 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.81
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3
 
 websockify = 0.8.0
-
-# Required by:
-# slapos.toolbox==0.71
-dnspython = 1.15.0
-
-# Required by:
-# slapos.toolbox==0.71
-erp5.util = 0.4.51
-
-# Required by:
-# slapos.toolbox==0.71
-passlib = 1.6.5
-
-# Required by:
-# slapos.toolbox==0.71
-GitPython = 2.0.8
-
-# Required by:
-# slapos.toolbox==0.71
-atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.71
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.71
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.71
-paramiko = 2.0.1
-
-# Required by:
-# slapos.toolbox==0.71
-pycurl = 7.43.0
-
-# Required by:
-# slapos.toolbox==0.71
-rpdb = 0.1.5

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -62,6 +62,9 @@ crl = {{ custom_ssl_directory }}/crl/
 
 {%   set slave_log_folder = logrotate_dict.get('backup') + '/' + slave_reference + "-logs" %}
 
+{#   Pass HTTP2 switch #}
+{%   do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
+
 {#   Set Up log files #}
 {%   do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
 {%   do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}
@@ -160,7 +163,6 @@ value = {{ dumps(slave_instance.get(cert_name)) }}
 {%   do slave_instance.__setitem__('login_certificate', login_certificate) %}
 {%   do slave_instance.__setitem__('login_key', login_key) %}
 {%   do slave_instance.__setitem__('login_ca_crt', login_ca_crt) %}
-{%   do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
 {%   do slave_parameter_dict.__setitem__('ssl_crt', login_certificate) %}
 {%   do slave_parameter_dict.__setitem__('ssl_key', login_key) %}
 {%   if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance %}
@@ -187,7 +189,7 @@ extra-context =
 < = jinja2-template-base
 template = {{ empty_template }}
 rendered = {{ cert_file }}
-cert-content = {{ dumps(slave_instance.get('ssl_crt')) }}
+cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '')) }}
 extra-context =
     key content :cert-content
 {%     endif %}

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -12,12 +12,10 @@
 {%- set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() %}
 {%- set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES %}
 {%- set slave_type = slave_parameter.get('type', '') %}
-{%- set host_list = [] %}
-{%- for host in [slave_parameter.get('custom_domain')] + server_alias_list %}
-{%-   if host not in host_list %}
-{%-     do host_list.append(host) %}
+{%- set host_list = server_alias_list %}
+{%- if slave_parameter.get('custom_domain') not in host_list %}
+{%-   do host_list.append(slave_parameter.get('custom_domain')) %}
 {%- endif %}
-{%- endfor %}
 {%- set backend_url = slave_parameter.get('https-url', slave_parameter.get('url', '')) %}
 {%- set http_host_list = [] %}
 {%- set https_host_list = [] %}
@@ -36,10 +34,6 @@
     status 501 /
 {%- endif %} {#- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter #}
   tls {{ slave_parameter.get('path_to_ssl_crt', slave_parameter.get('login_certificate')) }} {{ slave_parameter.get('path_to_ssl_key', slave_parameter.get('login_key')) }} {
-{%- if slave_parameter.get('path_to_ssl_ca_crt') %}
-    # Configuration of accepted clients
-    clients {{ slave_parameter.get('path_to_ssl_ca_crt') }}
-{%- endif %} {#- if slave_parameter.get('path_to_ssl_ca_crt') #}
 {%- if enable_h2 %}
     # Allow HTTP2
     alpn h2 http/1.1

software/caddy-frontend/test/test.py

@@ -97,12 +97,24 @@ def isHTTP2(domain, ip):
 
 
 class TestDataMixin(object):
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
   def getTrimmedProcessInfo(self):
     return '\n'.join(sorted([
       '%(group)s:%(name)s %(statename)s' % q for q
       in self.getSupervisorRPCServer().supervisor.getAllProcessInfo()]))
 
-  def assertTestData(self, runtime_data):
+  def assertTestData(self, runtime_data, hash_value=None):
     filename = '%s-%s.txt' % (self.id(), 'CADDY')
     test_data_file = os.path.join(
       os.path.dirname(os.path.realpath(__file__)), 'test_data', filename)
@@ -112,6 +124,9 @@ class TestDataMixin(object):
     except IOError:
       test_data = ''
 
+    if hash_value is not None:
+      runtime_data = runtime_data.replace(hash_value, '{hash}')
+
     maxDiff = self.maxDiff
     self.maxDiff = None
     try:
@@ -185,11 +200,22 @@ class TestDataMixin(object):
   def test_supervisor_state(self):
     # give a chance for etc/run scripts to finish
     time.sleep(1)
+
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+    h = self.generateHashFromFiles(hash_files)
+
     runtime_data = self.getTrimmedProcessInfo()
-    self.assertTestData(runtime_data)
+    self.assertTestData(runtime_data, hash_value=h)
 
 
 class HttpFrontendTestCase(SlapOSInstanceTestCase):
+  # show full diffs, as it is required for proper analysis of problems
+  maxDiff = None
+
   @classmethod
   def getSoftwareURLList(cls):
     return [os.path.realpath(os.environ['TEST_SR'])]
@@ -223,8 +249,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     frontend, url = entry
     result = requests.get(url, verify=False)
     self.assertEqual(
+      httplib.OK,
       result.status_code,
-      200,
       'While accessing %r of %r the status code was %r' % (
         url, frontend, result.status_code))
 
@@ -238,7 +264,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     except Exception:
       raise ValueError('JSON decode problem in:\n%s' % (result.text,))
     self.assertTrue(key in j, 'No key %r in %s' % (key, j))
-    self.assertEqual(j[key], value)
+    self.assertEqual(value, j[key])
 
 
 class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
@@ -559,6 +585,19 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'url': cls.backend_url,
         'server-alias': 'alias1.example.com alias2.example.com',
       },
+      'server-alias-wildcard': {
+        'url': cls.backend_url,
+        'server-alias': '*.alias1.example.com',
+      },
+      'server-alias-duplicated': {
+        'url': cls.backend_url,
+        'server-alias': 'alias3.example.com alias3.example.com',
+      },
+      'server-alias_custom_domain-duplicated': {
+        'url': cls.backend_url,
+        'custom_domain': 'alias4.example.com',
+        'server-alias': 'alias4.example.com alias4.example.com',
+      },
       'ssl-proxy-verify_ssl_proxy_ca_crt': {
         'url': cls.backend_https_url,
         'ssl-proxy-verify': True,
@@ -576,6 +615,10 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'url': cls.backend_url,
         'custom_domain': 'customdomain.example.com',
       },
+      'custom_domain_wildcard': {
+        'url': cls.backend_url,
+        'custom_domain': '*.customdomain.example.com',
+      },
       'custom_domain_ssl_crt_ssl_key': {
         'url': cls.backend_url,
         'custom_domain': 'customdomainsslcrtsslkey.example.com',
@@ -685,6 +728,12 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'caddy_custom_http': cls.caddy_custom_http % dict(
           url=cls.backend_url),
       },
+      # this has to be rejected
+      'caddy_custom_http_s': {
+        'url': cls.backend_url,
+        'caddy_custom_https': '# caddy_custom_https_filled_in_rejected_2',
+        'caddy_custom_http': '# caddy_custom_http_filled_in_rejected_2',
+      },
       'prefer-gzip-encoding-to-backend': {
         'url': cls.backend_url,
         'prefer-gzip-encoding-to-backend': 'true',
@@ -708,8 +757,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     # partition w/o etc/trafficserver, but with buildout.cfg
     return [
       q for q in glob.glob(os.path.join(self.instance_path, '*',))
-      if not os.path.exists(os.path.join(q, 'etc', 'trafficserver')) and
-      os.path.exists(os.path.join(q, 'buildout.cfg'))][0]
+      if not os.path.exists(
+        os.path.join(q, 'etc', 'trafficserver')) and os.path.exists(
+          os.path.join(q, 'buildout.cfg'))][0]
 
   def getSlavePartitionPath(self):
     # partition w/ etc/trafficserver
@@ -724,12 +774,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     expected_parameter_dict = {
       'monitor-base-url': None,
       'domain': 'example.com',
-      'accepted-slave-amount': '33',
-      'rejected-slave-amount': '2',
-      'slave-amount': '35',
+      'accepted-slave-amount': '37',
+      'rejected-slave-amount': '3',
+      'slave-amount': '40',
       'rejected-slave-dict':
-      '{"_apache_custom_http_s-rejected": ["slave not authorised"], '
-      '"_caddy_custom_http_s-rejected": ["slave not authorised"]}'
+      '{"_apache_custom_http_s-rejected": ["slave not authorized"], '
+      '"_caddy_custom_http_s-rejected": ["slave not authorized"], '
+      '"_caddy_custom_http_s": ["slave not authorized"]}'
     }
 
     self.assertEqual(
@@ -777,8 +828,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
             self.instance_path, '*', 'etc', 'promise',
             'monitor-httpd-listening-on-tcp'))])
       self.assertEqual(
-        result,
-        set([0])
+        set([0]),
+        result
       )
 
   def test_slave_partition_state(self):
@@ -813,10 +864,10 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     # check that log file contains verbose log
     log_file = glob.glob(
@@ -824,17 +875,17 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         self.instance_path, '*', 'var', 'log', 'httpd', '_empty_access_log'
       ))[0]
 
-    log_regexp = '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} SOME_REMOTE_USER ' \
-                 '\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
-                 '"GET \/test-path HTTP\/1.1" 404 \d+ "-" '\
-                 '"python-requests.*" \d+'
+    log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} SOME_REMOTE_USER ' \
+                 r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
+                 r'"GET \/test-path HTTP\/1.1" 404 \d+ "-" '\
+                 r'"python-requests.*" \d+'
 
     self.assertRegexpMatches(
       open(log_file, 'r').read(),
       log_regexp)
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(result_http.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # check that 404 is as configured
     result_missing = self.fakeHTTPSResult(
@@ -860,7 +911,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'url'].copy()
     self.assertLogAccessUrlWithPop(parameter_dict, 'url')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'url.example.com',
         'replication_number': '1',
@@ -868,15 +918,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://url.example.com',
         'secure_access': 'https://url.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -887,13 +938,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertFalse('remote_user' in j['Incoming Headers'].keys())
 
     self.assertEqual(
-      result.headers['Content-Encoding'],
-      'gzip'
+      'gzip',
+      result.headers['Content-Encoding']
     )
 
     self.assertEqual(
-      result.headers['Set-Cookie'],
-      'secured=value;secure, nonsecured=value'
+      'secured=value;secure, nonsecured=value',
+      result.headers['Set-Cookie']
     )
 
     result_http = self.fakeHTTPResult(
@@ -907,13 +958,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertFalse('remote_user' in j['Incoming Headers'].keys())
 
     self.assertEqual(
-      result_http.headers['Content-Encoding'],
-      'gzip'
+      'gzip',
+      result_http.headers['Content-Encoding']
     )
 
     self.assertEqual(
-      result_http.headers['Set-Cookie'],
-      'secured=value;secure, nonsecured=value'
+      'secured=value;secure, nonsecured=value',
+      result_http.headers['Set-Cookie']
     )
 
   @skip('Feature postponed')
@@ -922,7 +973,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'url'].copy()
     self.assertLogAccessUrlWithPop(parameter_dict, 'url')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'url.example.com',
         'replication_number': '1',
@@ -930,7 +980,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://url.example.com',
         'secure_access': 'https://url.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result_ipv6 = self.fakeHTTPSResult(
@@ -938,13 +989,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       source_ip=GLOBAL_IPV6)
 
     self.assertEqual(
-       result_ipv6.json()['Incoming Headers']['x-forwarded-for'],
-       GLOBAL_IPV6
+       GLOBAL_IPV6,
+       result_ipv6.json()['Incoming Headers']['x-forwarded-for']
     )
 
     self.assertEqual(
-      der2pem(result_ipv6.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result_ipv6.peercert))
 
     self.assertEqualResultJson(result_ipv6, 'Path', '/test-path')
 
@@ -953,7 +1004,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'type-zope-path']
     self.assertLogAccessUrlWithPop(parameter_dict, 'type-zope-path')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typezopepath.example.com',
         'replication_number': '1',
@@ -961,15 +1011,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://typezopepath.example.com',
         'secure_access': 'https://typezopepath.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(
       result,
@@ -983,7 +1034,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'type-zope-default-path']
     self.assertLogAccessUrlWithPop(parameter_dict, 'type-zope-default-path')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typezopedefaultpath.example.com',
         'replication_number': '1',
@@ -991,19 +1041,21 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://typezopedefaultpath.example.com',
         'secure_access': 'https://typezopedefaultpath.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], '')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.headers['Location'],
-      'https://typezopedefaultpath.example.com:%s/default-path' % (HTTPS_PORT,)
+      'https://typezopedefaultpath.example.com:%s/default-path' % (
+        HTTPS_PORT,),
+      result.headers['Location']
     )
 
   def test_server_alias(self):
@@ -1011,7 +1063,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'server-alias']
     self.assertLogAccessUrlWithPop(parameter_dict, 'server-alias')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'serveralias.example.com',
         'replication_number': '1',
@@ -1019,15 +1070,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://serveralias.example.com',
         'secure_access': 'https://serveralias.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1035,8 +1087,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'alias1.example.com', parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1044,8 +1096,100 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'alias2.example.com', parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+  def test_server_alias_wildcard(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'server-alias-wildcard']
+    self.assertLogAccessUrlWithPop(parameter_dict, 'server-alias-wildcard')
+    self.assertEqual(
+      {
+        'domain': 'serveraliaswildcard.example.com',
+        'replication_number': '1',
+        'url': 'http://serveraliaswildcard.example.com',
+        'site_url': 'http://serveraliaswildcard.example.com',
+        'secure_access': 'https://serveraliaswildcard.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+    result = self.fakeHTTPSResult(
+      'wild.alias1.example.com', parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+  def test_server_alias_duplicated(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'server-alias-duplicated']
+    self.assertLogAccessUrlWithPop(parameter_dict, 'server-alias-duplicated')
+    self.assertEqual(
+      {
+        'domain': 'serveraliasduplicated.example.com',
+        'replication_number': '1',
+        'url': 'http://serveraliasduplicated.example.com',
+        'site_url': 'http://serveraliasduplicated.example.com',
+        'secure_access': 'https://serveraliasduplicated.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+    result = self.fakeHTTPSResult(
+      'alias3.example.com', parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+  def test_server_alias_custom_domain_duplicated(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'server-alias_custom_domain-duplicated']
+    self.assertLogAccessUrlWithPop(
+      parameter_dict, 'server-alias_custom_domain-duplicated')
+    self.assertEqual(
+      {
+        'domain': 'alias4.example.com',
+        'replication_number': '1',
+        'url': 'http://alias4.example.com',
+        'site_url': 'http://alias4.example.com',
+        'secure_access': 'https://alias4.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1067,7 +1211,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'https-only']
     self.assertLogAccessUrlWithPop(parameter_dict, 'https-only')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'httpsonly.example.com',
         'replication_number': '1',
@@ -1075,15 +1218,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://httpsonly.example.com',
         'secure_access': 'https://httpsonly.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1091,8 +1235,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      result_http.headers['Location'],
-      'https://httpsonly.example.com/test-path'
+      'https://httpsonly.example.com/test-path',
+      result_http.headers['Location']
     )
 
   def test_custom_domain(self):
@@ -1100,7 +1244,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'custom_domain']
     self.assertLogAccessUrlWithPop(parameter_dict, 'custom_domain')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'customdomain.example.com',
         'replication_number': '1',
@@ -1108,15 +1251,42 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://customdomain.example.com',
         'secure_access': 'https://customdomain.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+  def test_custom_domain_wildcard(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'custom_domain_wildcard']
+    self.assertLogAccessUrlWithPop(parameter_dict, 'custom_domain_wildcard')
+    self.assertEqual(
+      {
+        'domain': '*.customdomain.example.com',
+        'replication_number': '1',
+        'url': 'http://*.customdomain.example.com',
+        'site_url': 'http://*.customdomain.example.com',
+        'secure_access': 'https://*.customdomain.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
+
+    result = self.fakeHTTPSResult(
+      'wild.customdomain.example.com', parameter_dict['public-ipv4'],
+      'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1125,7 +1295,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'custom_domain_ssl_crt_ssl_key']
     self.assertLogAccessUrlWithPop(parameter_dict, 'custom_domain_ssl_crt_key')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'customdomainsslcrtsslkey.example.com',
         'replication_number': '1',
@@ -1133,15 +1302,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://customdomainsslcrtsslkey.example.com',
         'secure_access': 'https://customdomainsslcrtsslkey.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('customdomainsslcrtsslkey.example.com.crt').read())
+      open('customdomainsslcrtsslkey.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1150,7 +1320,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'type-zope']
     self.assertLogAccessUrlWithPop(parameter_dict, 'type-zope')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typezope.example.com',
         'replication_number': '1',
@@ -1158,15 +1327,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://typezope.example.com',
         'secure_access': 'https://typezope.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     try:
       j = result.json()
@@ -1197,7 +1367,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'type-zope-virtualhostroot-http-port')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typezopevirtualhostroothttpport.example.com',
         'replication_number': '1',
@@ -1206,7 +1375,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://typezopevirtualhostroothttpport.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPResult(
@@ -1225,7 +1395,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'type-zope-virtualhostroot-https-port')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typezopevirtualhostroothttpsport.example.com',
         'replication_number': '1',
@@ -1234,15 +1403,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://typezopevirtualhostroothttpsport.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(
       result,
@@ -1256,7 +1426,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'type-notebook']
     self.assertLogAccessUrlWithPop(parameter_dict, 'type-notebook')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typenotebook.nginx.example.com',
         'replication_number': '1',
@@ -1264,7 +1433,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://typenotebook.nginx.example.com',
         'secure_access': 'https://typenotebook.nginx.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
@@ -1272,8 +1442,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       NGINX_HTTPS_PORT)
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1298,7 +1468,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'type-eventsource']
     self.assertLogAccessUrlWithPop(parameter_dict, 'type-eventsource')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typeeventsource.nginx.example.com',
         'replication_number': '1',
@@ -1306,7 +1475,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://typeeventsource.nginx.example.com',
         'secure_access': 'https://typeeventsource.nginx.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
@@ -1314,25 +1484,25 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       NGINX_HTTPS_PORT)
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.content,
-      ''
+      '',
+      result.content
     )
     headers = result.headers.copy()
     self.assertKeyWithPop('Expires', headers)
     self.assertKeyWithPop('Date', headers)
     self.assertEqual(
-      headers,
       {
         'X-Nginx-PushStream-Explain': 'No channel id provided.',
         'Content-Length': '0',
         'Cache-Control': 'no-cache, no-store, must-revalidate',
         'Connection': 'keep-alive',
         'Server': 'nginx'
-      }
+      },
+      headers
     )
 
   def test_type_redirect(self):
@@ -1340,7 +1510,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'type-redirect']
     self.assertLogAccessUrlWithPop(parameter_dict, 'type-redirect')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typeredirect.example.com',
         'replication_number': '1',
@@ -1348,19 +1517,20 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://typeredirect.example.com',
         'secure_access': 'https://typeredirect.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.headers['Location'],
-      '%s/test-path' % (self.backend_url,)
+      '%s/test-path' % (self.backend_url,),
+      result.headers['Location']
     )
 
   def test_ssl_proxy_verify_ssl_proxy_ca_crt(self):
@@ -1370,7 +1540,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'ssl-proxy-verify_ssl_proxy_ca_crt')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'sslproxyverifysslproxycacrt.example.com',
         'replication_number': '1',
@@ -1378,27 +1547,28 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://sslproxyverifysslproxycacrt.example.com',
         'secure_access': 'https://sslproxyverifysslproxycacrt.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.status_code,
-      httplib.NOT_IMPLEMENTED
+      httplib.NOT_IMPLEMENTED,
+      result.status_code
     )
 
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      result_http.status_code,
-      httplib.NOT_IMPLEMENTED
+      httplib.NOT_IMPLEMENTED,
+      result_http.status_code
     )
 
   def test_ssl_proxy_verify_unverified(self):
@@ -1408,7 +1578,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'ssl-proxy-verify-unverified')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'sslproxyverifyunverified.example.com',
         'replication_number': '1',
@@ -1416,19 +1585,20 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://sslproxyverifyunverified.example.com',
         'secure_access': 'https://sslproxyverifyunverified.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.status_code,
-      502
+      httplib.BAD_GATEWAY,
+      result.status_code
     )
 
   def test_enable_cache_ssl_proxy_verify_ssl_proxy_ca_crt(self):
@@ -1438,7 +1608,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablecachesslproxyverifysslproxycacrt.example.com',
         'replication_number': '1',
@@ -1448,27 +1617,28 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://enablecachesslproxyverifysslproxycacrt.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.status_code,
-      httplib.NOT_IMPLEMENTED
+      httplib.NOT_IMPLEMENTED,
+      result.status_code
     )
 
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      result_http.status_code,
-      httplib.NOT_IMPLEMENTED
+      httplib.NOT_IMPLEMENTED,
+      result_http.status_code
     )
 
   def test_enable_cache_ssl_proxy_verify_unverified(self):
@@ -1478,7 +1648,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'enable_cache-ssl-proxy-verify-unverified')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablecachesslproxyverifyunverified.example.com',
         'replication_number': '1',
@@ -1487,19 +1656,20 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://enablecachesslproxyverifyunverified.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.status_code,
-      502
+      httplib.BAD_GATEWAY,
+      result.status_code
     )
 
   def test_type_zope_ssl_proxy_verify_ssl_proxy_ca_crt(self):
@@ -1509,7 +1679,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typezopesslproxyverifysslproxycacrt.example.com',
         'replication_number': '1',
@@ -1518,27 +1687,28 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://typezopesslproxyverifysslproxycacrt.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.status_code,
-      httplib.NOT_IMPLEMENTED
+      httplib.NOT_IMPLEMENTED,
+      result.status_code
     )
 
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      result_http.status_code,
-      httplib.NOT_IMPLEMENTED
+      httplib.NOT_IMPLEMENTED,
+      result_http.status_code
     )
 
   def test_type_zope_ssl_proxy_verify_unverified(self):
@@ -1548,7 +1718,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'type-zope-ssl-proxy-verify-unverified')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'typezopesslproxyverifyunverified.example.com',
         'replication_number': '1',
@@ -1557,19 +1726,20 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://typezopesslproxyverifyunverified.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.status_code,
-      502
+      httplib.BAD_GATEWAY,
+      result.status_code
     )
 
   def test_monitor_ipv6_test(self):
@@ -1577,7 +1747,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'monitor-ipv6-test']
     self.assertLogAccessUrlWithPop(parameter_dict, 'monitor-ipv6-test')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'monitoripv6test.example.com',
         'replication_number': '1',
@@ -1585,21 +1754,22 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://monitoripv6test.example.com',
         'secure_access': 'https://monitoripv6test.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(result_http.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
     open(
@@ -1620,7 +1790,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'monitor-ipv4-test']
     self.assertLogAccessUrlWithPop(parameter_dict, 'monitor-ipv4-test')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'monitoripv4test.example.com',
         'replication_number': '1',
@@ -1628,21 +1797,22 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://monitoripv4test.example.com',
         'secure_access': 'https://monitoripv4test.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(result_http.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
     open(
@@ -1663,7 +1833,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       're6st-optimal-test']
     self.assertLogAccessUrlWithPop(parameter_dict, 're6st-optimal-test')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 're6stoptimaltest.example.com',
         'replication_number': '1',
@@ -1671,21 +1840,22 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://re6stoptimaltest.example.com',
         'secure_access': 'https://re6stoptimaltest.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(result_http.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
     open(
@@ -1707,7 +1877,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'enable_cache']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable_cache')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablecache.example.com',
         'replication_number': '1',
@@ -1715,15 +1884,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://enablecache.example.com',
         'secure_access': 'https://enablecache.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1741,10 +1911,10 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     headers.pop('Keep-Alive', None)
 
     self.assertEqual(
-      headers,
       {'Content-type': 'application/json',
        'Set-Cookie': 'secured=value;secure, nonsecured=value',
-       'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'}
+       'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'},
+      headers
     )
 
     result_direct = self.fakeHTTPResult(
@@ -1760,13 +1930,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertFalse('remote_user' in j['Incoming Headers'].keys())
 
     self.assertEqual(
-      result_direct.headers['Content-Encoding'],
-      'gzip'
+      'gzip',
+      result_direct.headers['Content-Encoding']
     )
 
     self.assertEqual(
-      result_direct.headers['Set-Cookie'],
-      'secured=value;secure, nonsecured=value'
+      'secured=value;secure, nonsecured=value',
+      result_direct.headers['Set-Cookie']
     )
 
     result_direct_https_backend = self.fakeHTTPResult(
@@ -1784,13 +1954,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertFalse('remote_user' in j['Incoming Headers'].keys())
 
     self.assertEqual(
-      result_direct_https_backend.headers['Content-Encoding'],
-      'gzip'
+      'gzip',
+      result_direct_https_backend.headers['Content-Encoding']
     )
 
     self.assertEqual(
-      result_direct_https_backend.headers['Set-Cookie'],
-      'secured=value;secure, nonsecured=value'
+      'secured=value;secure, nonsecured=value',
+      result_direct_https_backend.headers['Set-Cookie']
     )
 
   def test_enable_cache_disable_no_cache_request(self):
@@ -1799,7 +1969,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'enable_cache-disable-no-cache-request')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablecachedisablenocacherequest.example.com',
         'replication_number': '1',
@@ -1808,7 +1977,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://enablecachedisablenocacherequest.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
@@ -1816,8 +1986,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       headers={'Pragma': 'no-cache', 'Cache-Control': 'something'})
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1835,10 +2005,10 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     headers.pop('Keep-Alive', None)
 
     self.assertEqual(
-      headers,
       {'Content-type': 'application/json',
        'Set-Cookie': 'secured=value;secure, nonsecured=value',
-       'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'}
+       'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'},
+      headers
     )
 
     try:
@@ -1853,7 +2023,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'enable_cache-disable-via-header')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablecachedisableviaheader.example.com',
         'replication_number': '1',
@@ -1862,15 +2031,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://enablecachedisableviaheader.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1887,10 +2057,10 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     headers.pop('Keep-Alive', None)
 
     self.assertEqual(
-      headers,
       {'Content-type': 'application/json',
        'Set-Cookie': 'secured=value;secure, nonsecured=value',
-       'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'}
+       'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'},
+      headers
     )
 
   def test_enable_http2_false(self):
@@ -1898,7 +2068,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'enable-http2-false']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable-http2-false')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablehttp2false.example.com',
         'replication_number': '1',
@@ -1907,15 +2076,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://enablehttp2false.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1931,13 +2101,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     headers.pop('Keep-Alive', None)
 
     self.assertEqual(
-      headers,
       {
         'Vary': 'Accept-Encoding',
         'Content-Type': 'application/json',
         'Set-Cookie': 'secured=value;secure, nonsecured=value',
         'Content-Encoding': 'gzip',
-      }
+      },
+      headers
     )
 
     self.assertFalse(
@@ -1948,7 +2118,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'enable-http2-default']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable-http2-default')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablehttp2default.example.com',
         'replication_number': '1',
@@ -1957,15 +2126,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://enablehttp2default.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -1981,13 +2151,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     headers.pop('Keep-Alive', None)
 
     self.assertEqual(
-      headers,
       {
         'Vary': 'Accept-Encoding',
         'Content-type': 'application/json',
         'Set-Cookie': 'secured=value;secure, nonsecured=value',
         'Content-Encoding': 'gzip',
-      }
+      },
+      headers
     )
 
     self.assertTrue(
@@ -1999,7 +2169,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'prefer-gzip-encoding-to-backend')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'prefergzipencodingtobackend.example.com',
         'replication_number': '1',
@@ -2008,7 +2177,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'secure_access':
         'https://prefergzipencodingtobackend.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
@@ -2016,13 +2186,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       headers={'Accept-Encoding': 'gzip, deflate'})
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
     self.assertEqual(
-      result.json()['Incoming Headers']['accept-encoding'], 'gzip')
+      'gzip', result.json()['Incoming Headers']['accept-encoding'])
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
@@ -2031,14 +2201,13 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
     self.assertEqual(
-      result.json()['Incoming Headers']['accept-encoding'], 'deflate')
+      'deflate', result.json()['Incoming Headers']['accept-encoding'])
 
   def test_disabled_cookie_list(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'disabled-cookie-list']
     self.assertLogAccessUrlWithPop(parameter_dict, 'disabled-cookie-list')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'disabledcookielist.example.com',
         'replication_number': '1',
@@ -2046,7 +2215,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://disabledcookielist.example.com',
         'secure_access': 'https://disabledcookielist.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
@@ -2058,20 +2228,20 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         ))
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
     self.assertEqual(
-      result.json()['Incoming Headers']['cookie'], 'Coffee=present')
+      'Coffee=present', result.json()['Incoming Headers']['cookie'])
 
   def test_apache_custom_http_s_rejected(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'apache_custom_http_s-rejected']
     self.assertEqual(
       {
-        'request-error-list': '["slave not authorised"]'
+        'request-error-list': '["slave not authorized"]'
       },
       parameter_dict)
     slave_configuration_file_list = glob.glob(os.path.join(
@@ -2093,8 +2263,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'apache_custom_http_s-accepted')
     self.assertEqual(
-      parameter_dict,
-      {'replication_number': '1', 'public-ipv4': LOCAL_IPV4}
+      {'replication_number': '1', 'public-ipv4': LOCAL_IPV4},
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
@@ -2102,8 +2272,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -2119,11 +2289,11 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     headers.pop('Keep-Alive', None)
 
     self.assertEqual(
-      headers,
       {
         'Content-type': 'application/json',
         'Set-Cookie': 'secured=value;secure, nonsecured=value'
-      }
+      },
+      headers
     )
 
     result_http = self.fakeHTTPResult(
@@ -2149,7 +2319,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'caddy_custom_http_s-rejected']
     self.assertEqual(
       {
-        'request-error-list': '["slave not authorised"]'
+        'request-error-list': '["slave not authorized"]'
       },
       parameter_dict)
     slave_configuration_file_list = glob.glob(os.path.join(
@@ -2165,14 +2335,35 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       if 'caddy_custom_http_filled_in_rejected' in open(q).read()]
     self.assertEqual([], configuration_file_with_custom_http_list)
 
+  def test_caddy_custom_http_s(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'caddy_custom_http_s']
+    self.assertEqual(
+      {
+        'request-error-list': '["slave not authorized"]'
+      },
+      parameter_dict)
+    slave_configuration_file_list = glob.glob(os.path.join(
+      self.instance_path, '*', 'etc', '*slave-conf.d', '*.conf'))
+    # no configuration file contains provided custom http
+    configuration_file_with_custom_https_list = [
+      q for q in slave_configuration_file_list
+      if 'caddy_custom_https_filled_in_rejected_2' in open(q).read()]
+    self.assertEqual([], configuration_file_with_custom_https_list)
+
+    configuration_file_with_custom_http_list = [
+      q for q in slave_configuration_file_list
+      if 'caddy_custom_http_filled_in_rejected_2' in open(q).read()]
+    self.assertEqual([], configuration_file_with_custom_http_list)
+
   def test_caddy_custom_http_s_accepted(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'caddy_custom_http_s-accepted']
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'caddy_custom_http_s-accepted')
     self.assertEqual(
-      parameter_dict,
-      {'replication_number': '1', 'public-ipv4': LOCAL_IPV4}
+      {'replication_number': '1', 'public-ipv4': LOCAL_IPV4},
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
@@ -2180,8 +2371,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -2197,11 +2388,11 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     headers.pop('Keep-Alive', None)
 
     self.assertEqual(
-      headers,
       {
         'Content-type': 'application/json',
         'Set-Cookie': 'secured=value;secure, nonsecured=value'
-      }
+      },
+      headers
     )
 
     result_http = self.fakeHTTPResult(
@@ -2227,7 +2418,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'url_https-url']
     self.assertLogAccessUrlWithPop(parameter_dict, 'url_https-url')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'urlhttpsurl.example.com',
         'replication_number': '1',
@@ -2235,15 +2425,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'site_url': 'http://urlhttpsurl.example.com',
         'secure_access': 'https://urlhttpsurl.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/https/test-path')
 
@@ -2286,7 +2477,6 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       'replicate']
     self.assertLogAccessUrlWithPop(parameter_dict, 'replicate')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'replicate.example.com',
         'replication_number': '2',
@@ -2294,15 +2484,16 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'site_url': 'http://replicate.example.com',
         'secure_access': 'https://replicate.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -2360,7 +2551,6 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
       'enable-http2-default']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable-http2-default')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablehttp2default.example.com',
         'replication_number': '1',
@@ -2369,7 +2559,8 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
         'secure_access':
         'https://enablehttp2default.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     self.assertFalse(
@@ -2380,7 +2571,6 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
       'enable-http2-false']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable-http2-false')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablehttp2false.example.com',
         'replication_number': '1',
@@ -2389,7 +2579,8 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
         'secure_access':
         'https://enablehttp2false.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     self.assertFalse(
@@ -2400,7 +2591,6 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
       'enable-http2-true']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable-http2-true')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablehttp2true.example.com',
         'replication_number': '1',
@@ -2409,7 +2599,8 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
         'secure_access':
         'https://enablehttp2true.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     self.assertTrue(
@@ -2452,7 +2643,6 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
       'enable-http2-default']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable-http2-default')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablehttp2default.example.com',
         'replication_number': '1',
@@ -2461,7 +2651,8 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
         'secure_access':
         'https://enablehttp2default.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     self.assertTrue(
@@ -2472,7 +2663,6 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
       'enable-http2-false']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable-http2-false')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablehttp2false.example.com',
         'replication_number': '1',
@@ -2481,7 +2671,8 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
         'secure_access':
         'https://enablehttp2false.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     self.assertFalse(
@@ -2492,7 +2683,6 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
       'enable-http2-true']
     self.assertLogAccessUrlWithPop(parameter_dict, 'enable-http2-true')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'enablehttp2true.example.com',
         'replication_number': '1',
@@ -2501,7 +2691,8 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
         'secure_access':
         'https://enablehttp2true.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     self.assertTrue(
@@ -2533,7 +2724,6 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
       'default']
     self.assertLogAccessUrlWithPop(parameter_dict, 'default')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'default.None',
         'replication_number': '1',
@@ -2541,7 +2731,8 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
         'site_url': 'http://default.None',
         'secure_access': 'https://default.None',
         'public-ipv4': None,
-      }
+      },
+      parameter_dict
     )
 
     re6st_connectivity_promise_list = glob.glob(
@@ -2583,7 +2774,6 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
       'default']
     self.assertLogAccessUrlWithPop(parameter_dict, 'default')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'default.None',
         'replication_number': '1',
@@ -2591,7 +2781,8 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
         'site_url': 'http://default.None',
         'secure_access': 'https://default.None',
         'public-ipv4': None,
-      }
+      },
+      parameter_dict
     )
 
     re6st_connectivity_promise_list = glob.glob(
@@ -2664,7 +2855,6 @@ class TestMalformedBackenUrlSlave(SlaveHttpFrontendTestCase,
       'empty']
     self.assertLogAccessUrlWithPop(parameter_dict, 'empty')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'empty.example.com',
         'replication_number': '1',
@@ -2672,38 +2862,39 @@ class TestMalformedBackenUrlSlave(SlaveHttpFrontendTestCase,
         'site_url': 'http://empty.example.com',
         'secure_access': 'https://empty.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
   def test_url(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'url'].copy()
     self.assertEqual(
-      parameter_dict,
       {
         'request-error-list': '["slave url \\"https://[fd46::c2ae]:!py!'
         'u\'123123\'\\" invalid"]'
-      }
+      },
+      parameter_dict
     )
 
   def test_https_url(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'https-url'].copy()
     self.assertEqual(
-      parameter_dict,
       {
         'request-error-list': '["slave https-url \\"https://[fd46::c2ae]:'
         '!py!u\'123123\'\\" invalid"]'
-      }
+      },
+      parameter_dict
     )
 
 
@@ -2727,11 +2918,11 @@ class TestDefaultMonitorHttpdPort(SlaveHttpFrontendTestCase, TestDataMixin):
       'test']
     self.assertKeyWithPop('log-access-url', parameter_dict)
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'test.None', 'replication_number': '1',
         'url': 'http://test.None', 'site_url': 'http://test.None',
-        'secure_access': 'https://test.None', 'public-ipv4': None}
+        'secure_access': 'https://test.None', 'public-ipv4': None},
+      parameter_dict
     )
     master_monitor_conf = open(os.path.join(
       self.instance_path, 'TestDefaultMonitorHttpdPort-0', 'etc',
@@ -2779,8 +2970,9 @@ class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin):
     # partition w/o etc/trafficserver, but with buildout.cfg
     return [
       q for q in glob.glob(os.path.join(self.instance_path, '*',))
-      if not os.path.exists(os.path.join(q, 'etc', 'trafficserver')) and
-      os.path.exists(os.path.join(q, 'buildout.cfg'))][0]
+      if not os.path.exists(
+        os.path.join(q, 'etc', 'trafficserver')) and os.path.exists(
+          os.path.join(q, 'buildout.cfg'))][0]
 
   def getSlavePartitionPath(self):
     # partition w/ etc/trafficserver
@@ -2793,7 +2985,6 @@ class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin):
       'url'].copy()
     self.assertLogAccessUrlWithPop(parameter_dict, 'url')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'url.example.com',
         'replication_number': '1',
@@ -2801,15 +2992,16 @@ class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin):
         'site_url': 'http://url.example.com',
         'secure_access': 'https://url.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -2823,13 +3015,13 @@ class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertKeyWithPop('Content-Length', result.headers)
 
     self.assertEqual(
-      result.headers,
       {'Content-Encoding': 'gzip',
        'Alt-Svc': 'quic=":11443"; ma=2592000; v="39"',  # QUIC advertises
        'Set-Cookie': 'secured=value;secure, nonsecured=value',
        'Vary': 'Accept-Encoding',
        'Server': 'Caddy, BaseHTTP/0.3 Python/2.7.14',
-       'Content-Type': 'application/json'}
+       'Content-Type': 'application/json'},
+      result.headers
     )
 
     result_http = self.fakeHTTPResult(
@@ -2843,13 +3035,13 @@ class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertFalse('remote_user' in j['Incoming Headers'].keys())
 
     self.assertEqual(
-      result_http.headers['Content-Encoding'],
-      'gzip'
+      'gzip',
+      result_http.headers['Content-Encoding']
     )
 
     self.assertEqual(
-      result_http.headers['Set-Cookie'],
-      'secured=value;secure, nonsecured=value'
+      'secured=value;secure, nonsecured=value',
+      result_http.headers['Set-Cookie']
     )
 
 
@@ -2890,7 +3082,7 @@ https://www.google.com {}""",
       },
       're6st-optimal-test-unsafe': {
         're6st-optimal-test':
-        'new\nline;rm -fr ~;,new\line\n[s${esection:eoption}',
+        'new\nline;rm -fr ~;,new\\line\n[s${esection:eoption}',
       },
       'custom_domain-unsafe': {
         'custom_domain': '${section:option} afterspace\nafternewline',
@@ -2959,7 +3151,6 @@ https://www.google.com {}""",
       'server-alias-same']
     self.assertLogAccessUrlWithPop(parameter_dict, 'server-alias-same')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'serveraliassame.example.com',
         'replication_number': '1',
@@ -2967,15 +3158,16 @@ https://www.google.com {}""",
         'site_url': 'http://serveraliassame.example.com',
         'secure_access': 'https://serveraliassame.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
@@ -2984,7 +3176,6 @@ https://www.google.com {}""",
       're6st-optimal-test-unsafe']
     self.assertLogAccessUrlWithPop(parameter_dict, 're6st-optimal-test-unsafe')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 're6stoptimaltestunsafe.example.com',
         'replication_number': '1',
@@ -2992,17 +3183,18 @@ https://www.google.com {}""",
         'site_url': 'http://re6stoptimaltestunsafe.example.com',
         'secure_access': 'https://re6stoptimaltestunsafe.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
     open(
@@ -3029,7 +3221,6 @@ https://www.google.com {}""",
     self.assertLogAccessUrlWithPop(
       parameter_dict, 're6st-optimal-test-nocomma')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 're6stoptimaltestnocomma.example.com',
         'replication_number': '1',
@@ -3037,17 +3228,18 @@ https://www.google.com {}""",
         'site_url': 'http://re6stoptimaltestnocomma.example.com',
         'secure_access': 'https://re6stoptimaltestnocomma.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     # assert that there is no nocomma file
     monitor_file_list = glob.glob(
@@ -3063,24 +3255,24 @@ https://www.google.com {}""",
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'custom_domain-unsafe']
     self.assertEqual(
-      parameter_dict,
       {
         'request-error-list':
         '["custom_domain \'${section:option} afterspace\\\\nafternewline\' '
         'invalid"]'
-      }
+      },
+      parameter_dict
     )
 
   def test_server_alias_unsafe(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'server-alias-unsafe']
     self.assertEqual(
-      parameter_dict,
       {
         'request-error-list':
         '["server-alias \'${section:option}\' not valid", "server-alias '
         '\'afterspace\' not valid"]'
-      }
+      },
+      parameter_dict
     )
 
   def test_virtualhostroot_http_port_unsafe(self):
@@ -3089,7 +3281,6 @@ https://www.google.com {}""",
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'virtualhostroot-http-port-unsafe')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'virtualhostroothttpportunsafe.example.com',
         'replication_number': '1',
@@ -3098,7 +3289,8 @@ https://www.google.com {}""",
         'secure_access':
         'https://virtualhostroothttpportunsafe.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPResult(
@@ -3117,7 +3309,6 @@ https://www.google.com {}""",
     self.assertLogAccessUrlWithPop(
       parameter_dict, 'virtualhostroot-https-port-unsafe')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'virtualhostroothttpsportunsafe.example.com',
         'replication_number': '1',
@@ -3126,15 +3317,16 @@ https://www.google.com {}""",
         'secure_access':
         'https://virtualhostroothttpsportunsafe.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqualResultJson(
       result,
@@ -3148,7 +3340,6 @@ https://www.google.com {}""",
       'default-path-unsafe']
     self.assertLogAccessUrlWithPop(parameter_dict, 'default-path-unsafe')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'defaultpathunsafe.example.com',
         'replication_number': '1',
@@ -3156,20 +3347,21 @@ https://www.google.com {}""",
         'site_url': 'http://defaultpathunsafe.example.com',
         'secure_access': 'https://defaultpathunsafe.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], '')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
     self.assertEqual(
-      result.headers['Location'],
       'https://defaultpathunsafe.example.com:%s/%%24%%7Bsection%%3Aoption%%7D'
-      '%%0An%%22%%0Aewline%%0A%%7D%%0A%%7Dproxy%%0A/slashed' % (HTTPS_PORT,)
+      '%%0An%%22%%0Aewline%%0A%%7D%%0A%%7Dproxy%%0A/slashed' % (HTTPS_PORT,),
+      result.headers['Location']
     )
 
   def test_monitor_ipv4_test_unsafe(self):
@@ -3177,7 +3369,6 @@ https://www.google.com {}""",
       'monitor-ipv4-test-unsafe']
     self.assertLogAccessUrlWithPop(parameter_dict, 'monitor-ipv4-test-unsafe')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'monitoripv4testunsafe.example.com',
         'replication_number': '1',
@@ -3185,21 +3376,22 @@ https://www.google.com {}""",
         'site_url': 'http://monitoripv4testunsafe.example.com',
         'secure_access': 'https://monitoripv4testunsafe.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(result_http.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
     open(
@@ -3220,7 +3412,6 @@ https://www.google.com {}""",
       'monitor-ipv6-test-unsafe']
     self.assertLogAccessUrlWithPop(parameter_dict, 'monitor-ipv6-test-unsafe')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'monitoripv6testunsafe.example.com',
         'replication_number': '1',
@@ -3228,21 +3419,22 @@ https://www.google.com {}""",
         'site_url': 'http://monitoripv6testunsafe.example.com',
         'secure_access': 'https://monitoripv6testunsafe.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      der2pem(result.peercert),
-      open('wildcard.example.com.crt').read())
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
 
-    self.assertEqual(result.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result.status_code)
 
     result_http = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(result_http.status_code, httplib.NOT_FOUND)
+    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
 
     # rewrite SR/bin/is-icmp-packet-lost
     open(
@@ -3262,20 +3454,20 @@ https://www.google.com {}""",
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'ssl_key-ssl_crt-unsafe']
     self.assertEqual(
-      parameter_dict,
-      {'request-error-list': '["slave ssl_key and ssl_crt does not match"]'}
+      {'request-error-list': '["slave ssl_key and ssl_crt does not match"]'},
+      parameter_dict
     )
 
   def test_caddy_custom_http_s_reject(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'caddy_custom_http_s-reject']
     self.assertEqual(
-      parameter_dict,
       {
         'request-error-list':
         '["slave caddy_custom_http configuration invalid", '
         '"slave caddy_custom_https configuration invalid"]'
-      }
+      },
+      parameter_dict
     )
 
 
@@ -3327,10 +3519,9 @@ class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin):
       'rejected-slave-amount': '3',
       'slave-amount': '4',
       'rejected-slave-dict':
-      '{"_site_4": ["custom_domain \'duplicate.example.com\' clashes", '
-      '"server-alias \'duplicate.example.com\' clashes"], "_site_1": '
-      '["custom_domain \'duplicate.example.com\' clashes"], "_site_3": '
-      '["server-alias \'duplicate.example.com\' clashes"]}'
+      '{"_site_4": ["custom_domain \'duplicate.example.com\' clashes"], '
+      '"_site_1": ["custom_domain \'duplicate.example.com\' clashes"], '
+      '"_site_3": ["server-alias \'duplicate.example.com\' clashes"]}'
     }
 
     self.assertEqual(
@@ -3342,11 +3533,11 @@ class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'site_1']
     self.assertEqual(
-      parameter_dict,
       {
         'request-error-list':
         '["custom_domain \'duplicate.example.com\' clashes"]'
-      }
+      },
+      parameter_dict
     )
 
   def test_site_2(self):
@@ -3354,7 +3545,6 @@ class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin):
       'site_2']
     self.assertLogAccessUrlWithPop(parameter_dict, 'site_2')
     self.assertEqual(
-      parameter_dict,
       {
         'domain': 'duplicate.example.com',
         'replication_number': '1',
@@ -3362,28 +3552,81 @@ class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin):
         'site_url': 'http://duplicate.example.com',
         'secure_access': 'https://duplicate.example.com',
         'public-ipv4': LOCAL_IPV4,
-      }
+      },
+      parameter_dict
     )
 
   def test_site_3(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'site_3']
     self.assertEqual(
-      parameter_dict,
       {
         'request-error-list':
         '["server-alias \'duplicate.example.com\' clashes"]'
-      }
+      },
+      parameter_dict,
     )
 
   def test_site_4(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[
       'site_4']
     self.assertEqual(
-      parameter_dict,
       {
         'request-error-list':
-        '["custom_domain \'duplicate.example.com\' clashes", "server-alias '
-        '\'duplicate.example.com\' clashes"]'
-      }
+        '["custom_domain \'duplicate.example.com\' clashes"]'
+      },
+      parameter_dict
     )
+
+
+class AutoRestartTestCase(SlaveHttpFrontendTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '-frontend-1-state': 'stopped',
+    }
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'test': {
+        'url': cls.backend_url,
+      },
+    }
+
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  def test_hashes(self):
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    expected_process_names = [
+      'frontend_caddy-{hash}-on-watch',
+      'frontend_nginx-{hash}-on-watch',
+      'trafficserver-{hash}-on-watch',
+      'certificate_authority-{hash}-on-watch',
+      'crond-{hash}',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_names = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+
+    for name in expected_process_names:
+      h = self.generateHashFromFiles(hash_files)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_names)

software/caddy-frontend/test/test_data/test.TestDefaultMonitorHttpdPort.test_supervisor_state-CADDY.txt

 TestDefaultMonitorHttpdPort-0:bootstrap-monitor EXITED
 TestDefaultMonitorHttpdPort-0:certificate_authority-on-watch RUNNING
-TestDefaultMonitorHttpdPort-0:crond RUNNING
+TestDefaultMonitorHttpdPort-0:crond-{hash} RUNNING
+TestDefaultMonitorHttpdPort-0:monitor-httpd-{hash}-on-watch EXITED
 TestDefaultMonitorHttpdPort-0:monitor-httpd-graceful EXITED
-TestDefaultMonitorHttpdPort-0:monitor-httpd-on-watch EXITED
 TestDefaultMonitorHttpdPort-1:6tunnel-26011-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:6tunnel-26012-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:6tunnel-4443-on-watch STOPPED
@@ -10,14 +10,15 @@ TestDefaultMonitorHttpdPort-1:6tunnel-8080-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:6tunnel-8081-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:6tunnel-9443-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:bootstrap-monitor STOPPED
-TestDefaultMonitorHttpdPort-1:certificate_authority-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:certificate_authority-{hash}-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:crond-{hash} STOPPED
 TestDefaultMonitorHttpdPort-1:crond-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:frontend-caddy-safe-graceful STOPPED
 TestDefaultMonitorHttpdPort-1:frontend-nginx-safe-graceful STOPPED
-TestDefaultMonitorHttpdPort-1:frontend_caddy-on-watch STOPPED
-TestDefaultMonitorHttpdPort-1:frontend_nginx-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:frontend_caddy-{hash}-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:frontend_nginx-{hash}-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:monitor-httpd-{hash}-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:monitor-httpd-graceful STOPPED
-TestDefaultMonitorHttpdPort-1:monitor-httpd-on-watch STOPPED
-TestDefaultMonitorHttpdPort-1:trafficserver-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:trafficserver-{hash}-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:trafficserver-reload STOPPED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestDuplicateSiteKeyProtection.test_supervisor_state-CADDY.txt

 TestDuplicateSiteKeyProtection-0:bootstrap-monitor EXITED
 TestDuplicateSiteKeyProtection-0:certificate_authority-on-watch RUNNING
-TestDuplicateSiteKeyProtection-0:crond RUNNING
+TestDuplicateSiteKeyProtection-0:crond-{hash} RUNNING
+TestDuplicateSiteKeyProtection-0:monitor-httpd-{hash}-on-watch RUNNING
 TestDuplicateSiteKeyProtection-0:monitor-httpd-graceful EXITED
-TestDuplicateSiteKeyProtection-0:monitor-httpd-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-11080-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-11443-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestDuplicateSiteKeyProtection-1:6tunnel-12443-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-26011-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-26012-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:bootstrap-monitor EXITED
-TestDuplicateSiteKeyProtection-1:certificate_authority-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:certificate_authority-{hash}-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:crond-{hash} RUNNING
 TestDuplicateSiteKeyProtection-1:crond-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:frontend-caddy-safe-graceful EXITED
 TestDuplicateSiteKeyProtection-1:frontend-nginx-safe-graceful EXITED
-TestDuplicateSiteKeyProtection-1:frontend_caddy-on-watch RUNNING
-TestDuplicateSiteKeyProtection-1:frontend_nginx-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:frontend_caddy-{hash}-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:frontend_nginx-{hash}-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:monitor-httpd-{hash}-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:monitor-httpd-graceful EXITED
-TestDuplicateSiteKeyProtection-1:monitor-httpd-on-watch RUNNING
-TestDuplicateSiteKeyProtection-1:trafficserver-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:trafficserver-{hash}-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_supervisor_state-CADDY.txt

 TestEnableHttp2ByDefaultDefaultSlave-0:bootstrap-monitor EXITED
 TestEnableHttp2ByDefaultDefaultSlave-0:certificate_authority-on-watch RUNNING
-TestEnableHttp2ByDefaultDefaultSlave-0:crond RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-0:crond-{hash} RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-0:monitor-httpd-graceful EXITED
-TestEnableHttp2ByDefaultDefaultSlave-0:monitor-httpd-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-11080-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-11443-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-12443-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-26011-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-26012-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:bootstrap-monitor EXITED
-TestEnableHttp2ByDefaultDefaultSlave-1:certificate_authority-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:crond-{hash} RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:crond-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:frontend-caddy-safe-graceful EXITED
 TestEnableHttp2ByDefaultDefaultSlave-1:frontend-nginx-safe-graceful EXITED
-TestEnableHttp2ByDefaultDefaultSlave-1:frontend_caddy-on-watch RUNNING
-TestEnableHttp2ByDefaultDefaultSlave-1:frontend_nginx-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:monitor-httpd-graceful EXITED
-TestEnableHttp2ByDefaultDefaultSlave-1:monitor-httpd-on-watch RUNNING
-TestEnableHttp2ByDefaultDefaultSlave-1:trafficserver-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_supervisor_state-CADDY.txt

 TestEnableHttp2ByDefaultFalseSlave-0:bootstrap-monitor EXITED
 TestEnableHttp2ByDefaultFalseSlave-0:certificate_authority-on-watch RUNNING
-TestEnableHttp2ByDefaultFalseSlave-0:crond RUNNING
+TestEnableHttp2ByDefaultFalseSlave-0:crond-{hash} RUNNING
+TestEnableHttp2ByDefaultFalseSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-0:monitor-httpd-graceful EXITED
-TestEnableHttp2ByDefaultFalseSlave-0:monitor-httpd-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-11080-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-11443-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-12443-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-26011-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-26012-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:bootstrap-monitor EXITED
-TestEnableHttp2ByDefaultFalseSlave-1:certificate_authority-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:crond-{hash} RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:crond-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:frontend-caddy-safe-graceful EXITED
 TestEnableHttp2ByDefaultFalseSlave-1:frontend-nginx-safe-graceful EXITED
-TestEnableHttp2ByDefaultFalseSlave-1:frontend_caddy-on-watch RUNNING
-TestEnableHttp2ByDefaultFalseSlave-1:frontend_nginx-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:monitor-httpd-graceful EXITED
-TestEnableHttp2ByDefaultFalseSlave-1:monitor-httpd-on-watch RUNNING
-TestEnableHttp2ByDefaultFalseSlave-1:trafficserver-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_supervisor_state-CADDY.txt

 TestMalformedBackenUrlSlave-0:bootstrap-monitor EXITED
 TestMalformedBackenUrlSlave-0:certificate_authority-on-watch RUNNING
-TestMalformedBackenUrlSlave-0:crond RUNNING
+TestMalformedBackenUrlSlave-0:crond-{hash} RUNNING
+TestMalformedBackenUrlSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestMalformedBackenUrlSlave-0:monitor-httpd-graceful EXITED
-TestMalformedBackenUrlSlave-0:monitor-httpd-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-11080-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-11443-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestMalformedBackenUrlSlave-1:6tunnel-12443-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-26011-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-26012-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:bootstrap-monitor EXITED
-TestMalformedBackenUrlSlave-1:certificate_authority-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:crond-{hash} RUNNING
 TestMalformedBackenUrlSlave-1:crond-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:frontend-caddy-safe-graceful EXITED
 TestMalformedBackenUrlSlave-1:frontend-nginx-safe-graceful EXITED
-TestMalformedBackenUrlSlave-1:frontend_caddy-on-watch RUNNING
-TestMalformedBackenUrlSlave-1:frontend_nginx-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:monitor-httpd-graceful EXITED
-TestMalformedBackenUrlSlave-1:monitor-httpd-on-watch RUNNING
-TestMalformedBackenUrlSlave-1:trafficserver-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_supervisor_state-CADDY.txt

 TestMasterRequest-0:bootstrap-monitor EXITED
 TestMasterRequest-0:certificate_authority-on-watch RUNNING
-TestMasterRequest-0:crond RUNNING
+TestMasterRequest-0:crond-{hash} RUNNING
+TestMasterRequest-0:monitor-httpd-{hash}-on-watch RUNNING
 TestMasterRequest-0:monitor-httpd-graceful EXITED
-TestMasterRequest-0:monitor-httpd-on-watch RUNNING
 watchdog:watchdog RUNNING
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt

 TestMasterRequestDomain-0:bootstrap-monitor EXITED
 TestMasterRequestDomain-0:certificate_authority-on-watch RUNNING
-TestMasterRequestDomain-0:crond RUNNING
+TestMasterRequestDomain-0:crond-{hash} RUNNING
+TestMasterRequestDomain-0:monitor-httpd-{hash}-on-watch RUNNING
 TestMasterRequestDomain-0:monitor-httpd-graceful EXITED
-TestMasterRequestDomain-0:monitor-httpd-on-watch RUNNING
 watchdog:watchdog RUNNING
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_supervisor_state-CADDY.txt

 TestQuicEnabled-0:bootstrap-monitor EXITED
 TestQuicEnabled-0:certificate_authority-on-watch RUNNING
-TestQuicEnabled-0:crond RUNNING
+TestQuicEnabled-0:crond-{hash} RUNNING
+TestQuicEnabled-0:monitor-httpd-{hash}-on-watch RUNNING
 TestQuicEnabled-0:monitor-httpd-graceful EXITED
-TestQuicEnabled-0:monitor-httpd-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-11080-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-11443-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestQuicEnabled-1:6tunnel-12443-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-26011-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-26012-on-watch RUNNING
 TestQuicEnabled-1:bootstrap-monitor EXITED
-TestQuicEnabled-1:certificate_authority-on-watch RUNNING
+TestQuicEnabled-1:certificate_authority-{hash}-on-watch RUNNING
+TestQuicEnabled-1:crond-{hash} RUNNING
 TestQuicEnabled-1:crond-on-watch RUNNING
 TestQuicEnabled-1:frontend-caddy-safe-graceful EXITED
 TestQuicEnabled-1:frontend-nginx-safe-graceful EXITED
-TestQuicEnabled-1:frontend_caddy-on-watch RUNNING
-TestQuicEnabled-1:frontend_nginx-on-watch RUNNING
+TestQuicEnabled-1:frontend_caddy-{hash}-on-watch RUNNING
+TestQuicEnabled-1:frontend_nginx-{hash}-on-watch RUNNING
+TestQuicEnabled-1:monitor-httpd-{hash}-on-watch RUNNING
 TestQuicEnabled-1:monitor-httpd-graceful EXITED
-TestQuicEnabled-1:monitor-httpd-on-watch RUNNING
-TestQuicEnabled-1:trafficserver-on-watch RUNNING
+TestQuicEnabled-1:trafficserver-{hash}-on-watch RUNNING
 TestQuicEnabled-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_supervisor_state-CADDY.txt

 TestRe6stVerificationUrlDefaultSlave-0:bootstrap-monitor EXITED
 TestRe6stVerificationUrlDefaultSlave-0:certificate_authority-on-watch RUNNING
-TestRe6stVerificationUrlDefaultSlave-0:crond RUNNING
+TestRe6stVerificationUrlDefaultSlave-0:crond-{hash} RUNNING
+TestRe6stVerificationUrlDefaultSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-0:monitor-httpd-graceful EXITED
-TestRe6stVerificationUrlDefaultSlave-0:monitor-httpd-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-11080-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-11443-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestRe6stVerificationUrlDefaultSlave-1:6tunnel-12443-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-26011-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-26012-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:bootstrap-monitor EXITED
-TestRe6stVerificationUrlDefaultSlave-1:certificate_authority-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:crond-{hash} RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:crond-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:frontend-caddy-safe-graceful EXITED
 TestRe6stVerificationUrlDefaultSlave-1:frontend-nginx-safe-graceful EXITED
-TestRe6stVerificationUrlDefaultSlave-1:frontend_caddy-on-watch RUNNING
-TestRe6stVerificationUrlDefaultSlave-1:frontend_nginx-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:monitor-httpd-graceful EXITED
-TestRe6stVerificationUrlDefaultSlave-1:monitor-httpd-on-watch RUNNING
-TestRe6stVerificationUrlDefaultSlave-1:trafficserver-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_supervisor_state-CADDY.txt

 TestRe6stVerificationUrlSlave-0:bootstrap-monitor EXITED
 TestRe6stVerificationUrlSlave-0:certificate_authority-on-watch RUNNING
-TestRe6stVerificationUrlSlave-0:crond RUNNING
+TestRe6stVerificationUrlSlave-0:crond-{hash} RUNNING
+TestRe6stVerificationUrlSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlSlave-0:monitor-httpd-graceful EXITED
-TestRe6stVerificationUrlSlave-0:monitor-httpd-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-11080-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-11443-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestRe6stVerificationUrlSlave-1:6tunnel-12443-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-26011-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-26012-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:bootstrap-monitor EXITED
-TestRe6stVerificationUrlSlave-1:certificate_authority-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:crond-{hash} RUNNING
 TestRe6stVerificationUrlSlave-1:crond-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:frontend-caddy-safe-graceful EXITED
 TestRe6stVerificationUrlSlave-1:frontend-nginx-safe-graceful EXITED
-TestRe6stVerificationUrlSlave-1:frontend_caddy-on-watch RUNNING
-TestRe6stVerificationUrlSlave-1:frontend_nginx-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:monitor-httpd-graceful EXITED
-TestRe6stVerificationUrlSlave-1:monitor-httpd-on-watch RUNNING
-TestRe6stVerificationUrlSlave-1:trafficserver-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_supervisor_state-CADDY.txt

 TestReplicateSlave-0:bootstrap-monitor EXITED
 TestReplicateSlave-0:certificate_authority-on-watch RUNNING
-TestReplicateSlave-0:crond RUNNING
+TestReplicateSlave-0:crond-{hash} RUNNING
+TestReplicateSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestReplicateSlave-0:monitor-httpd-graceful EXITED
-TestReplicateSlave-0:monitor-httpd-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-11080-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-11443-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,15 +10,16 @@ TestReplicateSlave-1:6tunnel-12443-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-26011-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-26012-on-watch RUNNING
 TestReplicateSlave-1:bootstrap-monitor EXITED
-TestReplicateSlave-1:certificate_authority-on-watch RUNNING
+TestReplicateSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestReplicateSlave-1:crond-{hash} RUNNING
 TestReplicateSlave-1:crond-on-watch RUNNING
 TestReplicateSlave-1:frontend-caddy-safe-graceful EXITED
 TestReplicateSlave-1:frontend-nginx-safe-graceful EXITED
-TestReplicateSlave-1:frontend_caddy-on-watch RUNNING
-TestReplicateSlave-1:frontend_nginx-on-watch RUNNING
+TestReplicateSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestReplicateSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestReplicateSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestReplicateSlave-1:monitor-httpd-graceful EXITED
-TestReplicateSlave-1:monitor-httpd-on-watch RUNNING
-TestReplicateSlave-1:trafficserver-on-watch RUNNING
+TestReplicateSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestReplicateSlave-1:trafficserver-reload EXITED
 TestReplicateSlave-2:6tunnel-11080-on-watch STOPPED
 TestReplicateSlave-2:6tunnel-11443-on-watch STOPPED
@@ -27,14 +28,15 @@ TestReplicateSlave-2:6tunnel-12443-on-watch STOPPED
 TestReplicateSlave-2:6tunnel-26011-on-watch STOPPED
 TestReplicateSlave-2:6tunnel-26012-on-watch STOPPED
 TestReplicateSlave-2:bootstrap-monitor STOPPED
-TestReplicateSlave-2:certificate_authority-on-watch STOPPED
+TestReplicateSlave-2:certificate_authority-{hash}-on-watch STOPPED
+TestReplicateSlave-2:crond-{hash} STOPPED
 TestReplicateSlave-2:crond-on-watch STOPPED
 TestReplicateSlave-2:frontend-caddy-safe-graceful STOPPED
 TestReplicateSlave-2:frontend-nginx-safe-graceful STOPPED
-TestReplicateSlave-2:frontend_caddy-on-watch STOPPED
-TestReplicateSlave-2:frontend_nginx-on-watch STOPPED
+TestReplicateSlave-2:frontend_caddy-{hash}-on-watch STOPPED
+TestReplicateSlave-2:frontend_nginx-{hash}-on-watch STOPPED
+TestReplicateSlave-2:monitor-httpd-{hash}-on-watch STOPPED
 TestReplicateSlave-2:monitor-httpd-graceful STOPPED
-TestReplicateSlave-2:monitor-httpd-on-watch STOPPED
-TestReplicateSlave-2:trafficserver-on-watch STOPPED
+TestReplicateSlave-2:trafficserver-{hash}-on-watch STOPPED
 TestReplicateSlave-2:trafficserver-reload STOPPED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -9,6 +9,8 @@ TestSlave-1/var/log/httpd/_custom_domain_access_log
 TestSlave-1/var/log/httpd/_custom_domain_error_log
 TestSlave-1/var/log/httpd/_custom_domain_ssl_crt_ssl_key_access_log
 TestSlave-1/var/log/httpd/_custom_domain_ssl_crt_ssl_key_error_log
+TestSlave-1/var/log/httpd/_custom_domain_wildcard_access_log
+TestSlave-1/var/log/httpd/_custom_domain_wildcard_error_log
 TestSlave-1/var/log/httpd/_disabled-cookie-list_access_log
 TestSlave-1/var/log/httpd/_disabled-cookie-list_error_log
 TestSlave-1/var/log/httpd/_empty_access_log
@@ -37,7 +39,13 @@ TestSlave-1/var/log/httpd/_prefer-gzip-encoding-to-backend_access_log
 TestSlave-1/var/log/httpd/_prefer-gzip-encoding-to-backend_error_log
 TestSlave-1/var/log/httpd/_re6st-optimal-test_access_log
 TestSlave-1/var/log/httpd/_re6st-optimal-test_error_log
+TestSlave-1/var/log/httpd/_server-alias-duplicated_access_log
+TestSlave-1/var/log/httpd/_server-alias-duplicated_error_log
+TestSlave-1/var/log/httpd/_server-alias-wildcard_access_log
+TestSlave-1/var/log/httpd/_server-alias-wildcard_error_log
 TestSlave-1/var/log/httpd/_server-alias_access_log
+TestSlave-1/var/log/httpd/_server-alias_custom_domain-duplicated_access_log
+TestSlave-1/var/log/httpd/_server-alias_custom_domain-duplicated_error_log
 TestSlave-1/var/log/httpd/_server-alias_error_log
 TestSlave-1/var/log/httpd/_ssl-proxy-verify-unverified_access_log
 TestSlave-1/var/log/httpd/_ssl-proxy-verify-unverified_error_log

software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-CADDY.txt

@@ -6,6 +6,8 @@ TestSlave-1/etc/monitor-promise/check-_custom_domain-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_custom_domain-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_custom_domain_ssl_crt_ssl_key-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_custom_domain_ssl_crt_ssl_key-error-log-last-hour
+TestSlave-1/etc/monitor-promise/check-_custom_domain_wildcard-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_custom_domain_wildcard-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_disabled-cookie-list-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_disabled-cookie-list-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_empty-error-log-last-day
@@ -37,8 +39,14 @@ TestSlave-1/etc/monitor-promise/check-_prefer-gzip-encoding-to-backend-error-log
 TestSlave-1/etc/monitor-promise/check-_re6st-optimal-test-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_re6st-optimal-test-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_re6st-optimal-test-re6st-optimal-test
+TestSlave-1/etc/monitor-promise/check-_server-alias-duplicated-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_server-alias-duplicated-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_server-alias-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_server-alias-error-log-last-hour
+TestSlave-1/etc/monitor-promise/check-_server-alias-wildcard-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_server-alias-wildcard-error-log-last-hour
+TestSlave-1/etc/monitor-promise/check-_server-alias_custom_domain-duplicated-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_server-alias_custom_domain-duplicated-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify-unverified-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify-unverified-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify_ssl_proxy_ca_crt-error-log-last-day

software/caddy-frontend/test/test_data/test.TestSlave.test_supervisor_state-CADDY.txt

 TestSlave-0:bootstrap-monitor EXITED
 TestSlave-0:certificate_authority-on-watch RUNNING
-TestSlave-0:crond RUNNING
+TestSlave-0:crond-{hash} RUNNING
+TestSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestSlave-0:monitor-httpd-graceful EXITED
-TestSlave-0:monitor-httpd-on-watch RUNNING
 TestSlave-1:6tunnel-11080-on-watch RUNNING
 TestSlave-1:6tunnel-11443-on-watch RUNNING
 TestSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestSlave-1:6tunnel-12443-on-watch RUNNING
 TestSlave-1:6tunnel-26011-on-watch RUNNING
 TestSlave-1:6tunnel-26012-on-watch RUNNING
 TestSlave-1:bootstrap-monitor EXITED
-TestSlave-1:certificate_authority-on-watch RUNNING
+TestSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestSlave-1:crond-{hash} RUNNING
 TestSlave-1:crond-on-watch RUNNING
 TestSlave-1:frontend-caddy-safe-graceful EXITED
 TestSlave-1:frontend-nginx-safe-graceful EXITED
-TestSlave-1:frontend_caddy-on-watch RUNNING
-TestSlave-1:frontend_nginx-on-watch RUNNING
+TestSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestSlave-1:monitor-httpd-graceful EXITED
-TestSlave-1:monitor-httpd-on-watch RUNNING
-TestSlave-1:trafficserver-on-watch RUNNING
+TestSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestSlaveBadParameters.test_supervisor_state-CADDY.txt

 TestSlaveBadParameters-0:bootstrap-monitor EXITED
 TestSlaveBadParameters-0:certificate_authority-on-watch RUNNING
-TestSlaveBadParameters-0:crond RUNNING
+TestSlaveBadParameters-0:crond-{hash} RUNNING
+TestSlaveBadParameters-0:monitor-httpd-{hash}-on-watch RUNNING
 TestSlaveBadParameters-0:monitor-httpd-graceful EXITED
-TestSlaveBadParameters-0:monitor-httpd-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-11080-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-11443-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestSlaveBadParameters-1:6tunnel-12443-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-26011-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-26012-on-watch RUNNING
 TestSlaveBadParameters-1:bootstrap-monitor EXITED
-TestSlaveBadParameters-1:certificate_authority-on-watch RUNNING
+TestSlaveBadParameters-1:certificate_authority-{hash}-on-watch RUNNING
+TestSlaveBadParameters-1:crond-{hash} RUNNING
 TestSlaveBadParameters-1:crond-on-watch RUNNING
 TestSlaveBadParameters-1:frontend-caddy-safe-graceful EXITED
 TestSlaveBadParameters-1:frontend-nginx-safe-graceful EXITED
-TestSlaveBadParameters-1:frontend_caddy-on-watch RUNNING
-TestSlaveBadParameters-1:frontend_nginx-on-watch RUNNING
+TestSlaveBadParameters-1:frontend_caddy-{hash}-on-watch RUNNING
+TestSlaveBadParameters-1:frontend_nginx-{hash}-on-watch RUNNING
+TestSlaveBadParameters-1:monitor-httpd-{hash}-on-watch RUNNING
 TestSlaveBadParameters-1:monitor-httpd-graceful EXITED
-TestSlaveBadParameters-1:monitor-httpd-on-watch RUNNING
-TestSlaveBadParameters-1:trafficserver-on-watch RUNNING
+TestSlaveBadParameters-1:trafficserver-{hash}-on-watch RUNNING
 TestSlaveBadParameters-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/erp5/test/README.md

+Tests for ERP5 software release

software/erp5/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.erp5'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' ERP5 software releae",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'supervisor',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'psutil',
+        'requests',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/erp5/test/test/__init__.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import unittest
+import logging
+
+if os.environ.get('DEBUG'):
+  raise ValueError("Don't set DEBUG - it breaks postfix compilation - set SLAPOS_TEST_DEBUG instead.")
+
+debug_mode = os.environ.get('SLAPOS_TEST_DEBUG')
+# for development: debugging logs and install Ctrl+C handler
+if debug_mode:
+  logging.basicConfig(level=logging.DEBUG)
+  unittest.installHandler()

software/erp5/test/test/test_erp5.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import json
+import glob
+import urlparse
+import logging
+import time
+
+import requests
+
+from utils import SlapOSInstanceTestCase
+
+
+class ERP5TestCase(SlapOSInstanceTestCase):
+  """Test the remote driver on a minimal web server.
+  """
+  logger = logging.getLogger(__name__)
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')), )
+
+  def getRootPartitionConnectionParameterDict(self):
+    """Return the output paramters from the root partition"""
+    return json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])
+
+
+class TestPublishedURLIsReachableMixin(object):
+  """Mixin that checks that default page of ERP5 is reachable.
+  """
+  def _checkERP5IsReachable(self, url):
+    # What happens is that instanciation just create the services, but does not
+    # wait for ERP5 to be initialized. When this test run ERP5 instance is
+    # instanciated, but zope is still busy creating the site and haproxy replies
+    # with 503 Service Unavailable.
+    # If we can move the "create site" in slapos node instance, then this retry loop
+    # would not be necessary.
+    for i in range(1, 20):
+      r = requests.get(url, verify=False) # XXX can we get CA from caucase already ?
+      if r.status_code == requests.codes.service_unavailable:
+        delay = i * 2
+        self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
+        time.sleep(delay)
+        continue
+      if r.status_code != requests.codes.ok:
+        r.raise_for_status()
+      break
+
+    self.assertIn("ERP5", r.text)
+
+  def test_published_family_default_v6_is_reachable(self):
+    """Tests the IPv6 URL published by the root partition is reachable.
+    """
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    self._checkERP5IsReachable(
+      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
+
+  def test_published_family_default_v4_is_reachable(self):
+    """Tests the IPv4 URL published by the root partition is reachable.
+    """
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    self._checkERP5IsReachable(
+      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
+
+
+class TestDefaultParameters(ERP5TestCase, TestPublishedURLIsReachableMixin):
+  """Test ERP5 can be instanciated with no parameters
+  """
+  __partition_reference__ = 'defp'
+
+
+class TestDisableTestRunner(ERP5TestCase, TestPublishedURLIsReachableMixin):
+  """Test ERP5 can be instanciated without test runner.
+  """
+  __partition_reference__ = 'distr'
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'test-runner': {'enabled': False}})}
+
+  def test_no_runUnitTestScript(self):
+    """No runUnitTest script should be generated in any partition.
+    """
+    # self.computer_partition_root_path is the path of root partition.
+    # we want to assert that no scripts exist in any partition.
+    bin_programs = [os.path.basename(path) for path in
+      glob.glob("{}/../*/bin/*".format(self.computer_partition_root_path))]
+
+    self.assertTrue(bin_programs) # just to check the glob was correct.
+    self.assertNotIn('runUnitTest', bin_programs)
+    self.assertNotIn('runTestSuite', bin_programs)

software/erp5/test/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+import StringIO
+import xmlrpclib
+
+import supervisor.xmlrpc
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+
+
+# Utility functions
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+# TODO:
+#  - allow requesting multiple instances ?
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  """Install one slapos instance.
+
+  This test case install software(s) and request one instance during `setUpClass`
+  and destroy the instance during `tearDownClass`.
+
+  Software Release URL, Instance Software Type and Instance Parameters can be defined
+  on the class.
+
+  All tests from the test class will run with the same instance.
+
+  The following class attributes are available:
+
+    * `computer_partition`:  the `slapos.core.XXX` computer partition instance.
+
+    * `computer_partition_root_path`: the path of the instance root directory,
+
+  A note about paths:
+    SlapOS itself and some services running in SlapOS uses unix sockets and (sometimes very)
+    deep path, which does not play very well together. To workaround this, users can
+    set SLAPOS_TEST_WORKING_DIR enivonment variable to the path of a short enough directory
+    and local slapos will be in this directory.
+    The partitions references will be named after the unittest class name, which can also lead
+    to long paths. For this, unit test classes can define a __partition_reference__ attribute
+    which will be used as partition reference. The trick is then to use a shorter
+    __partition_reference__
+  See https://lab.nexedi.com/kirr/slapns for the solution to all these problems.
+
+  """
+  # Methods to be defined by subclasses.
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    """Return software type for instance, default "default"
+
+    To be defined by subclasses if they need to request instance with specific
+    software type.
+    """
+    return "default"
+
+  # Utility methods.
+  def getSupervisorRPCServer(self):
+    """Returns a XML-RPC connection to the supervisor used by slapos node
+
+    Refer to http://supervisord.org/api.html for details of available methods.
+    """
+    # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
+    return xmlrpclib.ServerProxy(
+       'http://slapos-supervisor',
+       transport=supervisor.xmlrpc.SupervisorTransport(
+           None,
+           None,
+           # XXX hardcoded socket path
+           serverurl="unix://{working_directory}/inst/supervisord.socket".format(
+             **self.config)))
+
+  # Unittest methods
+  @classmethod
+  def setUpClass(cls):
+    """Setup the class, build software and request an instance.
+    
+    If you have to override this method, do not forget to call this method on
+    parent class.
+    """
+    try:
+      cls.setUpWorkingDirectory()
+      cls.setUpConfig()
+      cls.setUpSlapOSController()
+
+      cls.runSoftwareRelease()
+      # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
+      # cls.supplySoftwares()
+      # cls.installSoftwares()
+
+      cls.runComputerPartition()
+      # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
+      # cls.requestInstances()
+      # cls.createInstances()
+      # cls.requestInstances()
+
+    except BaseException:
+      cls.stopSlapOSProcesses()
+      cls.setUp = lambda self: self.fail('Setup Class failed.')
+      raise
+
+  @classmethod
+  def tearDownClass(cls):
+    """Tear down class, stop the processes and destroy instance.
+    """
+    cls.stopSlapOSProcesses()
+
+  # Implementation
+  @classmethod
+  def stopSlapOSProcesses(cls):
+    if hasattr(cls, '_process_manager'):
+      cls._process_manager.killPreviousRun()
+
+  @classmethod
+  def setUpWorkingDirectory(cls):
+    """Initialise the directories"""
+    cls.working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    # Supervisord socket name contains the pid number, which is why we add
+    # .xxxxxxx in this check.
+    if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
+      raise RuntimeError('working directory ( {} ) is too deep, try setting '
+              'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
+
+    if not os.path.exists(cls.working_directory):
+      os.mkdir(cls.working_directory)
+
+  @classmethod
+  def setUpConfig(cls):
+    """Create slapos configuration"""
+    cls.config = {
+      "working_directory": cls.working_directory,
+      "slapos_directory": cls.working_directory,
+      "log_directory": cls.working_directory,
+      "computer_id": 'slapos.test',  # XXX
+      'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
+      'partition_reference': getattr(cls, '__partition_reference__', cls.__name__),
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+    }
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['GLOBAL_IPV6']
+
+    cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
+    cls.config['ipv6_address'] = ipv6_address
+    cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
+      **cls.config)
+
+  @classmethod
+  def setUpSlapOSController(cls):
+    """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
+
+    This is equivalent to:
+
+    slapos proxy start
+    for sr in getSoftwareURLList; do
+      slapos supply $SR $COMP
+    done
+    """ 
+    cls._process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    cls.slapos_controler = SlapOSControler(
+        cls.working_directory,
+        cls.config
+    )
+
+    slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    cls.software_url_list = cls.getSoftwareURLList()
+    cls.slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=cls._process_manager,
+        reset_software=False,
+        software_path_list=cls.software_url_list)
+
+    # XXX we should check *earlier* if that pidfile exist and if supervisord
+    # process still running, because if developer started supervisord (or bugs?)
+    # then another supervisord will start and starting services a second time
+    # will fail.
+    cls._process_manager.supervisord_pid_file = os.path.join(
+      cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+  @classmethod
+  def runSoftwareRelease(cls):
+    """Run all the software releases that were supplied before.
+
+    This is the equivalent of `slapos node software`.
+
+    The tests will be marked file if software building fail.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    try:
+      cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
+        cls.config, environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.software_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+
+  @classmethod
+  def runComputerPartition(cls):
+    """Instanciate the software.
+
+    This is the equivalent of doing:
+
+    slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
+    slapos node instance
+
+    and return the slapos request instance parameters.
+
+    This can be called by tests to simulate re-request with different parameters.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    if cls.getInstanceSoftwareType() != 'default':
+      raise NotImplementedError
+
+    instance_parameter_dict = cls.getInstanceParameterDict()
+    try:
+      cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
+        cls.config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.instance_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+    # FIXME: similar to test node, only one (root) partition is really
+    #        supported for now.
+    computer_partition_list = []
+    for i in range(len(cls.software_url_list)):
+      computer_partition_list.append(
+          cls.slapos_controler.slap.registerOpenOrder().request(
+            cls.software_url_list[i],
+            # This is how testnode's SlapOSControler name created partitions
+            partition_reference='testing partition {i}'.format(
+              i=i, **cls.config),
+            partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParameterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        cls.config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+

software/erp5testnode/software.cfg

@@ -74,34 +74,14 @@ waitress = 1.1.0
 z3c.etestbrowser = 2.0.1
 zope.testbrowser = 5.2.4
 
-# Required by:
-# slapos.toolbox==0.81
-GitPython = 2.1.11
-
 # Required by:
 # zope.testbrowser==5.2.4
 WSGIProxy2 = 0.4.4
 
-# Required by:
-# slapos.toolbox==0.81
-atomize = 0.2.0
-
 # Required by:
 # WebTest==2.0.30
 beautifulsoup4 = 4.6.3
 
-# Required by:
-# slapos.toolbox==0.81
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.81
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.81
-passlib = 1.7.1
-
 # Required by:
 # zope.testbrowser==5.2.4
 zope.cachedescriptors = 4.3.1

software/erp5testnode/testsuite/erp5/README.md

+# ERP5 test
+
+This software release is simply to run the test suite from `../erp5/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-ERP5-TEST` in test result module.
+

software/erp5testnode/testsuite/erp5/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = 6d126917e17c00ee012c4cf0a52429d1
+

software/erp5testnode/testsuite/erp5/instance.cfg.in

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+working-dir = $${buildout:directory}/tmp/
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/erp5/test
+
+
+# XXX we need "standard" path entries to compile softwares inside test
+# XXX can't we just inherit $PATH ?
+environment =
+  PATH=${buildout:bin-directory}:/bin/:/usr/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=$${create-directory:working-dir}

software/erp5testnode/testsuite/erp5/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.erp5-setup]
+<= setup-develop-egg
+egg = slapos.test.erp5
+setup = ${slapos-repository:location}/software/erp5/test/
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos.test.erp5-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+erp5.util = 0.4.56
+slapos.recipe.template = 4.3
+

software/erp5testnode/testsuite/kvm/README.md

+# kvm test
+
+This software release is simply to run the test suite from `../kvm/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-KVM-TEST` in test result module.
+

software/erp5testnode/testsuite/kvm/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = 8109649ec9ba66de3ac67b742c28c349
+

software/erp5testnode/testsuite/kvm/instance.cfg.in

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+working-dir = $${buildout:directory}/tmp/
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/kvm/test
+
+
+# XXX slapos.cookbook:wrapper does not allow extending env, so we add some default $PATH entries ( not sure they are needed )
+environment =
+  PATH=${buildout:bin-directory}:/usr/bin/:/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=$${create-directory:working-dir}

software/erp5testnode/testsuite/kvm/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.kvm-setup]
+<= setup-develop-egg
+egg = slapos.test.kvm
+setup = ${slapos-repository:location}/software/kvm/test/
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos.test.kvm-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+# clear the version of tested eggs, to make sure we installed the developped ones
+slapos.test.kvm =
+erp5.util = 0.4.56
+
+pyasn1 = 0.4.2
+slapos.recipe.template = 4.3
+pysftp = 0.2.9

software/erp5testnode/testsuite/powerdns/README.md

+# powerdns test
+
+This software release is simply to run the test suite from `../powerdns/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-POWERDNS-TEST` in test result module.
+

software/erp5testnode/testsuite/powerdns/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = af932783693bf8a7fec32d00a8e07ed2
+

software/erp5testnode/testsuite/powerdns/instance.cfg.in

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+working-dir = $${buildout:directory}/tmp/
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/powerdns/test
+
+
+# XXX slapos.cookbook:wrapper does not allow extending env, so we add some default $PATH entries ( not sure they are needed )
+environment =
+  PATH=${buildout:bin-directory}:/usr/bin/:/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=$${create-directory:working-dir}

software/erp5testnode/testsuite/powerdns/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.powerdns-setup]
+<= setup-develop-egg
+egg = slapos.test.powerdns
+setup = ${slapos-repository:location}/software/powerdns/test/
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos.test.powerdns-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+# clear the version of tested eggs, to make sure we installed the developped ones
+slapos.test.powerdns =
+erp5.util = 0.4.56
+
+pyasn1 = 0.4.2
+slapos.recipe.template = 4.3
+pysftp = 0.2.9

software/erp5testnode/testsuite/seleniumrunner/README.md

+# Selenium Server test
+
+This software release is simply to run the test suite from `../seleniumrunner/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-SELENIUMRUNNER-TEST` in test result module.
+

software/erp5testnode/testsuite/seleniumrunner/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = 7e75c9eccb580f278da1784941363432
+

software/erp5testnode/testsuite/seleniumrunner/instance.cfg.in

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+working-dir = $${buildout:directory}/tmp/
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/seleniumrunner/test
+
+
+# XXX we need "standard" path entries to compile softwares inside test
+# XXX can't we just inherit $PATH ?
+environment =
+  PATH=${buildout:bin-directory}:/bin/:/usr/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=$${create-directory:working-dir}

software/erp5testnode/testsuite/seleniumrunner/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../component/pillow/buildout.cfg
+  ../../../../component/python-pynacl/buildout.cfg
+  ../../../../component/bcrypt/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.seleniumrunner-setup]
+<= setup-develop-egg
+egg = slapos.test.seleniumrunner
+setup = ${slapos-repository:location}/software/seleniumrunner/test/
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${pillow-python:egg}
+  ${python-pynacl:egg}
+  ${bcrypt:egg}
+  ${slapos.test.seleniumrunner-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+erp5.util = 0.4.56
+slapos.recipe.template = 4.3
+
+image = 1.5.25
+Pillow = 5.3.0
+selenium = 3.14.1
+urllib3 = 1.23
+# Django 1.11 is python 2 compatible
+Django = 1.11
+paramiko = 2.4.2
+pyasn1 = 0.4.2
+PyNaCl = 1.3.0
+bcrypt = 3.1.4
\ No newline at end of file

software/erp5testnode/testsuite/slaprunner/README.md

+# slaprunner test
+
+This software release is simply to run the test suite from `../slaprunner/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-SLAPRUNNER-TEST` in test result module.
+

software/erp5testnode/testsuite/slaprunner/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = 6fa1420739f58e722631564b08727060
+

software/erp5testnode/testsuite/slaprunner/instance.cfg.in

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+working-dir = $${buildout:directory}/tmp/
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/slaprunner/test
+
+
+# XXX slapos.cookbook:wrapper does not allow extending env, so we add some default $PATH entries ( not sure they are needed )
+environment =
+  PATH=${buildout:bin-directory}:/usr/bin/:/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=$${create-directory:working-dir}

software/erp5testnode/testsuite/slaprunner/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.slaprunner-setup]
+<= setup-develop-egg
+egg = slapos.test.slaprunner
+setup = ${slapos-repository:location}/software/slaprunner/test/
+
+[erp5.util-setup]
+<= setup-develop-egg
+egg = erp5.util[testnode]
+setup = ${erp5.util-repository:location}
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos.test.slaprunner-setup:egg}
+  ${erp5.util-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+# XXX we need an unreleased ( 0.4.51 ) version of erp5.util runTestSuite
+# later we can stop fetching it from git and just use egg
+[erp5.util-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/erp5.git
+revision = 69013fa0fb67501089c776ab5e75d7bbf2e0e3bc
+
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+# clear the version of tested eggs, to make sure we installed the developped ones
+slapos.test.slaprunner =
+erp5.util =
+#erp5.util = 0.4.51
+
+pyasn1 = 0.4.2
+slapos.recipe.template = 4.3
+pysftp = 0.2.9

software/gitlab/software.cfg

@@ -191,9 +191,9 @@ environment =
 [gowork]
 golang  = ${golang19:location}
 install = 
-  lab.nexedi.com/kirr/git-backup \
-  gitlab.com/gitlab-org/gitlab-workhorse \
-  gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat \
+  lab.nexedi.com/kirr/git-backup
+  gitlab.com/gitlab-org/gitlab-workhorse
+  gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat
   gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata
 
 cpkgpath =

software/html5as/software.cfg

@@ -73,22 +73,6 @@ gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.40.4
 smmap = 0.8.2
 plone.recipe.command = 1.1
 
-# Required by:
-# slapos.toolbox==0.40.2
-GitPython = 2.0.8
-
-# Required by:
-# slapos.toolbox==0.40.2
-atomize = 0.1.1
-
-# Required by:
-# slapos.toolbox==0.40.2
-feedparser = 5.1.3
-
-# Required by:
-# slapos.toolbox==0.40.2
-paramiko = 2.0.1

software/htmlvalidatorserver/software.cfg

@@ -59,30 +59,3 @@ apache-libcloud = 2.2.1
 gitdb2 = 2.0.3
 smmap2 = 2.0.3
 
-# Required by:
-# slapos.toolbox==0.73
-GitPython = 2.1.8
-
-# Required by:
-# slapos.toolbox==0.73
-atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.73
-dnspython = 1.15.0
-
-# Required by:
-# slapos.toolbox==0.73
-erp5.util = 0.4.51
-
-# Required by:
-# slapos.toolbox==0.73
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.73
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.73
-passlib = 1.7.1

software/jstestnode/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 74d4c0a5105fa3d74af894fa1afa9ba2
+md5sum = 2a79bb6c4f593d7c4c7f4e0de97d9803
 
 [template-nginx-service]
 filename = template-nginx-service.sh.in
@@ -27,4 +27,4 @@ md5sum = 9f22db89a2679534aa8fd37dbca86782
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = d26572727ef1679a8a9e51b021ece524
+md5sum = bd9ff3543f0dfaf2702624e3ed74d334

software/jstestnode/instance-jstestnode-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Parameters to instantiate JSTestNode",
+  "additionalProperties": false,
+  "properties": {
+    "test-suite": {
+      "description": "The test suite to run",
+      "type": "string",
+      "enum": [
+        "jio",
+        "renderjs"
+      ]
+    },
+    "remote-access-url": {
+      "description": "URL that controlled browser must access to run tests",
+      "type": "string",
+      "format": "uri",
+      "default": "(the web server started by this instance)",
+      "example": "https://softinst1234.host.vifib.net/"
+    },
+    "oneOf": [
+      {
+        "title": "selenium server",
+        "description": "Configuration for running tests on selenium server",
+        "properties": {
+          "target": {
+            "description": "Target system",
+            "const": "selenium-server"
+          },
+          "server-url": {
+            "description": "URL of the selenium server",
+            "type": "string"
+          },
+          "verify-server-certificate": {
+            "description": "Verify the SSL/TLS Certificats of the selenium server when using HTTPS",
+            "type": "boolean",
+            "default": true
+          },
+          "server-ca-certificate": {
+            "description": "PEM encoded bundle of CA Certificates to verify the SSL/TLS Certificate of the selenium server when using HTTPS",
+            "type": "string",
+            "default": "root certificates from http://certifi.io/en/latest/"
+          },
+          "desired-capabilities": {
+            "description": "Desired browser capabilities",
+            "type": "object",
+            "properties": {
+              "browserName": {
+                "description": "Name of the browser being used, for example firefox, chrome",
+                "type": "string",
+                "required": true
+              },
+              "version": {
+                "description": "The browser version",
+                "type": "string"
+              }
+            },
+            "additionalProperties": true
+          }
+        }
+      },
+      {
+        "title": "firefox",
+        "description": "Configuration for running tests on local firefox process",
+        "properties": {
+          "target": {
+            "description": "Target system",
+            "const": "firefox",
+            "default": "firefox"
+          }
+        }
+      },
+      {
+        "title": "node",
+        "description": "Configuration for running tests on local nodejs",
+        "properties": {
+          "target": {
+            "description": "Target system",
+            "const": "node"
+          }
+        }
+      }
+    ]
+  }
+}

software/jstestnode/instance-jstestnode-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by JSTestNode instantiation",
+  "additionalProperties": false,
+  "type": "object",
+  "nginx": {
+    "description": "Nginx web server serving html test runner",
+    "type": "string"
+  }
+}

software/jstestnode/instance.cfg.in

@@ -2,12 +2,13 @@
 parts =
   nginx-service
   runTestSuite-instance
+  publish
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 [publish]
-recipe = slapos.cookbook:publish
+recipe = slapos.cookbook:publish.serialised
 nginx = http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/
 
 [directory]
@@ -28,9 +29,8 @@ ssl = $${:etc}/ssl
 framebuffer = $${:srv}/framebuffer
 
 #################################
-# Firefox
+# Test runner
 #################################
-
 [runTestSuite-instance]
 recipe = slapos.recipe.template
 url = ${template-runTestSuite:output}
@@ -38,6 +38,15 @@ output = $${directory:bin}/runTestSuite
 buildout-directory = $${buildout:directory}
 mode = 0700
 
+[runTestSuite-config-file]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:etc}/$${:_buildout_section_name_}.json
+template = inline:
+  $${instance-parameter:configuration._}
+
+#################################
+# Xvfb / Firefox
+#################################
 [xvfb-instance]
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}

software/jstestnode/runTestSuite.in

@@ -12,13 +12,17 @@ from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.remote.remote_connection import RemoteConnection
 from subprocess import check_output
+import urllib3
+import certifi
 import json
 
 os.environ['XORG_LOCK_DIR'] = '$${xvfb-instance:lock-dir}'
 os.environ['DISPLAY'] = '$${xvfb-instance:display}'
 
 BASE_URL = 'http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/'
+ETC_DIRECTORY = '$${directory:etc}'
 
 def main():
   parser = argparse.ArgumentParser(description='Run a test suite.')
@@ -33,40 +37,11 @@ def main():
                       help='The Url of Master controling many suites')
   parser.add_argument('--frontend_url',
                       help='The url of frontend of the test suite')
-  parser.add_argument('--target',
-                      help='Target OS to run tests on',
-                      type=str)
-  parser.add_argument('--target_version',
-                      help='Target OS version to use',
-                      type=str,)
-  parser.add_argument('--target_browser',
-                      help='The desired browser of the target OS to be used. Example: Firefox if target is Android.',
-                      type=str,)
-  parser.add_argument('--target_device',
-                      help='The desired device running the target OS. Example: iPad Simulator, if target is iOS.',
-                      type=str,)
-  parser.add_argument('--appium_server_auth',
-                      help='Combination of user and token to access SauceLabs service. (i.e. user:token)',
-                      type=str)
 
   args = parser.parse_args()
 
-  import json
-  parsed_parameters = json.loads('$${instance-parameter:configuration._}')
-
-
-  if not getattr(args, 'target', None):
-    args.target = parsed_parameters.get('target', 'firefox')
-  if not getattr(args, 'test_suite', None):
-    args.test_suite = parsed_parameters.get('test-suite')
-  if not getattr(args, 'target_version', None):
-    args.target_version = parsed_parameters.get('target-version')
-  if not getattr(args, 'appium_server_auth', None):
-    args.appium_server_auth = parsed_parameters.get('appium-server-auth')
-  if not getattr(args, 'target_browser', None):
-    args.target_browser = parsed_parameters.get('target-browser')
-  if not getattr(args, 'target_device', None):
-    args.target_device = parsed_parameters.get('target-device')
+  parsed_parameters = json.load(
+      open('$${runTestSuite-config-file:rendered}', 'rb'))
 
   is_browser_running = False
   try:
@@ -87,38 +62,8 @@ def main():
     ##########################
     # Run all tests
     ##########################
-
-    is_appium = False
-
-    if args.target == 'firefox':
-      firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX
-      firefox_capabilities['marionette'] = True
-      browser = webdriver.Firefox(capabilities=firefox_capabilities,
-                                  firefox_binary='${firefox-wrapper:location}',
-                                  executable_path='${geckodriver:location}')
-    elif args.target in ['iOS', 'Android']:
-      # parameters for mobile emulators have different names then parameters for
-      # desktop OSes
-      is_appium = True
-      capabilities = {
-        'platformName': args.target,
-        'platformVersion': args.target_version,
-        'deviceName': args.target_device,
-        'browserName': args.target_browser
-
-      }
-    elif 'Windows' in args.target or 'OS X' in args.target:
-      # parameters for mobile emulators have different names then parameters for
-      # desktop OSes
-      is_appium = True
-      capabilities = {
-        'browserName': args.target_browser,
-        'platform': args.target,
-        'version': args.target_version
-      }
-
-
-    if args.target == 'node':
+    target = parsed_parameters.get('target', 'firefox')
+    if target == 'node':
       # Execute NodeJS tests
       result_string = check_output(['${nodejs-output:node}', '${jio-repository.git:location}/test/node.js'],
                                    cwd='${jio-repository.git:location}',
@@ -139,21 +84,41 @@ def main():
 
     else:
       # Execute WebBrowser tests
-      if is_appium:
-        if not args.appium_server_auth:
-          raise RuntimeError('--appium_server_auth is required.')
-        appium_url = "http://%s@ondemand.saucelabs.com/wd/hub" % (args.appium_server_auth)
-        browser = webdriver.Remote(appium_url, capabilities)
+      if target == 'firefox':
+        firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX
+        firefox_capabilities['marionette'] = True
+        browser = webdriver.Firefox(
+            capabilities=firefox_capabilities,
+            firefox_binary='${firefox-wrapper:location}',
+            executable_path='${geckodriver:location}')
+      else:
+        assert target == 'selenium-server', "Unsupported target {}".format(parsed_parameters['target'])
+        # use a remote connection which verifies TLS certificate
+        # workaround for https://github.com/SeleniumHQ/selenium/issues/6534
+        executor = RemoteConnection(parsed_parameters['server-url'], keep_alive=True)
+        cert_reqs = 'CERT_REQUIRED'
+        ca_certs = certifi.where()
+        if not parsed_parameters.get('verify-server-certificate', True):
+          cert_reqs = 'CERT_NONE'
+          ca_certs = None
+        if parsed_parameters.get('server-ca-certificate'):
+          ca_certs = os.path.join(ETC_DIRECTORY, "cacerts.pem")
+          with open(ca_certs, 'w') as f:
+            f.write(parsed_parameters.get('server-ca-certificate'))
+        executor._conn = urllib3.PoolManager(cert_reqs=cert_reqs, ca_certs=ca_certs)
+
+        browser = webdriver.Remote(
+            command_executor=executor,
+            desired_capabilities=parsed_parameters['desired-capabilities'],
+        )
 
       # adjust path for remote test url
-        remote_access_url = parsed_parameters.get('remote-access-url', None)
+      remote_access_url = parsed_parameters.get('remote-access-url')
       if remote_access_url:
         if ('jio' in test_suite):
-            url = os.path.join(remote_access_url, 'jio/test/tests.html')
-          else:
-            url = os.path.join(remote_access_url, 'renderjs/test/')
+          url = '{}/jio/test/tests.html'.format(remote_access_url)
         else:
-          raise ValueError('remote-access-url is not defined in instance parameter')
+          url = '{}/renderjs/test/'.format(remote_access_url)
 
       is_browser_running = True
       agent = browser.execute_script("return navigator.userAgent")

software/jstestnode/software.cfg

@@ -36,6 +36,7 @@ parts =
 recipe = zc.recipe.egg
 eggs =
   erp5.util
+  urllib3[secure]
   selenium
   ${lxml-python:egg}
 interpreter = pythonwitheggs
@@ -116,4 +117,6 @@ output = ${buildout:directory}/runTestSuite.in
 [versions]
 erp5.util = 0.4.51
 slapos.recipe.template = 4.3
-selenium = 3.8.0
+selenium = 3.14.1
+urllib3 = 1.24
+certifi = 2018.10.15
\ No newline at end of file

software/jstestnode/software.cfg.json

+{
+  "name": "JSTestNode",
+  "description": "Simple Test Runner for Javascrip projects",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "request": "instance-jstestnode-input-schema.json",
+      "response": "instance-jstestnode-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/kvm/instance-kvm-cluster-simplified-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Input Parameters",
+  "properties": {
+    "kvm-partition-dict": {
+      "title": "kvm instances definition",
+      "description": "kvm instances definition",
+      "patternProperties": {
+        ".*": {
+          "properties": {
+            "state": {
+              "title": "State of this Virtual Machine",
+              "description": "Define if SlapOS should start or stop this VM.",
+              "type": "string",
+              "default": "started",
+              "enum": [
+                "started",
+                "stopped"
+              ]
+            },
+            "keyboard-layout-language": {
+              "title": "Use keyboard layout language",
+              "description": "Use keyboard layout language (for example fr for French). Can be usefull with VNC display",
+              "type": "string",
+              "enum": [
+                "ar",
+                "da",
+                "de",
+                "de-ch",
+                "en-gb",
+                "en-us",
+                "es",
+                "et",
+                "fi",
+                "fo",
+                "fr",
+                "fr-be",
+                "fr-ca",
+                "fr-ch",
+                "hr",
+                "hu",
+                "is",
+                "it",
+                "ja",
+                "lt",
+                "lv",
+                "mk",
+                "nl",
+                "nl-be",
+                "no",
+                "pl",
+                "pt",
+                "pt-br",
+                "ru",
+                "sl",
+                "sv",
+                "th",
+                "tr"
+              ],
+              "default": "fr"
+            },
+            "nbd-host": {
+              "title": "NBD hostname or IP",
+              "description": "hostname (or IP) of the NBD server containing the boot image.",
+              "type": "string",
+              "format": [
+                "host-name",
+                "ip-address",
+                "ipv6"
+              ],
+              "default": "debian.nbd.vifib.net"
+            },
+            "nbd-port": {
+              "title": "NBD port",
+              "description": "Port of the NBD server containing the boot image.",
+              "type": "integer",
+              "default": 1024,
+              "minimum": 1,
+              "maximum": 65535
+            }
+          },
+          "type": "object"
+        }
+      },
+      "type": "object"
+    }
+  }
+}

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -35,7 +35,14 @@ config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 <= request-common
 software-type = kvm
 name = {{ instance_name }}
+{% if kvm_parameter_dict.get('sticky-computer', '') -%}
+sla-computer_guid = ${slap-connection:computer-id} 
+{% else -%}
 sla-computer_guid = {{ dumps(kvm_parameter_dict.get('computer-guid', '')) }}
+{% endif -%}
+
+sla-network_guid = {{ dumps(kvm_parameter_dict.get('network-guid', '')) }}
+sla-project_guid = {{ dumps(kvm_parameter_dict.get('project-guid', '')) }}
 {% if kvm_parameter_dict.get('state', '') == 'stopped' -%}
 state = stopped
 {% endif -%}
@@ -69,6 +76,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
 {{ setconfig('disk-aio', kvm_parameter_dict.get('disk-aio', '')) }}
 {{ setconfig('cpu-model', kvm_parameter_dict.get('cpu-model', '')) }}
 {{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
+{{ setconfig('disk-device-path', kvm_parameter_dict.get('disk-device-path', '')) }}
 
 {% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
 {{ setconfig('nat-rules', nat_rules_list | join(' ')) }}

software/kvm/instance-kvm.cfg.jinja2

@@ -180,6 +180,7 @@ ipv4 = ${slap-network-information:local-ipv4}
 ipv6 = ${slap-network-information:global-ipv6}
 wrapper-path = ${directory:services}/6tunnel-${:ipv6-port}
 command-line = {{ sixtunnel_executable_location }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 {% if use_nat == 'true' and nat_rule_list -%}
 {%   for port in nat_rule_list.split(' ') -%}
@@ -199,6 +200,7 @@ socket-path = ${kvm-controller-parameter-dict:socket-path}
 wrapper-path = ${directory:services}/kvm
 command-line = ${kvm-run:rendered}
 kvm-controller = ${kvm-controller-wrapper:wrapper-path}
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 
 [kvm-controller-wrapper]
@@ -267,20 +269,32 @@ ssl-cert-path = ${ca-novnc:cert-file}
 
 [websockify-sighandler]
 recipe = slapos.cookbook:signalwrapper
-wrapper-path = ${directory:services}/websockify
+wrapper-path = ${directory:bin}/websockify-sighandler
 wrapped-path = ${novnc-instance:path}
 
+[websockify-sighandler-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${websockify-sighandler:wrapper-path}
+wrapper-path = ${directory:services}/websockify
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
 openssl-binary = {{ openssl_executable_location }}
 ca-dir = ${directory:ca-dir}
 requests-directory = ${cadirectory:requests}
-wrapper = ${directory:services}/certificate_authority
+wrapper = ${directory:bin}/certificate_authority
 ca-private = ${cadirectory:private}
 ca-certs = ${cadirectory:certs}
 ca-newcerts = ${cadirectory:newcerts}
 ca-crl = ${cadirectory:crl}
 
+[certificate-authority-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${certificate-authority:wrapper}
+wrapper-path = ${directory:services}/certificate_authority
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
 requests = ${directory:ca-dir}/requests/
@@ -315,7 +329,13 @@ cron-entries = ${directory:cron-entries}
 crontabs = ${directory:crontabs}
 cronstamps = ${directory:cronstamps}
 catcher = ${cron-simplelogger:wrapper}
-binary = ${directory:services}/crond
+binary = ${directory:bin}/crond_raw
+
+[cron-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${cron:binary}
+wrapper-path = ${directory:services}/crond
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
@@ -356,10 +376,16 @@ recipe = slapos.cookbook:simplehttpserver
 host = ${slap-network-information:local-ipv4}
 port = ${slap-parameter:httpd-port}
 base-path = ${directory:public}
-wrapper = ${directory:services}/http-server
+wrapper = ${directory:bin}/http-server
 log-file = ${directory:log}/httpd.log
 use-hash-url = false
 
+[httpd-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${httpd:wrapper}
+wrapper-path = ${directory:services}/http-server
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
 [httpd-promise]
 recipe = slapos.cookbook:check_port_listening
 path = ${directory:promises}/httpd
@@ -526,7 +552,13 @@ ipv6-port = {{ slapparameter_dict.get('document-port', '') }}
 ipv4-port = 16936
 shell-path = {{ dash_executable_location }}
 6tunnel-path = {{ sixtunnel_executable_location }}
-runner-path = ${directory:services}/6tunnel-cluster
+runner-path = ${directory:bin}/6tunnel-cluster
+
+[tunnel-cluster-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${tunnel-cluster-url:runner-path}
+wrapper-path = ${directory:services}/6tunnel-cluster
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 [ansible-vm-promise]
 recipe = slapos.recipe.template:jinja2
@@ -617,7 +649,7 @@ keyboard-layout-language = fr
 {%   do part_list.append('cluster-url-path') -%}
 {% endif -%}
 {% if enable_http == 'true' %}
-{%   do part_list.extend(['httpd', 'httpd-promise', 'publish-host-config']) -%}
+{%   do part_list.extend(['httpd', 'httpd-service', 'httpd-promise', 'publish-host-config']) -%}
 {% if slapparameter_dict.get('data-to-vm', '') %}
 {%   do part_list.append('vm-data-content') -%}
 {% endif -%}
@@ -630,6 +662,9 @@ keyboard-layout-language = fr
 {% if slapparameter_dict.get('bootstrap-script-url', '') -%}
 {%   do part_list.append('download-bootstrap-script') -%}
 {% endif -%}
+{% if slapparameter_dict.get('document-port', '') -%}
+{%   do part_list.append('tunnel-cluster-service') -%}
+{% endif -%}
 
 {% endif -%}
 
@@ -650,15 +685,18 @@ context =
 [buildout]
 parts =
   certificate-authority
+  certificate-authority-service
   publish-connection-information
   kvm-instance
   kvm-controller-wrapper
   kvm-vnc-promise
   kvm-disk-image-corruption-promise
   websockify-sighandler
+  websockify-sighandler-service
   novnc-promise
   kvm-started-promise
   cron
+  cron-service
   cron-entry-logrotate
   frontend-promise
 # monitor parts

software/kvm/software.cfg

@@ -99,7 +99,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = 0fd548b8cac9278496d9d83dde26d09c
+md5sum = 149df1bc788ce68c86a5fda4872e008e
 download-only = true
 on-update = true
 
@@ -108,7 +108,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = cccabafc001c6df35da95af6bf2ae28e
+md5sum = fa40e4729c27236c655f66966e43becf 
 download-only = true
 on-update = true
 
@@ -255,7 +255,6 @@ context =
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.81
 erp5.util = 0.4.51
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
@@ -265,34 +264,7 @@ slapos.recipe.template = 4.3
 smmap = 0.9.0
 # websockify = 0.8.0
 
-# Required by:
-# slapos.toolbox==0.71
-GitPython = 2.0.8
-
-# Required by:
-# slapos.toolbox==0.71
-atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.71
-dnspython = 1.14.0
-
-# Required by:
-# slapos.toolbox==0.71
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.71
-lockfile = 0.12.2
-
 # Required by:
 # websockify==0.8.0
 numpy = 1.11.2rc1
 
-# Required by:
-# slapos.toolbox==0.71
-paramiko = 2.0.2
-
-# Required by:
-# slapos.toolbox==0.71
-passlib = 1.6.5

software/kvm/software.cfg.json

@@ -24,6 +24,15 @@
       "request": "instance-kvm-cluster-input-schema.json",
       "response": "instance-kvm-output-schema.json",
       "index": 2
+    },
+    "kvm-cluster-simplified": {
+      "title": "Basic Cluster",
+      "description": "Simplified Schema for Cluster KVM",
+      "software-type": "kvm-cluster",
+      "serialisation": "json-in-xml",
+      "request": "instance-kvm-cluster-simplified-input-schema.json",
+      "response": "instance-kvm-output-schema.json",
+      "index": 3
     }
   }
 }

software/kvm/test/README.md

+Tests for kvm Software Release

software/kvm/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.kvm'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' kvm",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+          'slapos.core',
+          'slapos.libnetworkcache',
+          'erp5.util',
+          'supervisor',
+          'psutil',
+      ],
+      zip_safe=True,
+      test_suite='test',
+      )

software/kvm/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import shutil
+import urlparse
+import tempfile
+import requests
+import socket
+import StringIO
+import subprocess
+import json
+
+import psutil
+
+import utils
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+
+class InstanceTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+class ServicesTestCase(InstanceTestCase):
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  def test_hashes(self):
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    expected_process_names = [
+      '6tunnel-10022-{hash}-on-watch',
+      '6tunnel-10080-{hash}-on-watch',
+      '6tunnel-10443-{hash}-on-watch',
+      'certificate_authority-{hash}-on-watch',
+      'crond-{hash}-on-watch',
+      'kvm-{hash}-on-watch',
+      'websockify-{hash}-on-watch',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_names = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+
+    for name in expected_process_names:
+      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_names)

software/kvm/test/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+import StringIO
+import xmlrpclib
+
+import supervisor.xmlrpc
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+
+
+# Utility functions
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+# TODO:
+#  - allow requesting multiple instances ?
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  """Install one slapos instance.
+
+  This test case install software(s) and request one instance during `setUpClass`
+  and destroy the instance during `tearDownClass`.
+
+  Software Release URL, Instance Software Type and Instance Parameters can be defined
+  on the class.
+
+  All tests from the test class will run with the same instance.
+
+  The following class attributes are available:
+
+    * `computer_partition`:  the computer partition instance, implementing
+    `slapos.slap.interface.slap.IComputerPartition`.
+
+    * `computer_partition_root_path`: the path of the instance root directory.
+
+  """
+
+  # Methods to be defined by subclasses.
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    """Return software type for instance, default "default"
+
+    To be defined by subclasses if they need to request instance with specific
+    software type.
+    """
+    return "default"
+
+  # Utility methods.
+  def getSupervisorRPCServer(self):
+    """Returns a XML-RPC connection to the supervisor used by slapos node
+
+    Refer to http://supervisord.org/api.html for details of available methods.
+    """
+    # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
+    return xmlrpclib.ServerProxy(
+       'http://slapos-supervisor',
+       transport=supervisor.xmlrpc.SupervisorTransport(
+           None,
+           None,
+           # XXX hardcoded socket path
+           serverurl="unix://{working_directory}/inst/supervisord.socket".format(
+             **self.config)))
+
+  # Unittest methods
+  @classmethod
+  def setUpClass(cls):
+    """Setup the class, build software and request an instance.
+
+    If you have to override this method, do not forget to call this method on
+    parent class.
+    """
+    try:
+      cls.setUpWorkingDirectory()
+      cls.setUpConfig()
+      cls.setUpSlapOSController()
+
+      cls.runSoftwareRelease()
+      # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
+      # cls.supplySoftwares()
+      # cls.installSoftwares()
+
+      cls.runComputerPartition()
+      # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
+      # cls.requestInstances()
+      # cls.createInstances()
+      # cls.requestInstances()
+
+    except Exception:
+      cls.stopSlapOSProcesses()
+      raise
+
+  @classmethod
+  def tearDownClass(cls):
+    """Tear down class, stop the processes and destroy instance.
+    """
+    cls.stopSlapOSProcesses()
+
+  # Implementation
+  @classmethod
+  def stopSlapOSProcesses(cls):
+    if hasattr(cls, '_process_manager'):
+      cls._process_manager.killPreviousRun()
+
+  @classmethod
+  def setUpWorkingDirectory(cls):
+    """Initialise the directories"""
+    cls.working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    # Supervisord socket name contains the pid number, which is why we add
+    # .xxxxxxx in this check.
+    if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
+      raise RuntimeError('working directory ( {} ) is too deep, try setting '
+              'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
+
+    if not os.path.exists(cls.working_directory):
+      os.mkdir(cls.working_directory)
+
+  @classmethod
+  def setUpConfig(cls):
+    """Create slapos configuration"""
+    cls.config = {
+      "working_directory": cls.working_directory,
+      "slapos_directory": cls.working_directory,
+      "log_directory": cls.working_directory,
+      "computer_id": 'slapos.test',  # XXX
+      'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
+      'partition_reference': cls.__name__,
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+    }
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['GLOBAL_IPV6']
+
+    cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
+    cls.config['ipv6_address'] = ipv6_address
+    cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
+      **cls.config)
+
+  @classmethod
+  def setUpSlapOSController(cls):
+    """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
+
+    This is equivalent to:
+
+    slapos proxy start
+    for sr in getSoftwareURLList; do
+      slapos supply $SR $COMP
+    done
+    """
+    cls._process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    cls.slapos_controler = SlapOSControler(
+        cls.working_directory,
+        cls.config
+    )
+
+    slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    cls.software_url_list = cls.getSoftwareURLList()
+    cls.slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=cls._process_manager,
+        reset_software=False,
+        software_path_list=cls.software_url_list)
+
+    # XXX we should check *earlier* if that pidfile exist and if supervisord
+    # process still running, because if developer started supervisord (or bugs?)
+    # then another supervisord will start and starting services a second time
+    # will fail.
+    cls._process_manager.supervisord_pid_file = os.path.join(
+      cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+  @classmethod
+  def runSoftwareRelease(cls):
+    """Run all the software releases that were supplied before.
+
+    This is the equivalent of `slapos node software`.
+
+    The tests will be marked file if software building fail.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    try:
+      cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
+        cls.config, environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.software_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+
+  @classmethod
+  def runComputerPartition(cls):
+    """Instanciate the software.
+
+    This is the equivalent of doing:
+
+    slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
+    slapos node instance
+
+    and return the slapos request instance parameters.
+
+    This can be called by tests to simulate re-request with different parameters.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    if cls.getInstanceSoftwareType() != 'default':
+      raise NotImplementedError
+
+    instance_parameter_dict = cls.getInstanceParameterDict()
+    try:
+      cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
+        cls.config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.instance_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+    # FIXME: similar to test node, only one (root) partition is really
+    #        supported for now.
+    computer_partition_list = []
+    for i in range(len(cls.software_url_list)):
+      computer_partition_list.append(
+          cls.slapos_controler.slap.registerOpenOrder().request(
+            cls.software_url_list[i],
+            # This is how testnode's SlapOSControler name created partitions
+            partition_reference='testing partition {i}'.format(
+              i=i, **cls.config),
+            partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParameterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        cls.config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+
+

software/neoppod/software-common.cfg

@@ -134,7 +134,6 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 setproctitle = 1.1.10
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.81
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0
@@ -143,37 +142,6 @@ cython-zstd = 0.2
 python-dateutil = 2.7.3
 pyasn1 = 0.4.3
 
-# Required by:
-# slapos.toolbox==0.71
-GitPython = 2.1.3
-
-# Required by:
-# slapos.toolbox==0.71
-PyRSS2Gen = 1.1
-
-# Required by:
-# slapos.toolbox==0.71
-atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.71
-dnspython = 1.14.0
-
-# Required by:
-# slapos.toolbox==0.71
-feedparser = 5.2.1
-
-# slapos.toolbox==0.71
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.71
-paramiko = 2.1.2
-
-# Required by:
-# slapos.toolbox==0.71
-passlib = 1.7.1
-
 # Required by:
 # zodbtools==0.0.0dev4
 zodburi = 2.3.0

software/neotest/software.cfg

@@ -46,8 +46,8 @@ parts =
 # go packages to install (+ automatically their dependencies)
 [gowork]
 install =
-    lab.nexedi.com/kirr/neo/go/...  \
-    github.com/pkg/profile          \
+    lab.nexedi.com/kirr/neo/go/...
+    github.com/pkg/profile
     golang.org/x/perf/cmd/benchstat
 
 cpkgpath =

software/nginx-push-stream/software.cfg

@@ -46,16 +46,8 @@ mode = 0644
 
 [versions]
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.81
-dnspython = 1.15.0
-PyRSS2Gen = 1.1
 erp5.util = 0.4.51
-passlib = 1.7.1
-GitPython = 2.1.8
-lockfile = 0.12.2
 apache-libcloud = 2.2.1
-feedparser = 5.2.1
-atomize = 0.2.0
 inotifyx = 0.2.2
 gitdb2 = 2.0.3
 pyasn1 = 0.4.2

software/powerdns/instance-powerdns.cfg

@@ -86,6 +86,7 @@ extra-context =
 recipe = slapos.cookbook:wrapper
 command-line = ${powerdns:location}/sbin/pdns_server --config-dir=$${pdns-directory:configuration}
 wrapper-path = $${directory:service}/pdns
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [pdns-reload]
 recipe = slapos.cookbook:wrapper

software/powerdns/software.cfg

@@ -25,7 +25,7 @@ mode = 0644
 [template-powerdns]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-powerdns.cfg
-md5sum = 8ef7c87e23dd63c945c7e292fef7385d
+md5sum = e623ae588e14cbfda69e28a3002eaa29
 output = ${buildout:directory}/template-powerdns.cfg
 mode = 0644
 

software/powerdns/test/README.md

+Tests for powerdns Software Release

software/powerdns/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.powerdns'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' powerdns",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+          'slapos.core',
+          'slapos.libnetworkcache',
+          'erp5.util',
+          'supervisor',
+          'psutil',
+      ],
+      zip_safe=True,
+      test_suite='test',
+      )

software/powerdns/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import shutil
+import urlparse
+import tempfile
+import requests
+import socket
+import StringIO
+import subprocess
+import json
+
+import psutil
+
+import utils
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+
+class InstanceTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+
+class ServicesTestCase(InstanceTestCase):
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  def test_hashes(self):
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    expected_process_names = [
+      'pdns-{hash}-on-watch',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_names = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+
+    for name in expected_process_names:
+      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_names)

software/powerdns/test/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+import StringIO
+import xmlrpclib
+
+import supervisor.xmlrpc
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+
+
+# Utility functions
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+# TODO:
+#  - allow requesting multiple instances ?
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  """Install one slapos instance.
+
+  This test case install software(s) and request one instance during `setUpClass`
+  and destroy the instance during `tearDownClass`.
+
+  Software Release URL, Instance Software Type and Instance Parameters can be defined
+  on the class.
+
+  All tests from the test class will run with the same instance.
+
+  The following class attributes are available:
+
+    * `computer_partition`:  the computer partition instance, implementing
+    `slapos.slap.interface.slap.IComputerPartition`.
+
+    * `computer_partition_root_path`: the path of the instance root directory.
+
+  """
+
+  # Methods to be defined by subclasses.
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    """Return software type for instance, default "default"
+
+    To be defined by subclasses if they need to request instance with specific
+    software type.
+    """
+    return "default"
+
+  # Utility methods.
+  def getSupervisorRPCServer(self):
+    """Returns a XML-RPC connection to the supervisor used by slapos node
+
+    Refer to http://supervisord.org/api.html for details of available methods.
+    """
+    # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
+    return xmlrpclib.ServerProxy(
+       'http://slapos-supervisor',
+       transport=supervisor.xmlrpc.SupervisorTransport(
+           None,
+           None,
+           # XXX hardcoded socket path
+           serverurl="unix://{working_directory}/inst/supervisord.socket".format(
+             **self.config)))
+
+  # Unittest methods
+  @classmethod
+  def setUpClass(cls):
+    """Setup the class, build software and request an instance.
+
+    If you have to override this method, do not forget to call this method on
+    parent class.
+    """
+    try:
+      cls.setUpWorkingDirectory()
+      cls.setUpConfig()
+      cls.setUpSlapOSController()
+
+      cls.runSoftwareRelease()
+      # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
+      # cls.supplySoftwares()
+      # cls.installSoftwares()
+
+      cls.runComputerPartition()
+      # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
+      # cls.requestInstances()
+      # cls.createInstances()
+      # cls.requestInstances()
+
+    except Exception:
+      cls.stopSlapOSProcesses()
+      raise
+
+  @classmethod
+  def tearDownClass(cls):
+    """Tear down class, stop the processes and destroy instance.
+    """
+    cls.stopSlapOSProcesses()
+
+  # Implementation
+  @classmethod
+  def stopSlapOSProcesses(cls):
+    if hasattr(cls, '_process_manager'):
+      cls._process_manager.killPreviousRun()
+
+  @classmethod
+  def setUpWorkingDirectory(cls):
+    """Initialise the directories"""
+    cls.working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    # Supervisord socket name contains the pid number, which is why we add
+    # .xxxxxxx in this check.
+    if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
+      raise RuntimeError('working directory ( {} ) is too deep, try setting '
+              'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
+
+    if not os.path.exists(cls.working_directory):
+      os.mkdir(cls.working_directory)
+
+  @classmethod
+  def setUpConfig(cls):
+    """Create slapos configuration"""
+    cls.config = {
+      "working_directory": cls.working_directory,
+      "slapos_directory": cls.working_directory,
+      "log_directory": cls.working_directory,
+      "computer_id": 'slapos.test',  # XXX
+      'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
+      'partition_reference': cls.__name__,
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+    }
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['GLOBAL_IPV6']
+
+    cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
+    cls.config['ipv6_address'] = ipv6_address
+    cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
+      **cls.config)
+
+  @classmethod
+  def setUpSlapOSController(cls):
+    """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
+
+    This is equivalent to:
+
+    slapos proxy start
+    for sr in getSoftwareURLList; do
+      slapos supply $SR $COMP
+    done
+    """
+    cls._process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    cls.slapos_controler = SlapOSControler(
+        cls.working_directory,
+        cls.config
+    )
+
+    slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    cls.software_url_list = cls.getSoftwareURLList()
+    cls.slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=cls._process_manager,
+        reset_software=False,
+        software_path_list=cls.software_url_list)
+
+    # XXX we should check *earlier* if that pidfile exist and if supervisord
+    # process still running, because if developer started supervisord (or bugs?)
+    # then another supervisord will start and starting services a second time
+    # will fail.
+    cls._process_manager.supervisord_pid_file = os.path.join(
+      cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+  @classmethod
+  def runSoftwareRelease(cls):
+    """Run all the software releases that were supplied before.
+
+    This is the equivalent of `slapos node software`.
+
+    The tests will be marked file if software building fail.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    try:
+      cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
+        cls.config, environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.software_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+
+  @classmethod
+  def runComputerPartition(cls):
+    """Instanciate the software.
+
+    This is the equivalent of doing:
+
+    slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
+    slapos node instance
+
+    and return the slapos request instance parameters.
+
+    This can be called by tests to simulate re-request with different parameters.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    if cls.getInstanceSoftwareType() != 'default':
+      raise NotImplementedError
+
+    instance_parameter_dict = cls.getInstanceParameterDict()
+    try:
+      cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
+        cls.config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.instance_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+    # FIXME: similar to test node, only one (root) partition is really
+    #        supported for now.
+    computer_partition_list = []
+    for i in range(len(cls.software_url_list)):
+      computer_partition_list.append(
+          cls.slapos_controler.slap.registerOpenOrder().request(
+            cls.software_url_list[i],
+            # This is how testnode's SlapOSControler name created partitions
+            partition_reference='testing partition {i}'.format(
+              i=i, **cls.config),
+            partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParameterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        cls.config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+
+

software/re6stnet/instance-re6stnet.cfg.in

@@ -80,13 +80,19 @@ wrapper-path = {{ apache_wrapper }}
 command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
 
 {% if uri_scheme == 'https' %}
-[{{ section('apache-ca') }}]
+[apache-ca]
 <= certificate-authority
 recipe = slapos.cookbook:certificate_authority.request
 executable = ${apache-httpd:wrapper-path}
-wrapper = ${directory:services}/httpd
+wrapper = ${directory:bin}/httpd
 key-file = ${certificate-authority:ca-private}/apache.key
 cert-file = ${certificate-authority:ca-certs}/apache.crt
+
+[{{ section('apache-ca-service') }}]
+recipe = slapos.cookbook:wrapper
+command-line = ${apache-ca:wrapper}
+wrapper-path = ${directory:services}/httpd
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 {% endif %}
 
 [apache-httpd-graceful]

software/re6stnet/software.cfg

@@ -86,7 +86,7 @@ extra-context =
 [template-re6stnet]
 < = download-base
 filename = instance-re6stnet.cfg.in
-md5sum = 8c167f2adb2ed36aeaff773f59214981
+md5sum = 09683c66c16361b77867d8f2e1140a35
 
 [template-apache-conf]
 < = download-base
@@ -117,41 +117,16 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.81
 smmap = 0.9.0
 dnspython = 1.15.0
 erp5.util = 0.4.51
 passlib = 1.7.1
 
-# Required by:
-# slapos.toolbox==0.71
-GitPython = 2.0.8
-
-# Required by:
-# slapos.toolbox==0.71
-atomize = 0.2.0
-
 # Required by:
 # apache-libcloud==0.17.0
 backports.ssl-match-hostname = 3.4.0.2
 
-# Required by:
-# slapos.toolbox==0.71
-feedparser = 5.1.3
-
-# Required by:
-# slapos.toolbox==0.71
-lockfile = 0.12.2
-
 # Required by:
 # re6stnet===0-413.gbec6b3c.dirty
 miniupnpc = 1.9
 
-# Required by:
-# slapos.toolbox==0.71
-paramiko = 2.0.1
-
-# Required by:
-# slapos.toolbox==0.71
-rpdb = 0.1.5
-

software/re6stnet/test/test.py

@@ -81,3 +81,41 @@ class TestPortRedirection(Re6stnetTestCase):
       'srcPort': 9201,
       'destPort': 9201,
     }, portredir_config[0])
+
+class ServicesTestCase(Re6stnetTestCase):
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'uri-scheme': 'https'}
+
+  def test_hashes(self):
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    expected_process_names = [
+      'httpd-{hash}-on-watch',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_names = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+
+    for name in expected_process_names:
+      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_names)

software/seleniumrunner/buildout.hash.cfg

@@ -19,4 +19,4 @@ md5sum = c4ac5de141ae6a64848309af03e51d88
 
 [template-selenium]
 filename = instance-selenium.cfg.in
-md5sum = 8f06bef6ed0737afa7916bc5ac7db537
+md5sum = 4167621b473f81892d38389ac427c6ba

software/seleniumrunner/instance-selenium.cfg.in

 [buildout]
 parts =
-  selenium-instance
+  promises
+  publish-connection-parameter
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory} 
 offline = true
 
-[rootdirectory]
+[fontconfig-instance]
+recipe = slapos.cookbook:fontconfig
+conf-path = $${directory:etc}/font.conf
+font-system-folder = ${fonts:location}
+font-folder = $${directory:fonts}
+service-folder = $${directory:services}
+
+[xvfb-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line =
+  ${xserver:location}/bin/Xvfb
+    $${:display}
+    -screen 0 1024x768x24
+    -fbdir $${directory:framebuffer}
+
+environment=
+  XORG_LOCK_DIR=$${directory:tmp}
+# We can safely use any $DISPLAY, because our slapos patched X11 isolate $DISPLAYs
+# by placing sockets in $XORG_LOCK_DIR and using different $XORG_LOCK_DIR per partitions.
+display = :0
+
+# For selenium server options:
+# https://github.com/SeleniumHQ/selenium/wiki/Grid2#configuring-the-nodes-by-command-line
+[selenium-server-hub-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line =
+  ${java-re-8:location}/bin/java
+    -jar ${selenium-server:target}
+    -host $${:hostname}
+    -port $${:port}
+    -role hub
+    -maxSession 20
+    -newSessionWaitTimeout 600000
+    --debug
+
+# newSessionWaitTimeout: let clients wait in the queue when no node are available
+# maxSession: to accept enough clients
+hostname = $${instance-parameter:ipv4-random}
+port = 4444
+base-url = http://$${:hostname}:$${:port}
+url = $${:base-url}/wd/hub
+register-url = $${:base-url}/grid/register/
+
+
+[selenium-server-node-instance]
+# Macro for a selenium server node
+capabilities =
+port =
+java-args =
+
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line =
+  ${java-re-8:location}/bin/java $${:java-args}
+    -jar ${selenium-server:target}
+    -host $${:hostname}
+    -port $${:port}
+    -role node
+    -timeout 300
+    -hub $${selenium-server-hub-instance:register-url}
+    -capabilities $${:capabilities}
+    --debug
+
+# browsers and drivers are in buildout bin-directory, so we add that directory to to $PATH
+environment =
+  PATH=${buildout:bin-directory}
+  XORG_LOCK_DIR=$${directory:tmp}
+  DISPLAY=$${xvfb-instance:display}
+  FONTCONFIG_FILE=$${fontconfig-instance:conf-path}
+hostname = $${instance-parameter:ipv4-random}
+
+
+[selenium-server-node-instance-firefox-52]
+<= selenium-server-node-instance
+capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-52:version},firefox_binary=${firefox-wrapper-52:location}
+java-args = -Dwebdriver.gecko.driver=${geckodriver-0.16.1:location}
+port = 7777
+
+[selenium-server-node-instance-firefox-60]
+<= selenium-server-node-instance
+capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-60:version},firefox_binary=${firefox-wrapper-60:location}
+java-args = -Dwebdriver.gecko.driver=${geckodriver-0.22.0:location}
+port = 7778
+
+[selenium-server-node-instance-chromium-69]
+<= selenium-server-node-instance
+capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-69:version},chrome_binary=${chromium-wrapper-69:location}
+java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-2.41:location}
+port = 7779
+
+
+[selenium-server-admin-password]
+recipe = slapos.cookbook:generate.password
+username = admin
+bytes = 12
+
+[selenium-server-selenium-password]
+recipe = slapos.cookbook:generate.password
+username = selenium
+bytes = 12
+
+
+[selenium-server-frontend-config]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:etc}/$${:_buildout_section_name_}
+template = inline:
+  https://$${:hostname}:$${:port} {
+    bind $${:ip}
+    tls self_signed # TODO
+    proxy / $${selenium-server-hub-instance:base-url} {
+      transparent
+    }
+    basicauth $${selenium-server-admin-password:username} $${selenium-server-admin-password:passwd} {
+      realm "Grid Admin"
+      $${:path-admin}
+    }
+    basicauth $${selenium-server-selenium-password:username} $${selenium-server-selenium-password:passwd} {
+      realm "Selenium Server"
+      $${:path-hub}
+    }
+  }
+ip = $${instance-parameter:ipv6-random}
+hostname = [$${:ip}]
+port = 9443
+path-admin = /grid/console
+path-hub = /wd/hub
+
+
+[selenium-server-frontend-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line =
+  ${caddy:output} -conf $${selenium-server-frontend-config:rendered}
+
+ip = $${selenium-server-frontend-config:ip}
+hostname = $${selenium-server-frontend-config:hostname}
+port = $${selenium-server-frontend-config:port}
+
+admin-url = https://$${selenium-server-admin-password:username}:$${selenium-server-admin-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-config:path-admin}
+url = https://$${selenium-server-selenium-password:username}:$${selenium-server-selenium-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-config:path-hub}
+
+
+[userinfo]
+recipe = slapos.cookbook:userinfo
+
+[sshd-address]
+recipe = slapos.cookbook:free_port
+minimum = 22222
+maximum = 22231
+ip = $${slap-network-information:global-ipv6}
+hostname = $${:ip}
+
+[ssh-keygen-base]
+recipe = plone.recipe.command
+output = $${directory:etc}/$${:_buildout_section_name_}
+command = ${openssh-output:keygen} -f $${:output} -N '' $${:extra-args}
+
+[ssh-host-rsa-key]
+<=ssh-keygen-base
+extra-args=-t rsa
+[ssh-host-dsa-key]
+<=ssh-keygen-base
+extra-args=-t dsa
+[ssh-host-ecdsa-key]
+<=ssh-keygen-base
+extra-args=-t ecdsa -b 521
+
+[sshd-config]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:etc}/sshd.conf
+path_pid = $${directory:run}/sshd.pid
+template = inline:
+  PidFile $${:path_pid}
+  Port $${sshd-address:port}
+  ListenAddress $${sshd-address:ip}
+  Protocol 2
+  UsePrivilegeSeparation no
+  HostKey $${ssh-host-rsa-key:output}
+  HostKey $${ssh-host-dsa-key:output}
+  HostKey $${ssh-host-ecdsa-key:output}
+  PasswordAuthentication no
+  PubkeyAuthentication yes
+  AuthorizedKeysFile $${sshd-authorized-key:authorized-keys-file}
+
+  ClientAliveInterval 30
+  ClientAliveCountMax 10
+  ForceCommand echo "Welcome to SlapOS Selenium Server."; ${coreutils:location}/bin/sleep infinity
+
+[sshd-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${openssh:location}/sbin/sshd -D -e -f $${sshd-config:rendered}
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+username = $${userinfo:pw-name}
+ip = $${sshd-address:ip}
+port = $${sshd-address:port}
+url = ssh://$${:username}@[$${:ip}]:$${:port}
+
+
+[sshd-authorized-key]
+recipe = plone.recipe.command
+stop-on-error = true
+location = $${buildout:directory}/.ssh
+authorized-keys-file = $${:location}/authorized_keys
+command = mkdir -p $${:location} && echo '$${instance-parameter:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
+
+
+[promises]
+recipe =
+instance-promises =
+  $${sshd-listen-promise:path}
+  $${selenium-server-frontend-listen-promise:path}
+  $${selenium-server-hub-listen-promise:path}
+  $${selenium-server-node-firefox-52-listen-promise:path}
+  $${selenium-server-node-firefox-60-listen-promise:path}
+  $${selenium-server-node-instance-chromium-69-listen-promise:path}
+
+
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${directory:promises}/$${:_buildout_section_name_}
+
+[sshd-listen-promise]
+<= check-port-listening-promise
+hostname = $${sshd-address:hostname}
+port = $${sshd-address:port}
+
+[selenium-server-frontend-listen-promise]
+<= check-port-listening-promise
+hostname = $${selenium-server-frontend-instance:ip}
+port = $${selenium-server-frontend-instance:port}
+
+[selenium-server-hub-listen-promise]
+<= check-port-listening-promise
+hostname = $${selenium-server-hub-instance:hostname}
+port = $${selenium-server-hub-instance:port}
+
+[selenium-server-node-firefox-52-listen-promise]
+<= check-port-listening-promise
+hostname = $${selenium-server-node-instance-firefox-52:hostname}
+port = $${selenium-server-node-instance-firefox-52:port}
+
+[selenium-server-node-firefox-60-listen-promise]
+<= check-port-listening-promise
+hostname = $${selenium-server-node-instance-firefox-60:hostname}
+port = $${selenium-server-node-instance-firefox-60:port}
+
+[selenium-server-node-instance-chromium-69-listen-promise]
+<= check-port-listening-promise
+hostname = $${selenium-server-node-instance-chromium-69:hostname}
+port = $${selenium-server-node-instance-chromium-69:port}
+
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+backend-url = $${selenium-server-hub-instance:url}
+url = $${selenium-server-frontend-instance:url}
+admin-url = $${selenium-server-frontend-instance:admin-url}
+
+ssh-url = $${sshd-service:url}
+# to run a local node - useful to see what tests are doing or
+# using to use unsupported browsers like safari or edge or to test
+# on mobile using appium.
+# $PORT must be free on both hosts, different clients must use different ports.
+run-node-command = PORT=7999 bash -c 'trap '"'"'kill -TERM $SSHPID; wait $SSHPID '"'"' TERM INT; ssh -L 4444:$${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port} -R $PORT:127.0.0.1:$PORT -p $${sshd-service:port} $${sshd-service:username}@$${sshd-service:ip} & SSHPID=$!; java -jar selenium-server-standalone-3.14.0.jar -role node -host 127.0.0.1 -port $PORT ; wait "$SSHPID"'
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+# A ssh public key, as found in ~/.ssh/authorized_keys
+# multiple keys can be given, indented in a buildout compatible format (mmmh)
+configuration.ssh-authorized-key =
+
+[directory]
 recipe = slapos.cookbook:mkdirectory
 etc = $${buildout:directory}/etc
 var = $${buildout:directory}/var
 srv = $${buildout:directory}/srv
 bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
 
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-services = $${rootdirectory:etc}/run
-run = $${rootdirectory:var}/run
-framebuffer = $${rootdirectory:srv}/framebuffer
+services = $${:etc}/service
+promises = $${:etc}/promise
+framebuffer = $${:srv}/framebuffer
+fonts = $${:srv}/fonts/
+ssh = $${:etc}/ssh
+run = $${:var}/run
\ No newline at end of file

software/seleniumrunner/software.cfg

+# Selenium server
+# https://seleniumhq.github.io/docs/grid.html
+
 [buildout]
 extends =
    ../../component/xorg/buildout.cfg
    ../../component/lxml-python/buildout.cfg
    ../../component/firefox/buildout.cfg
-   ../../component/dash/buildout.cfg
+   ../../component/chromium/buildout.cfg
+   ../../component/chromedriver/buildout.cfg
+   ../../component/coreutils/buildout.cfg
+   ../../component/java/buildout.cfg
+   ../../component/caddy/buildout.cfg
+   ../../component/openssh/buildout.cfg
    ../../stack/slapos.cfg
    ./buildout.hash.cfg
 
-# develop += /opt/slapdev
-
 parts =
    slapos-cookbook
    template
-   eggs
-   instance-recipe-egg
-   xserver
    firefox-wrapper
    geckodriver
-   xwd
+# seleniumserver needs to install a default firefox as ${buildout:bin-directory}/firefox and
+# a geckodriver as ${buildout:bin-directory}/geckodriver for erp5testnode
 
 [instance-recipe]
 egg = slapos.cookbook
 module = seleniumrunner
 
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-eggs = ${instance-recipe:egg}
-
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-  ${lxml-python:egg}
-
+[selenium-server]
+recipe = slapos.recipe.build:download
+version = 3.14.0
+md5sum = 376450bd517510442b60018646deadfe
+filename = selenium-server-standalone-${:version}.jar
+url = https://selenium-release.storage.googleapis.com/3.14/${:filename}
 
 [macro-template]
 recipe = slapos.recipe.template

software/seleniumrunner/test/README.md

+Tests for SeleniumRunner software release

software/seleniumrunner/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.seleniumrunner'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' Seleniumrunner",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'supervisor',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'selenium',
+        'psutil',
+        'image',
+        'requests',
+        'paramiko',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/seleniumrunner/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import cgi
+import json
+import multiprocessing
+import os
+import tempfile
+import unittest
+import urlparse
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from io import BytesIO
+
+import paramiko
+import requests
+from PIL import Image
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+from utils import SlapOSInstanceTestCase, findFreeTCPPort
+
+debug_mode = os.environ.get('DEBUG')
+# for development: debugging logs and install Ctrl+C handler
+if debug_mode:
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  unittest.installHandler()
+
+
+class WebServerMixin(object):
+  """Mixin class which provides a simple web server reachable at self.server_url
+  """
+  def setUp(self):
+    """Start a minimal web server.
+    """
+    class TestHandler(BaseHTTPRequestHandler):
+      """Request handler for our test server.
+
+      The implemented server is:
+       - submit q and you'll get a page with q as title
+       - upload a file and the file content will be displayed in div.uploadedfile
+      """
+      def log_message(self, *args, **kw):
+        if debug_mode:
+          BaseHTTPRequestHandler.log_message(self, *args, **kw)
+      def do_GET(self):
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write('''
+          <html>
+            <title>Test page</title>
+            <body>
+              <form action="/" method="POST" enctype="multipart/form-data">
+                <input name="q" type="text"></input>
+                <input name="f" type="file" ></input>
+                <input type="submit" value="I'm feeling lucky"></input>
+              </form>
+            </body>
+          </html>''')
+      def do_POST(self):
+        form = cgi.FieldStorage(
+            fp=self.rfile,
+            headers=self.headers,
+            environ={'REQUEST_METHOD':'POST',
+                     'CONTENT_TYPE':self.headers['Content-Type'],})
+        self.send_response(200)
+        self.end_headers()
+        file_data = 'no file'
+        if form.has_key('f'):
+          file_data = form['f'].file.read()
+        self.wfile.write('''
+          <html>
+            <title>%s</title>
+            <div>%s</div>
+          </html>
+        ''' % (form['q'].value, file_data))
+
+    super(WebServerMixin, self).setUp()
+    ip = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    port = findFreeTCPPort(ip)
+    server = HTTPServer((ip, port), TestHandler)
+    self.server_process = multiprocessing.Process(target=server.serve_forever)
+    self.server_process.start()
+    self.server_url = 'http://%s:%s/' % (ip, port)
+
+  def tearDown(self):
+    self.server_process.terminate()
+    self.server_process.join()
+    super(WebServerMixin, self).tearDown()
+
+
+class BrowserCompatibilityMixin(WebServerMixin):
+  """Mixin class to run validation tests on a specific browser
+  """
+  desired_capabilities = NotImplemented
+  user_agent = NotImplemented
+
+  def setUp(self):
+    super(BrowserCompatibilityMixin, self).setUp()
+    self.driver = webdriver.Remote(
+         command_executor=self.computer_partition.getConnectionParameterDict()['backend-url'],
+         desired_capabilities=self.desired_capabilities)
+
+  def tearDown(self):
+    self.driver.quit()
+    super(BrowserCompatibilityMixin, self).tearDown()
+
+  def test_user_agent(self):
+    self.assertIn(
+        self.user_agent,
+        self.driver.execute_script('return navigator.userAgent'))
+
+  def test_simple_submit_scenario(self):
+    self.driver.get(self.server_url)
+    input_element  = WebDriverWait(self.driver, 3).until(
+        EC.visibility_of_element_located((By.NAME, 'q')))
+    input_element.send_keys(self.id())
+    input_element.submit()
+    WebDriverWait(self.driver, 3).until(EC.title_is(self.id()))
+
+  def test_upload_file(self):
+    f = tempfile.NamedTemporaryFile(delete=False)
+    f.write(self.id())
+    f.close()
+    self.addCleanup(lambda: os.remove(f.name))
+
+    self.driver.get(self.server_url)
+    self.driver.find_element_by_xpath('//input[@name="f"]').send_keys(f.name)
+    self.driver.find_element_by_xpath('//input[@type="submit"]').click()
+
+    self.assertEqual(
+        self.id(),
+        self.driver.find_element_by_xpath('//div').text)
+
+  def test_screenshot(self):
+    self.driver.get(self.server_url)
+    screenshot = Image.open(BytesIO(self.driver.get_screenshot_as_png()))
+    # just check it's not a white screen
+    self.assertGreater(len(screenshot.getcolors(maxcolors=512)), 2)
+
+  def test_window_and_screen_size(self):
+    size = json.loads(self.driver.execute_script('''
+      return JSON.stringify({
+        'screen.width': window.screen.width,
+        'screen.height': window.screen.height,
+        'screen.pixelDepth': window.screen.pixelDepth,
+        'innerWidth': window.innerWidth,
+        'innerHeight': window.innerHeight
+      })'''))
+    # Xvfb is configured like this
+    self.assertEqual(1024, size['screen.width'])
+    self.assertEqual(768, size['screen.height'])
+    self.assertEqual(24, size['screen.pixelDepth'])
+
+    # window size must not be 0 (wrong firefox integration report this)
+    self.assertGreater(size['innerWidth'], 0)
+    self.assertGreater(size['innerHeight'], 0)
+
+  def test_resize_window(self):
+    self.driver.set_window_size(800, 900)
+    size = json.loads(self.driver.execute_script('''
+      return JSON.stringify({
+        'outerWidth': window.outerWidth,
+        'outerHeight': window.outerHeight
+        })'''))
+    self.assertEqual(800, size['outerWidth'])
+    self.assertEqual(900, size['outerHeight'])
+
+  def test_multiple_clients(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    webdriver_url = parameter_dict['backend-url']
+
+    queue = multiprocessing.Queue()
+    def _test(q, server_url):
+      driver = webdriver.Remote(
+           command_executor=webdriver_url,
+           desired_capabilities=self.desired_capabilities)
+      try:
+        driver.get(server_url)
+        q.put(driver.title == 'Test page')
+      finally:
+        driver.quit()
+
+    nb_workers = 10
+    workers = []
+    for i in range(nb_workers):
+      worker = multiprocessing.Process(
+          target=_test,
+          args=(queue, self.server_url))
+
+      worker.start()
+      workers.append(worker)
+    del worker # pylint
+    _ = [worker.join(timeout=30) for worker in workers]
+
+    # terminate workers if they are still alive after 30 seconds
+    _ = [worker.terminate() for worker in workers if worker.is_alive()]
+    _ = [worker.join() for worker in workers]
+
+    del _ # pylint
+    self.assertEqual(
+        [True] * nb_workers,
+        [queue.get() for _ in range(nb_workers)])
+
+
+class SeleniumServerTestCase(SlapOSInstanceTestCase):
+  """Test the remote driver on a minimal web server.
+  """
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+
+class TestBrowserSelection(WebServerMixin, SeleniumServerTestCase):
+  """Test browser can be selected by `desiredCapabilities``
+  """
+  def test_chrome(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    webdriver_url = parameter_dict['backend-url']
+
+    driver = webdriver.Remote(
+         command_executor=webdriver_url,
+         desired_capabilities=DesiredCapabilities.CHROME)
+
+    driver.get(self.server_url)
+    self.assertEqual('Test page', driver.title)
+
+    self.assertIn(
+        'Chrome',
+        driver.execute_script('return navigator.userAgent'))
+    self.assertNotIn(
+        'Firefox',
+        driver.execute_script('return navigator.userAgent'))
+    driver.quit()
+
+  def test_firefox(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    webdriver_url = parameter_dict['backend-url']
+
+    driver = webdriver.Remote(
+         command_executor=webdriver_url,
+         desired_capabilities=DesiredCapabilities.FIREFOX)
+
+    driver.get(self.server_url)
+    self.assertEqual('Test page', driver.title)
+
+    self.assertIn(
+        'Firefox',
+        driver.execute_script('return navigator.userAgent'))
+    driver.quit()
+
+  def test_firefox_desired_version(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    webdriver_url = parameter_dict['backend-url']
+
+    desired_capabilities = DesiredCapabilities.FIREFOX.copy()
+    desired_capabilities['version'] = '60.0.2esr'
+    driver = webdriver.Remote(
+         command_executor=webdriver_url,
+         desired_capabilities=desired_capabilities)
+    self.assertIn(
+        'Gecko/20100101 Firefox/60.0',
+        driver.execute_script('return navigator.userAgent'))
+    driver.quit()
+    desired_capabilities['version'] = '52.9.0esr'
+    driver = webdriver.Remote(
+         command_executor=webdriver_url,
+         desired_capabilities=desired_capabilities)
+    self.assertIn(
+        'Gecko/20100101 Firefox/52.0',
+        driver.execute_script('return navigator.userAgent'))
+    driver.quit()
+
+
+class TestFrontend(WebServerMixin, SeleniumServerTestCase):
+  """Test hub's https frontend.
+  """
+  def test_admin(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    admin_url = parameter_dict['admin-url']
+
+    parsed = urlparse.urlparse(admin_url)
+    self.assertEqual('admin', parsed.username)
+    self.assertTrue(parsed.password)
+
+    self.assertIn(
+        'Grid Console',
+        requests.get(admin_url, verify=False).text)
+
+  def test_browser_use_hub(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    webdriver_url = parameter_dict['url']
+    parsed = urlparse.urlparse(webdriver_url)
+    self.assertEqual('selenium', parsed.username)
+    self.assertTrue(parsed.password)
+
+    driver = webdriver.Remote(
+         command_executor=webdriver_url,
+         desired_capabilities=DesiredCapabilities.CHROME)
+
+    driver.get(self.server_url)
+    self.assertEqual('Test page', driver.title)
+    driver.quit()
+
+
+class TestSSHServer(SeleniumServerTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    cls.ssh_key = paramiko.RSAKey.generate(1024)
+    return {'ssh-authorized-key': 'ssh-rsa {}'.format(cls.ssh_key.get_base64())}
+
+  def test_connect(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    ssh_url = parameter_dict['ssh-url']
+    parsed = urlparse.urlparse(ssh_url)
+    self.assertEqual('ssh', parsed.scheme)
+
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
+    client.connect(
+      username=urlparse.urlparse(ssh_url).username,
+      hostname=urlparse.urlparse(ssh_url).hostname,
+      port=urlparse.urlparse(ssh_url).port,
+      pkey=self.ssh_key,
+    )
+    channel = client.invoke_shell()
+    channel.settimeout(30)
+    # openssh prints a warning 'Attempt to write login records by non-root user (aborting)'
+    # so we received more than the lenght of the asserted message.
+    self.assertIn("Welcome to SlapOS Selenium Server.", channel.recv(100))
+
+
+class TestFirefox52(BrowserCompatibilityMixin, SeleniumServerTestCase):
+  desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='52.9.0esr')
+  user_agent = 'Gecko/20100101 Firefox/52.0'
+  # resizing window is not supported on firefox 52 geckodriver
+  test_resize_window = unittest.expectedFailure(
+    BrowserCompatibilityMixin.test_resize_window)
+
+
+class TestFirefox60(BrowserCompatibilityMixin, SeleniumServerTestCase):
+  desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='60.0.2esr')
+  user_agent = 'Gecko/20100101 Firefox/60.0'
+
+
+class TestChrome69(BrowserCompatibilityMixin, SeleniumServerTestCase):
+  desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
+  user_agent = 'Chrome/69.0.3497.0'

software/seleniumrunner/test/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+import StringIO
+import xmlrpclib
+
+import supervisor.xmlrpc
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+
+
+# Utility functions
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+# TODO:
+#  - allow requesting multiple instances ?
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  """Install one slapos instance.
+
+  This test case install software(s) and request one instance during `setUpClass`
+  and destroy the instance during `tearDownClass`.
+
+  Software Release URL, Instance Software Type and Instance Parameters can be defined
+  on the class.
+
+  All tests from the test class will run with the same instance.
+  
+  The following class attributes are available:
+
+    * `computer_partition`:  the `slapos.core.XXX` computer partition instance.
+
+    * `computer_partition_root_path`: the path of the instance root directory,
+        
+  """
+  
+  # Methods to be defined by subclasses.
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    """Return software type for instance, default "default"
+
+    To be defined by subclasses if they need to request instance with specific
+    software type.
+    """
+    return "default"
+
+  # Utility methods.
+  def getSupervisorRPCServer(self):
+    """Returns a XML-RPC connection to the supervisor used by slapos node
+
+    Refer to http://supervisord.org/api.html for details of available methods.
+    """
+    # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
+    return xmlrpclib.ServerProxy(
+       'http://slapos-supervisor',
+       transport=supervisor.xmlrpc.SupervisorTransport(
+           None,
+           None,
+           # XXX hardcoded socket path
+           serverurl="unix://{working_directory}/inst/supervisord.socket".format(
+             **self.config)))
+
+  # Unittest methods
+  @classmethod
+  def setUpClass(cls):
+    """Setup the class, build software and request an instance.
+    
+    If you have to override this method, do not forget to call this method on
+    parent class.
+    """
+    try:
+      cls.setUpWorkingDirectory()
+      cls.setUpConfig()
+      cls.setUpSlapOSController()
+
+      cls.runSoftwareRelease()
+      # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
+      # cls.supplySoftwares()
+      # cls.installSoftwares()
+
+      cls.runComputerPartition()
+      # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
+      # cls.requestInstances()
+      # cls.createInstances()
+      # cls.requestInstances()
+
+    except BaseException:
+      cls.stopSlapOSProcesses()
+      cls.setUp = lambda self: self.fail('Setup Class failed.')
+      raise
+
+  @classmethod
+  def tearDownClass(cls):
+    """Tear down class, stop the processes and destroy instance.
+    """
+    cls.stopSlapOSProcesses()
+
+  # Implementation
+  @classmethod
+  def stopSlapOSProcesses(cls):
+    if hasattr(cls, '_process_manager'):
+      cls._process_manager.killPreviousRun()
+
+  @classmethod
+  def setUpWorkingDirectory(cls):
+    """Initialise the directories"""
+    cls.working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    # Supervisord socket name contains the pid number, which is why we add
+    # .xxxxxxx in this check.
+    if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
+      raise RuntimeError('working directory ( {} ) is too deep, try setting '
+              'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
+
+    if not os.path.exists(cls.working_directory):
+      os.mkdir(cls.working_directory)
+
+  @classmethod
+  def setUpConfig(cls):
+    """Create slapos configuration"""
+    cls.config = {
+      "working_directory": cls.working_directory,
+      "slapos_directory": cls.working_directory,
+      "log_directory": cls.working_directory,
+      "computer_id": 'slapos.test',  # XXX
+      'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
+      'partition_reference': cls.__name__,
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+    }
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['GLOBAL_IPV6']
+
+    cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
+    cls.config['ipv6_address'] = ipv6_address
+    cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
+      **cls.config)
+
+  @classmethod
+  def setUpSlapOSController(cls):
+    """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
+
+    This is equivalent to:
+
+    slapos proxy start
+    for sr in getSoftwareURLList; do
+      slapos supply $SR $COMP
+    done
+    """ 
+    cls._process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    cls.slapos_controler = SlapOSControler(
+        cls.working_directory,
+        cls.config
+    )
+
+    slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    cls.software_url_list = cls.getSoftwareURLList()
+    cls.slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=cls._process_manager,
+        reset_software=False,
+        software_path_list=cls.software_url_list)
+
+    # XXX we should check *earlier* if that pidfile exist and if supervisord
+    # process still running, because if developer started supervisord (or bugs?)
+    # then another supervisord will start and starting services a second time
+    # will fail.
+    cls._process_manager.supervisord_pid_file = os.path.join(
+      cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+  @classmethod
+  def runSoftwareRelease(cls):
+    """Run all the software releases that were supplied before.
+
+    This is the equivalent of `slapos node software`.
+
+    The tests will be marked file if software building fail.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    try:
+      cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
+        cls.config, environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.software_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+
+  @classmethod
+  def runComputerPartition(cls):
+    """Instanciate the software.
+
+    This is the equivalent of doing:
+
+    slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
+    slapos node instance
+
+    and return the slapos request instance parameters.
+
+    This can be called by tests to simulate re-request with different parameters.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    if cls.getInstanceSoftwareType() != 'default':
+      raise NotImplementedError
+
+    instance_parameter_dict = cls.getInstanceParameterDict()
+    try:
+      cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
+        cls.config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.instance_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+    # FIXME: similar to test node, only one (root) partition is really
+    #        supported for now.
+    computer_partition_list = []
+    for i in range(len(cls.software_url_list)):
+      computer_partition_list.append(
+          cls.slapos_controler.slap.registerOpenOrder().request(
+            cls.software_url_list[i],
+            # This is how testnode's SlapOSControler name created partitions
+            partition_reference='testing partition {i}'.format(
+              i=i, **cls.config),
+            partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParameterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        cls.config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+
+

software/slapos-testing/software.cfg

@@ -173,7 +173,6 @@ mode = 640
 Pygments = 2.1.3
 
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.81
 
 # All depencies should be pinned.
 apache-libcloud = 2.3.0
@@ -191,30 +190,6 @@ pyflakes = 2.0.0
 smmap2 = 2.0.4
 zope.testing = 4.6.2
 
-# Required by:
-# slapos.toolbox==0.76
-GitPython = 2.1.11
-
-# Required by:
-# slapos.toolbox==0.76
-PyRSS2Gen = 1.1
-
-# Required by:
-# slapos.toolbox==0.76
-atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.76
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.76
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.76
-passlib = 1.7.1
-
 # Required by:
 # caucase
 PyJWT = 1.6.4

software/slaprunner/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 713db528880282d568278f09458d2aab
+md5sum = f0cab61c7b8478afb8b676fc725d50d5
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = e12255a8c946b3eb8c6373fff481339f
+md5sum = cd855670076979919c0fd00cc0f5938c
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -26,15 +26,11 @@ md5sum = ed2e08c07a6727b2012f15da67c0705d
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = 7a879739afe55320ee96409bcc8a52ab
-
-[template-runner-export-script]
-filename = template/runner-export.sh.jinja2
-md5sum = 231f9b74862f8991f54326511e76f5ec
+md5sum = 2d516c6f1337efb7def32771c2eabc2b
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 1a812a06cc02bb11636009f4ec043d54
+md5sum = 01395b98534066ec23a5ad5c96d5f1e7
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2

software/slaprunner/instance-runner-export.cfg.in

@@ -7,6 +7,8 @@ parts +=
   nginx-launcher
   certificate-authority
   ca-nginx
+  certificate-authority-service
+  ca-nginx-service
   logrotate-entry-nginx
   gunicorn-launcher
   gunicorn-graceful
@@ -19,10 +21,12 @@ parts +=
   runner-sshd-graceful
   runner-sshd-promise
   runner-sshkeys-authority
+  runner-sshkeys-authority-service
   runner-sshkeys-sshd
   runtestsuite
   symlinks
   shellinabox
+  shellinabox-service
   slapos-cfg
   cron-entry-prepare-software
   deploy-instance-parameters
@@ -71,21 +75,19 @@ ip = ${slaprunner:ipv4}
 [supervisord]
 port = ${supervisord-free-port:port}
 
+[exporter-raw]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter-raw
+command-line = {{ software_release_bin }}/${slap-parameter:namebase}-exporter --srv-path ${directory:srv} --etc-path ${directory:etc} --backup-path ${directory:backup} --backup-wait-time {{ backup_wait_time }}
+environment =
+  PATH={{ rsync_bin_folder }}:/bin:/usr/bin
+
 [exporter]
-recipe = slapos.recipe.template:jinja2
-template = {{ exporter_script_path }}
-rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
-# backward compatibility for resilient stack
-wrapper = ${:rendered}
-mode = 700
-context =
-  import sys sys
-  import easy_install zc.buildout.easy_install
-  section directory directory
-  raw  backup_wait_time {{ backup_wait_time }}
-  raw  output_log_file ${directory:log}/resilient.log
-  raw  shell_binary {{ bash_executable_location }}
-  raw  rsync_binary {{ rsync_executable_location }}
+recipe = slapos.cookbook:wrapper
+command-line = {{ bash_executable_location }} -c "exec ${exporter-raw:wrapper-path} > >(tee -ai "${directory:log}/resilient.log") 2>&1"
+wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter
+# wrapper parameter is needed by resilience stack
+wrapper = ${:wrapper-path}
 
 [monitor-httpd-free-port]
 recipe = slapos.cookbook:free_port

software/slaprunner/instance-runner-import.cfg.in

@@ -7,6 +7,8 @@ parts +=
   nginx-launcher
   certificate-authority
   ca-nginx
+  certificate-authority-service
+  ca-nginx-service
   gunicorn-launcher
   gunicorn-graceful
   slaprunner-promise
@@ -15,9 +17,11 @@ parts +=
   runner-sshd-graceful
   runner-sshd-promise
   runner-sshkeys-authority
+  runner-sshkeys-authority-service
   runner-sshkeys-sshd
   runtestsuite
   shellinabox
+  shellinabox-service
   symlinks
   slapos-cfg
   cron-entry-prepare-software
@@ -185,3 +189,10 @@ configuration-file-path = ${buildout:directory}/knowledge0.cfg
 [monitor-conf-parameters]
 private-path-list +=
   ${directory:logrotate-backup}
+
+[post-notification-run]
+recipe = slapos.cookbook:wrapper
+command-line = {{ software_release_bin }}/runner-importer-post-notification-run --diff-file ${:diff-file} --proof-signature-file ${:proof-signature-file} --srv-path ${directory:srv} --backup-path ${directory:backup}
+wrapper-path = ${rootdirectory:bin}/post-notification-run
+output = ${:wrapper-path}
+mode = 0700
\ No newline at end of file

software/slaprunner/instance-runner.cfg

@@ -4,6 +4,8 @@ parts =
   nginx-launcher
   certificate-authority
   ca-nginx
+  certificate-authority-service
+  ca-nginx-service
   logrotate-entry-nginx
   gunicorn-launcher
   gunicorn-graceful
@@ -16,10 +18,13 @@ parts =
   runner-sshd-graceful
   runner-sshd-promise
   runner-sshkeys-authority
+  runner-sshkeys-authority-service
   runner-sshkeys-sshd
+  runner-sshkeys-sshd-service
   runtestsuite
   symlinks
   shellinabox
+  shellinabox-service
   slapos-cfg
   cron-entry-prepare-software
   deploy-instance-parameters
@@ -202,8 +207,9 @@ default_repository_branch = $${slap-parameter:slapos-reference}
 [slaprunner-supervisord-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX hardcoded locations
-command-line = $${buildout:directory}/bin/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
+command-line = $${directory:bin}/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
 wrapper-path = $${directory:services}/slaprunner-supervisord
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 
 [test-runner]
@@ -287,9 +293,15 @@ keys = $${directory:sshkeys}/runner-keys/
 recipe = slapos.cookbook:sshkeys_authority
 request-directory = $${runner-sshkeys-directory:requests}
 keys-directory = $${runner-sshkeys-directory:keys}
-wrapper = $${directory:services}/runner_sshkeys_authority
+wrapper = $${directory:bin}/runner_sshkeys_authority
 keygen-binary = ${openssh:location}/bin/ssh-keygen
 
+[runner-sshkeys-authority-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${runner-sshkeys-authority:wrapper}
+wrapper-path = $${directory:services}/runner-sshkeys-authority
+hash-files = $${buildout:directory}/software_release/buildout.cfg
+
 [runner-sshkeys-sshd]
 <= runner-sshkeys-authority
 recipe = slapos.cookbook:sshkeys_authority.request
@@ -298,7 +310,13 @@ type = rsa
 executable = $${runner-sshd-server:output}
 public-key = $${runner-sshd-raw-server:rsa-keyfile}.pub
 private-key = $${runner-sshd-raw-server:rsa-keyfile}
-wrapper = $${directory:services}/runner-sshd
+wrapper = $${directory:bin}/runner-sshd
+
+[runner-sshkeys-sshd-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${runner-sshkeys-sshd:wrapper}
+wrapper-path = $${directory:services}/runner-sshd
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [runner-sshd-add-authorized-key]
 recipe = slapos.cookbook:dropbear.add_authorized_key
@@ -411,6 +429,7 @@ access-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global
 wait-for-files =
   $${ca-nginx:cert-file}
   $${ca-nginx:key-file}
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [logrotate-entry-apache-httpd]
 <= logrotate-entry-base
@@ -461,6 +480,7 @@ wrapper-path = $${gunicorn:bin_launcher}
 environment = PATH=$${shell-environment:path}
   RUNNER_CONFIG=$${slaprunner:slapos.cfg}
   LANG=en_GB.UTF-8
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [gunicorn-graceful]
 recipe = slapos.cookbook:wrapper
@@ -476,7 +496,7 @@ recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${openssl:location}/bin/openssl
 ca-dir = $${directory:ca-dir}
 requests-directory = $${cadirectory:requests}
-wrapper = $${directory:services}/certificate_authority
+wrapper = $${directory:bin}/certificate_authority
 ca-private = $${cadirectory:private}
 ca-certs = $${cadirectory:certs}
 ca-newcerts = $${cadirectory:newcerts}
@@ -496,10 +516,22 @@ recipe = slapos.cookbook:certificate_authority.request
 key-file = $${cadirectory:certs}/nginx_frontend.key
 cert-file = $${cadirectory:certs}/nginx_frontend.crt
 executable = $${nginx-launcher:rendered}
-wrapper = $${directory:services}/nginx-frontend
+wrapper = $${directory:bin}/nginx-frontend
 # Put domain name
 name = example.com
 
+[ca-nginx-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${directory:bin}/nginx-frontend
+wrapper-path = $${directory:services}/nginx-frontend
+hash-files = $${buildout:directory}/software_release/buildout.cfg
+
+[certificate-authority-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${directory:bin}/certificate_authority
+wrapper-path = $${directory:services}/certificate_authority
+hash-files = $${buildout:directory}/software_release/buildout.cfg
+
 #--------------------
 #--
 #-- Request frontend
@@ -667,7 +699,7 @@ recipe = slapos.recipe.template:jinja2
 # We cannot use slapos.cookbook:wrapper here because this recipe escapes too much
 socket = $${directory:run}/siab.sock
 mode = 0700
-rendered = $${directory:services}/shellinaboxd
+rendered = $${directory:bin}/shellinaboxd
 template = inline:
   #!/bin/sh
   exec ${shellinabox:location}/bin/shellinaboxd \
@@ -676,6 +708,12 @@ template = inline:
     --unixdomain-only=$${:socket}:$(id -u):$(id -g):0600 \
     --service "/:$(id -u):$(id -g):HOME:$${shell-environment:shell} -l"
 
+[shellinabox-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${directory:bin}/shellinaboxd
+wrapper-path = $${directory:services}/shellinaboxd
+hash-files = $${buildout:directory}/software_release/buildout.cfg
+
 [shell-environment]
 shell = ${bash:location}/bin/bash
 path = ${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin:${mosh:location}/bin:${bash:location}/bin:$${buildout:directory}/bin/:/usr/bin:/bin/
@@ -808,6 +846,7 @@ context =
 recipe = slapos.cookbook:wrapper
 command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
 wrapper-path = $${directory:services}/supervisord
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [logrotate-entry-supervisord]
 <= logrotate-entry-base

software/slaprunner/instance.cfg

@@ -56,12 +56,12 @@ context =
   key pbsready_export_template_path template-pbsready-export:rendered
   key template_runner_path instance-base-runner:rendered
   key slapparameter_dict slap-configuration:configuration
+  raw software_release_bin ${buildout:bin-directory}
   raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
-  raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
   raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
   raw buildout_executable_location ${buildout:executable}
   raw bash_executable_location ${bash:location}/bin/bash
-  raw rsync_executable_location ${rsync:location}/bin/rsync
+  raw rsync_bin_folder ${rsync:location}/bin
 
 [template-runner-import]
 recipe = slapos.recipe.template:jinja2
@@ -71,6 +71,7 @@ mode = 640
 context =
   key template_runner_path instance-base-runner:rendered
   key slapparameter_dict slap-configuration:configuration
+  raw software_release_bin ${buildout:bin-directory}
   raw importer_script_path ${template-runner-import-script:location}/${template-runner-import-script:filename}
   raw buildout_executable_location ${buildout:executable}
   raw bash_executable_location ${bash:location}/bin/bash

software/slaprunner/software.cfg

@@ -76,10 +76,6 @@ output = ${buildout:directory}/template-runner.cfg.in
 < = template-download-base
 filename = runner-import.sh.jinja2
 
-[template-runner-export-script]
-< = template-download-base
-filename = runner-export.sh.jinja2
-
 [instance-runner-import]
 < = download-base
 recipe = slapos.recipe.build:download
@@ -167,42 +163,6 @@ prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 4.3
 collective.recipe.environment = 0.2.0
-slapos.toolbox = 0.81
 smmap = 0.9.0
 lockfile = 0.12.2
 
-# Required by:
-# slapos.toolbox==0.71
-GitPython = 2.0.8
-
-# Required by:
-# slapos.toolbox==0.71
-PyRSS2Gen = 1.1
-
-# Required by:
-# slapos.toolbox==0.71
-atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.71
-dnspython = 1.14.0
-
-# Required by:
-# slapos.toolbox==0.71
-erp5.util = 0.4.51
-
-# Required by:
-# slapos.toolbox==0.71
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.71
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.71
-paramiko = 2.0.2
-
-# Required by:
-# slapos.toolbox==0.71
-passlib = 1.6.5

software/slaprunner/template/runner-export.sh.jinja2

-#!{{ shell_binary }}
-LC_ALL=C
-export LC_ALL
-umask 077
-
-# Exit on any error, to prevent inconsistent backup
-# Error on unset variable expansion
-set -eu
-set -o pipefail
-
-# Redirect output to log
-exec > >(tee -ai '{{ output_log_file }}')
-exec 2>&1
-
-echo -e "\n\n$0 run at : $(date)"
-
-srv_directory='{{ directory["srv"] }}'
-backup_directory='{{ directory["backup"] }}'
-etc_directory='{{ directory["etc"] }}'
-tmp_directory='{{ directory["tmp"] }}'
-
-rsync () {
-  # Workaround for bug https://bugzilla.samba.org/show_bug.cgi?id=3653
-  IGNOREEXIT=24
-  IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
-
-  set -x
-  '{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@" 2>&1 | (egrep -v "$IGNOREOUT" || true) || [ $? = "$IGNOREEXIT" ]
-  set +x
-}
-
-(
-  # XXX: code duplication with runner-import.sh.jinja2
-  path=$srv_directory/runner
-  backup_path=$backup_directory/runner/
-  cd "$path"
-
-  if [ -d instance ]; then
-    # Concatenate the exclude file of each partition of webrunner
-    # to create a global exclude file.
-    # Also, ignore all buildout-managed files.
-    exclude=$({{ sys.executable }} - "$path" <<EOF
-if 1:
-        import glob, errno, os, sys
-        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
-        from zc.buildout.configparser import parse
-        path = sys.argv[1]
-
-        def print_relative(path_list):
-            for p in path_list:
-                p = p.strip()
-                if p:
-                    print(os.path.relpath(p, path))
-        print("*.sock")
-        print("*.socket")
-        print("*.pid")
-        print(".installed*.cfg")
-        for partition in glob.glob(path + "/instance/slappart*"):
-            try:
-              os.chdir(partition)
-            except OSError as e:
-              if e.errno != errno.ENOTDIR:
-                raise
-              continue
-            try:
-                with open("srv/exporter.exclude") as f:
-                    exclude = f.readlines()
-            except IOError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-            else:
-                print_relative(exclude)
-            for installed in glob.glob(".installed*.cfg"):
-                try:
-                    with open(installed) as f:
-                        installed = parse(f, installed)
-                except IOError as e:
-                    if e.errno != errno.ENOENT:
-                        raise
-                else:
-                    for section in installed.itervalues():
-                        print_relative(section.get(
-                            '__buildout_installed__', '').splitlines())
-EOF
-)
-    echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
-  fi
-
-  test -d project  && rsync project "$backup_path"
-  test -d public  && rsync public "$backup_path"
-  test -f proxy.db && rsync proxy.db "$backup_path"
-)
-# We sync .* appart
-(
-  cd "$etc_directory"
-  date +%s -u > .resilient-timestamp
-  rsync config.json "$backup_directory"/etc/
-  # Hidden files are related to the webrunner's internals
-  cp -r .??* "$backup_directory/etc/"
-)
-if [ -d "$backup_directory"/runner/software ]; then
-  rm "$backup_directory"/runner/software/*
-fi
-
-(
-  cd "$backup_directory"
-  find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | sort -k 66 > backup.signature
-)
-
-# Check that export didn't happen during backup of instances
-tmp_backup_sum=$(mktemp -p "$tmp_directory")
-
-remove_tmp_files () {
-  rm "$tmp_backup_sum"
-}
-trap remove_tmp_files EXIT
-
-cd {{ directory['backup'] }}
-
-# Wait a little to increase the probability to detect an ongoing backup.
-sleep 5
-
-# Getting files from runner backup directory, as instance backup files may be
-# explicitely excluded from the backup, using the srv/exporter.exclude
-find -path "./runner/instance/slappart*/srv/backup/*" -type f ! -name backup.signature -print0 |
-xargs -r0 sha256sum | sort -k 66 > "$tmp_backup_sum"
-
-# If no backup found, it's over
-if [ ! -s "$tmp_backup_sum" ]; then
-  exit 0
-fi
-
-# If the diff fails, then the notifier will restart this script
-grep "instance/slappart.*/srv/backup/" "$backup_directory/backup.signature" |
-diff "$tmp_backup_sum" - || {
-  echo "ERROR: Some backups are not consistent, exporter should be re-run."
-  echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
-  sleep {{ backup_wait_time }}m
-  exit 1
-}

software/slaprunner/test/README.md

+Tests for slaprunner Software Release

software/slaprunner/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.slaprunner'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' slaprunner",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+          'slapos.core',
+          'slapos.libnetworkcache',
+          'erp5.util',
+          'supervisor',
+          'psutil',
+      ],
+      zip_safe=True,
+      test_suite='test',
+      )

software/slaprunner/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import shutil
+import urlparse
+import tempfile
+import requests
+import socket
+import StringIO
+import subprocess
+import json
+
+import psutil
+
+import utils
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+
+class InstanceTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+class ServicesTestCase(InstanceTestCase):
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  def test_hashes(self):
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    expected_process_names = [
+      'slaprunner-supervisord-{hash}-on-watch',
+      'runner-sshkeys-authority-{hash}-on-watch',
+      'runner-sshd-{hash}-on-watch',
+      'slaprunner-httpd-{hash}-on-watch',
+      'gunicorn-{hash}-on-watch',
+      'nginx-frontend-{hash}-on-watch',
+      'certificate_authority-{hash}-on-watch',
+      'shellinaboxd-{hash}-on-watch',
+      'supervisord-{hash}-on-watch',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_names = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+
+    for name in expected_process_names:
+      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_names)

software/slaprunner/test/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+import StringIO
+import xmlrpclib
+
+import supervisor.xmlrpc
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+
+
+# Utility functions
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+# TODO:
+#  - allow requesting multiple instances ?
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  """Install one slapos instance.
+
+  This test case install software(s) and request one instance during `setUpClass`
+  and destroy the instance during `tearDownClass`.
+
+  Software Release URL, Instance Software Type and Instance Parameters can be defined
+  on the class.
+
+  All tests from the test class will run with the same instance.
+
+  The following class attributes are available:
+
+    * `computer_partition`:  the computer partition instance, implementing
+    `slapos.slap.interface.slap.IComputerPartition`.
+
+    * `computer_partition_root_path`: the path of the instance root directory.
+
+  """
+
+  # Methods to be defined by subclasses.
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    """Return software type for instance, default "default"
+
+    To be defined by subclasses if they need to request instance with specific
+    software type.
+    """
+    return "default"
+
+  # Utility methods.
+  def getSupervisorRPCServer(self):
+    """Returns a XML-RPC connection to the supervisor used by slapos node
+
+    Refer to http://supervisord.org/api.html for details of available methods.
+    """
+    # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
+    return xmlrpclib.ServerProxy(
+       'http://slapos-supervisor',
+       transport=supervisor.xmlrpc.SupervisorTransport(
+           None,
+           None,
+           # XXX hardcoded socket path
+           serverurl="unix://{working_directory}/inst/supervisord.socket".format(
+             **self.config)))
+
+  # Unittest methods
+  @classmethod
+  def setUpClass(cls):
+    """Setup the class, build software and request an instance.
+
+    If you have to override this method, do not forget to call this method on
+    parent class.
+    """
+    try:
+      cls.setUpWorkingDirectory()
+      cls.setUpConfig()
+      cls.setUpSlapOSController()
+
+      cls.runSoftwareRelease()
+      # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
+      # cls.supplySoftwares()
+      # cls.installSoftwares()
+
+      cls.runComputerPartition()
+      # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
+      # cls.requestInstances()
+      # cls.createInstances()
+      # cls.requestInstances()
+
+    except Exception:
+      cls.stopSlapOSProcesses()
+      raise
+
+  @classmethod
+  def tearDownClass(cls):
+    """Tear down class, stop the processes and destroy instance.
+    """
+    cls.stopSlapOSProcesses()
+
+  # Implementation
+  @classmethod
+  def stopSlapOSProcesses(cls):
+    if hasattr(cls, '_process_manager'):
+      cls._process_manager.killPreviousRun()
+
+  @classmethod
+  def setUpWorkingDirectory(cls):
+    """Initialise the directories"""
+    cls.working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    # Supervisord socket name contains the pid number, which is why we add
+    # .xxxxxxx in this check.
+    if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
+      raise RuntimeError('working directory ( {} ) is too deep, try setting '
+              'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
+
+    if not os.path.exists(cls.working_directory):
+      os.mkdir(cls.working_directory)
+
+  @classmethod
+  def setUpConfig(cls):
+    """Create slapos configuration"""
+    cls.config = {
+      "working_directory": cls.working_directory,
+      "slapos_directory": cls.working_directory,
+      "log_directory": cls.working_directory,
+      "computer_id": 'slapos.test',  # XXX
+      'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
+      'partition_reference': cls.__name__,
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+    }
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['GLOBAL_IPV6']
+
+    cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
+    cls.config['ipv6_address'] = ipv6_address
+    cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
+      **cls.config)
+
+  @classmethod
+  def setUpSlapOSController(cls):
+    """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
+
+    This is equivalent to:
+
+    slapos proxy start
+    for sr in getSoftwareURLList; do
+      slapos supply $SR $COMP
+    done
+    """
+    cls._process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    cls.slapos_controler = SlapOSControler(
+        cls.working_directory,
+        cls.config
+    )
+
+    slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    cls.software_url_list = cls.getSoftwareURLList()
+    cls.slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=cls._process_manager,
+        reset_software=False,
+        software_path_list=cls.software_url_list)
+
+    # XXX we should check *earlier* if that pidfile exist and if supervisord
+    # process still running, because if developer started supervisord (or bugs?)
+    # then another supervisord will start and starting services a second time
+    # will fail.
+    cls._process_manager.supervisord_pid_file = os.path.join(
+      cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+  @classmethod
+  def runSoftwareRelease(cls):
+    """Run all the software releases that were supplied before.
+
+    This is the equivalent of `slapos node software`.
+
+    The tests will be marked file if software building fail.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    try:
+      cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
+        cls.config, environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.software_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+
+  @classmethod
+  def runComputerPartition(cls):
+    """Instanciate the software.
+
+    This is the equivalent of doing:
+
+    slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
+    slapos node instance
+
+    and return the slapos request instance parameters.
+
+    This can be called by tests to simulate re-request with different parameters.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    if cls.getInstanceSoftwareType() != 'default':
+      raise NotImplementedError
+
+    instance_parameter_dict = cls.getInstanceParameterDict()
+    try:
+      cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
+        cls.config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.instance_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+    # FIXME: similar to test node, only one (root) partition is really
+    #        supported for now.
+    computer_partition_list = []
+    for i in range(len(cls.software_url_list)):
+      computer_partition_list.append(
+          cls.slapos_controler.slap.registerOpenOrder().request(
+            cls.software_url_list[i],
+            # This is how testnode's SlapOSControler name created partitions
+            partition_reference='testing partition {i}'.format(
+              i=i, **cls.config),
+            partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParameterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        cls.config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+
+

software/theia/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance]
+filename = instance.cfg.in
+md5sum = c35e6baca37c690bce39ef43825ec42b
+
+[yarn.lock]
+filename = yarn.lock
+md5sum = eb30f0839ef6ce19b89757cf027e3302
+
+[python-language-server-requirements.txt]
+filename = python-language-server-requirements.txt
+md5sum = c21a684783b3e746134c3505ce00b757

software/theia/gowork.cfg

+# Code generated by gowork-snapshot; DO NOT EDIT.
+
+# list of go git repositories to fetch
+[gowork.goinstall]
+depends_gitfetch  =
+    ${go_github.com_9fans_go:recipe}
+    ${go_github.com_acroca_go-symbols:recipe}
+    ${go_github.com_alecthomas_gometalinter:recipe}
+    ${go_github.com_alecthomas_units:recipe}
+    ${go_github.com_beorn7_perks:recipe}
+    ${go_github.com_cosiner_argv:recipe}
+    ${go_github.com_cweill_gotests:recipe}
+    ${go_github.com_davidrjenni_reftools:recipe}
+    ${go_github.com_derekparker_delve:recipe}
+    ${go_github.com_fatih_camelcase:recipe}
+    ${go_github.com_fatih_gomodifytags:recipe}
+    ${go_github.com_fatih_structtag:recipe}
+    ${go_github.com_flynn_go-shlex:recipe}
+    ${go_github.com_gogo_protobuf:recipe}
+    ${go_github.com_golang_lint:recipe}
+    ${go_github.com_golang_protobuf:recipe}
+    ${go_github.com_google_shlex:recipe}
+    ${go_github.com_google_uuid:recipe}
+    ${go_github.com_hashicorp_golang-lru:recipe}
+    ${go_github.com_haya14busa_goplay:recipe}
+    ${go_github.com_josharian_impl:recipe}
+    ${go_github.com_karrick_godirwalk:recipe}
+    ${go_github.com_kisielk_gotool:recipe}
+    ${go_github.com_mattn_go-isatty:recipe}
+    ${go_github.com_mattn_go-runewidth:recipe}
+    ${go_github.com_matttproud_golang_protobuf_extensions:recipe}
+    ${go_github.com_mholt_caddy:recipe}
+    ${go_github.com_neelance_parallel:recipe}
+    ${go_github.com_nicksnyder_go-i18n:recipe}
+    ${go_github.com_nsf_gocode:recipe}
+    ${go_github.com_opentracing_basictracer-go:recipe}
+    ${go_github.com_opentracing_opentracing-go:recipe}
+    ${go_github.com_pelletier_go-toml:recipe}
+    ${go_github.com_peterh_liner:recipe}
+    ${go_github.com_pkg_errors:recipe}
+    ${go_github.com_pmezard_go-difflib:recipe}
+    ${go_github.com_prometheus_client_golang:recipe}
+    ${go_github.com_prometheus_client_model:recipe}
+    ${go_github.com_prometheus_common:recipe}
+    ${go_github.com_prometheus_procfs:recipe}
+    ${go_github.com_ramya-rao-a_go-outline:recipe}
+    ${go_github.com_rogpeppe_godef:recipe}
+    ${go_github.com_sirupsen_logrus:recipe}
+    ${go_github.com_skratchdot_open-golang:recipe}
+    ${go_github.com_slimsag_godocmd:recipe}
+    ${go_github.com_sourcegraph_ctxvfs:recipe}
+    ${go_github.com_sourcegraph_go-langserver:recipe}
+    ${go_github.com_sourcegraph_go-lsp:recipe}
+    ${go_github.com_sourcegraph_jsonrpc2:recipe}
+    ${go_github.com_spf13_cobra:recipe}
+    ${go_github.com_spf13_pflag:recipe}
+    ${go_github.com_sqs_goreturns:recipe}
+    ${go_github.com_tylerb_gotype-live:recipe}
+    ${go_github.com_uudashr_gopkgs:recipe}
+    ${go_github.com_zmb3_gogetdoc:recipe}
+    ${go_golang.org_x_arch:recipe}
+    ${go_golang.org_x_crypto:recipe}
+    ${go_golang.org_x_lint:recipe}
+    ${go_golang.org_x_net:recipe}
+    ${go_golang.org_x_sys:recipe}
+    ${go_golang.org_x_tools:recipe}
+    ${go_gopkg.in_alecthomas_kingpin.v3-unstable:recipe}
+    ${go_gopkg.in_yaml.v2:recipe}
+    ${go_honnef.co_go_tools:recipe}
+    ${go_sourcegraph.com_sqs_goreturns:recipe}
+
+
+[go_github.com_9fans_go]
+<= go-git-package
+go.importpath = github.com/9fans/go
+repository    = https://github.com/9fans/go
+revision      = 5d4fa602e1
+
+[go_github.com_acroca_go-symbols]
+<= go-git-package
+go.importpath = github.com/acroca/go-symbols
+repository    = https://github.com/acroca/go-symbols
+revision      = 953befd75e
+
+[go_github.com_alecthomas_gometalinter]
+<= go-git-package
+go.importpath = github.com/alecthomas/gometalinter
+repository    = https://github.com/alecthomas/gometalinter
+revision      = 8edca99e8a
+
+[go_github.com_alecthomas_units]
+<= go-git-package
+go.importpath = github.com/alecthomas/units
+repository    = https://github.com/alecthomas/units
+revision      = 2efee857e7
+
+[go_github.com_beorn7_perks]
+<= go-git-package
+go.importpath = github.com/beorn7/perks
+repository    = https://github.com/beorn7/perks
+revision      = 3a771d9929
+
+[go_github.com_cosiner_argv]
+<= go-git-package
+go.importpath = github.com/cosiner/argv
+repository    = https://github.com/cosiner/argv
+revision      = 13bacc38a0
+
+[go_github.com_cweill_gotests]
+<= go-git-package
+go.importpath = github.com/cweill/gotests
+repository    = https://github.com/cweill/gotests
+revision      = c51312f508
+
+[go_github.com_davidrjenni_reftools]
+<= go-git-package
+go.importpath = github.com/davidrjenni/reftools
+repository    = https://github.com/davidrjenni/reftools
+revision      = 654d0ba4f9
+
+[go_github.com_derekparker_delve]
+<= go-git-package
+go.importpath = github.com/derekparker/delve
+repository    = https://github.com/derekparker/delve
+revision      = ea3428550d
+
+[go_github.com_fatih_camelcase]
+<= go-git-package
+go.importpath = github.com/fatih/camelcase
+repository    = https://github.com/fatih/camelcase
+revision      = 9db1b65eb3
+
+[go_github.com_fatih_gomodifytags]
+<= go-git-package
+go.importpath = github.com/fatih/gomodifytags
+repository    = https://github.com/fatih/gomodifytags
+revision      = 141225bf62
+
+[go_github.com_fatih_structtag]
+<= go-git-package
+go.importpath = github.com/fatih/structtag
+repository    = https://github.com/fatih/structtag
+revision      = 76ae1d6d21
+
+[go_github.com_flynn_go-shlex]
+<= go-git-package
+go.importpath = github.com/flynn/go-shlex
+repository    = https://github.com/flynn/go-shlex
+revision      = 3f9db97f85
+
+[go_github.com_gogo_protobuf]
+<= go-git-package
+go.importpath = github.com/gogo/protobuf
+repository    = https://github.com/gogo/protobuf
+revision      = 6f222ca738
+
+[go_github.com_golang_lint]
+<= go-git-package
+go.importpath = github.com/golang/lint
+repository    = https://github.com/golang/lint
+revision      = 1baf3a9d7d
+
+[go_github.com_golang_protobuf]
+<= go-git-package
+go.importpath = github.com/golang/protobuf
+repository    = https://github.com/golang/protobuf
+revision      = 5e0eda4b6d
+
+[go_github.com_google_shlex]
+<= go-git-package
+go.importpath = github.com/google/shlex
+repository    = https://github.com/google/shlex
+revision      = 6f45313302
+
+[go_github.com_google_uuid]
+<= go-git-package
+go.importpath = github.com/google/uuid
+repository    = https://github.com/google/uuid
+revision      = 9b3b1e0f5f
+
+[go_github.com_hashicorp_golang-lru]
+<= go-git-package
+go.importpath = github.com/hashicorp/golang-lru
+repository    = https://github.com/hashicorp/golang-lru
+revision      = 20f1fb78b0
+
+[go_github.com_haya14busa_goplay]
+<= go-git-package
+go.importpath = github.com/haya14busa/goplay
+repository    = https://github.com/haya14busa/goplay
+revision      = v1.0.0-0-gf2d63a841a
+
+[go_github.com_josharian_impl]
+<= go-git-package
+go.importpath = github.com/josharian/impl
+repository    = https://github.com/josharian/impl
+revision      = 3d0f908298
+
+[go_github.com_karrick_godirwalk]
+<= go-git-package
+go.importpath = github.com/karrick/godirwalk
+repository    = https://github.com/karrick/godirwalk
+revision      = v1.7.5-0-g2de2192f9e
+
+[go_github.com_kisielk_gotool]
+<= go-git-package
+go.importpath = github.com/kisielk/gotool
+repository    = https://github.com/kisielk/gotool
+revision      = 80517062f5
+
+[go_github.com_mattn_go-isatty]
+<= go-git-package
+go.importpath = github.com/mattn/go-isatty
+repository    = https://github.com/mattn/go-isatty
+revision      = 3fb116b820
+
+[go_github.com_mattn_go-runewidth]
+<= go-git-package
+go.importpath = github.com/mattn/go-runewidth
+repository    = https://github.com/mattn/go-runewidth
+revision      = b20a3daf6a
+
+[go_github.com_matttproud_golang_protobuf_extensions]
+<= go-git-package
+go.importpath = github.com/matttproud/golang_protobuf_extensions
+repository    = https://github.com/matttproud/golang_protobuf_extensions
+revision      = v1.0.0-2-gc12348ce28
+
+[go_github.com_mholt_caddy]
+<= go-git-package
+go.importpath = github.com/mholt/caddy
+repository    = https://lab.nexedi.com/nexedi/caddy.git
+revision      = v0.11.0-3-g12438f6cff
+
+[go_github.com_neelance_parallel]
+<= go-git-package
+go.importpath = github.com/neelance/parallel
+repository    = https://github.com/neelance/parallel
+revision      = 4de9ce63d1
+
+[go_github.com_nicksnyder_go-i18n]
+<= go-git-package
+go.importpath = github.com/nicksnyder/go-i18n
+repository    = https://github.com/nicksnyder/go-i18n
+revision      = fc57a7d765
+
+[go_github.com_nsf_gocode]
+<= go-git-package
+go.importpath = github.com/nsf/gocode
+repository    = https://github.com/nsf/gocode
+revision      = 7b1d4e18cd
+
+[go_github.com_opentracing_basictracer-go]
+<= go-git-package
+go.importpath = github.com/opentracing/basictracer-go
+repository    = https://github.com/opentracing/basictracer-go
+revision      = 98b91394c2
+
+[go_github.com_opentracing_opentracing-go]
+<= go-git-package
+go.importpath = github.com/opentracing/opentracing-go
+repository    = https://github.com/opentracing/opentracing-go
+revision      = be550b025b
+
+[go_github.com_pelletier_go-toml]
+<= go-git-package
+go.importpath = github.com/pelletier/go-toml
+repository    = https://github.com/pelletier/go-toml
+revision      = v0.2.0-212-g81a861c69d
+
+[go_github.com_peterh_liner]
+<= go-git-package
+go.importpath = github.com/peterh/liner
+repository    = https://github.com/peterh/liner
+revision      = v1.1.0-0-g5a0dfa99e2
+
+[go_github.com_pkg_errors]
+<= go-git-package
+go.importpath = github.com/pkg/errors
+repository    = https://github.com/pkg/errors
+revision      = v0.8.0-17-g059132a15d
+
+[go_github.com_pmezard_go-difflib]
+<= go-git-package
+go.importpath = github.com/pmezard/go-difflib
+repository    = https://github.com/pmezard/go-difflib
+revision      = v1.0.0-0-g792786c740
+
+[go_github.com_prometheus_client_golang]
+<= go-git-package
+go.importpath = github.com/prometheus/client_golang
+repository    = https://github.com/prometheus/client_golang
+revision      = v0.9.0-6-g16f375c74d
+
+[go_github.com_prometheus_client_model]
+<= go-git-package
+go.importpath = github.com/prometheus/client_model
+repository    = https://github.com/prometheus/client_model
+revision      = model-0.0.2-18-g5c3871d899
+
+[go_github.com_prometheus_common]
+<= go-git-package
+go.importpath = github.com/prometheus/common
+repository    = https://github.com/prometheus/common
+revision      = 7e9e6cabbd
+
+[go_github.com_prometheus_procfs]
+<= go-git-package
+go.importpath = github.com/prometheus/procfs
+repository    = https://github.com/prometheus/procfs
+revision      = 185b428841
+
+[go_github.com_ramya-rao-a_go-outline]
+<= go-git-package
+go.importpath = github.com/ramya-rao-a/go-outline
+repository    = https://github.com/ramya-rao-a/go-outline
+revision      = 9e9d089bb6
+
+[go_github.com_rogpeppe_godef]
+<= go-git-package
+go.importpath = github.com/rogpeppe/godef
+repository    = https://github.com/rogpeppe/godef
+revision      = v1.0.0-0-g7b4626be9f
+
+[go_github.com_sirupsen_logrus]
+<= go-git-package
+go.importpath = github.com/sirupsen/logrus
+repository    = https://github.com/sirupsen/logrus
+revision      = 4fabf2fffc
+
+[go_github.com_skratchdot_open-golang]
+<= go-git-package
+go.importpath = github.com/skratchdot/open-golang
+repository    = https://github.com/skratchdot/open-golang
+revision      = 75fb7ed420
+
+[go_github.com_slimsag_godocmd]
+<= go-git-package
+go.importpath = github.com/slimsag/godocmd
+repository    = https://github.com/slimsag/godocmd
+revision      = a1005ad29f
+
+[go_github.com_sourcegraph_ctxvfs]
+<= go-git-package
+go.importpath = github.com/sourcegraph/ctxvfs
+repository    = https://github.com/sourcegraph/ctxvfs
+revision      = 2b65f1b1ea
+
+[go_github.com_sourcegraph_go-langserver]
+<= go-git-package
+go.importpath = github.com/sourcegraph/go-langserver
+repository    = https://github.com/sourcegraph/go-langserver
+revision      = 2b83206020
+
+[go_github.com_sourcegraph_go-lsp]
+<= go-git-package
+go.importpath = github.com/sourcegraph/go-lsp
+repository    = https://github.com/sourcegraph/go-lsp
+revision      = 4631ffd93a
+
+[go_github.com_sourcegraph_jsonrpc2]
+<= go-git-package
+go.importpath = github.com/sourcegraph/jsonrpc2
+repository    = https://github.com/sourcegraph/jsonrpc2
+revision      = 549eb959f0
+
+[go_github.com_spf13_cobra]
+<= go-git-package
+go.importpath = github.com/spf13/cobra
+repository    = https://github.com/spf13/cobra
+revision      = fe5e611709
+
+[go_github.com_spf13_pflag]
+<= go-git-package
+go.importpath = github.com/spf13/pflag
+repository    = https://github.com/spf13/pflag
+revision      = 082b515c94
+
+[go_github.com_sqs_goreturns]
+<= go-git-package
+go.importpath = github.com/sqs/goreturns
+repository    = https://github.com/sqs/goreturns
+revision      = 83e02874ec
+
+[go_github.com_tylerb_gotype-live]
+<= go-git-package
+go.importpath = github.com/tylerb/gotype-live
+repository    = https://github.com/tylerb/gotype-live
+revision      = 440f9c77a4
+
+[go_github.com_uudashr_gopkgs]
+<= go-git-package
+go.importpath = github.com/uudashr/gopkgs
+repository    = https://github.com/uudashr/gopkgs
+revision      = 84fe2e5def
+
+[go_github.com_zmb3_gogetdoc]
+<= go-git-package
+go.importpath = github.com/zmb3/gogetdoc
+repository    = https://github.com/zmb3/gogetdoc
+revision      = 0d07153ccc
+
+[go_golang.org_x_arch]
+<= go-git-package
+go.importpath = golang.org/x/arch
+repository    = https://go.googlesource.com/arch
+revision      = b19384d3c1
+
+[go_golang.org_x_crypto]
+<= go-git-package
+go.importpath = golang.org/x/crypto
+repository    = https://go.googlesource.com/crypto
+revision      = 85e1b3f913
+
+[go_golang.org_x_lint]
+<= go-git-package
+go.importpath = golang.org/x/lint
+repository    = https://go.googlesource.com/lint
+revision      = 1baf3a9d7d
+
+[go_golang.org_x_net]
+<= go-git-package
+go.importpath = golang.org/x/net
+repository    = https://go.googlesource.com/net
+revision      = 9b4f9f5ad5
+
+[go_golang.org_x_sys]
+<= go-git-package
+go.importpath = golang.org/x/sys
+repository    = https://go.googlesource.com/sys
+revision      = d989b31c87
+
+[go_golang.org_x_tools]
+<= go-git-package
+go.importpath = golang.org/x/tools
+repository    = https://go.googlesource.com/tools
+revision      = a2dc47679d
+
+[go_gopkg.in_alecthomas_kingpin.v3-unstable]
+<= go-git-package
+go.importpath = gopkg.in/alecthomas/kingpin.v3-unstable
+repository    = https://gopkg.in/alecthomas/kingpin.v3-unstable
+revision      = df19058c87
+
+[go_gopkg.in_yaml.v2]
+<= go-git-package
+go.importpath = gopkg.in/yaml.v2
+repository    = https://gopkg.in/yaml.v2
+revision      = v2.1.1-17-g5420a8b674
+
+[go_honnef.co_go_tools]
+<= go-git-package
+go.importpath = honnef.co/go/tools
+repository    = https://github.com/dominikh/go-tools
+revision      = e3ad64cb4e
+
+[go_sourcegraph.com_sqs_goreturns]
+<= go-git-package
+go.importpath = sourcegraph.com/sqs/goreturns
+repository    = https://github.com/sqs/goreturns
+revision      = 83e02874ec

software/theia/instance.cfg.in

+[buildout]
+parts =
+  promises
+  .bashrc
+  frontend-reload
+  publish-connection-parameter
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory} 
+offline = true
+
+
+[frontend-instance-password]
+recipe = slapos.cookbook:generate.password
+username = node
+bytes = 12
+
+[frontend-instance-config]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:etc}/$${:_buildout_section_name_}
+template = inline:
+  https://$${:hostname}:$${:port} {
+    bind $${:ip}
+    tls self_signed # TODO
+    log stdout
+    errors stderr
+    gzip
+    # because caddy does not support upgrade http2 to websocket
+    # https://tools.ietf.org/html/rfc8441
+    tls {
+      alpn http/1.1
+    }
+    proxy / $${theia-instance:base-url} {
+    #  transparent
+    }
+    proxy /services $${theia-instance:base-url} {
+      websocket
+    }
+    basicauth $${frontend-instance-password:username} $${frontend-instance-password:passwd} {
+      realm "Theia"
+      /
+    }
+  }
+ip = $${instance-parameter:ipv6-random}
+hostname = [$${:ip}]
+port = 3001
+
+[frontend-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line =
+  ${caddy:output} -conf $${frontend-instance-config:rendered} -pidfile $${:pidfile}
+
+ip = $${frontend-instance-config:ip}
+hostname = $${frontend-instance-config:hostname}
+port = $${frontend-instance-config:port}
+pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid
+url = https://$${frontend-instance-password:username}:$${frontend-instance-password:passwd}@$${:hostname}:$${:port}/
+
+[frontend-reload]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line =
+  ${bash:location}/bin/bash -c
+  "kill -s USR1 $$(${coreutils:location}/bin/cat $${frontend-instance:pidfile}) \
+    && ${coreutils:location}/bin/sleep infinity"
+hash-files =
+  $${frontend-instance-config:rendered}
+  $${frontend-instance:wrapper-path}
+wait-for-files = $${frontend-instance:pidfile}
+
+[user]
+recipe = slapos.cookbook:userinfo
+
+[theia-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line =
+  env -i HOME=$${directory:home} LC_ALL=C.UTF-8 USER=$${user:pw-name} LOGNAME=$${user:pw-name} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
+
+ip =  $${instance-parameter:ipv4-random}
+hostname =  $${:ip}
+port = 3000
+base-url = http://$${:hostname}:$${:port}/
+
+[.bashrc]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:home}/$${:_buildout_section_name_}
+template = inline:
+  export PS1="$ " # because we are in a gowork workspace
+  # XXX .bash_profile is not executed, so we introduce a bashrc.theia file
+  # to allow customizations.
+  if [ -f "$HOME/.bashrc.theia" ] ; then
+    source $HOME/.bashrc.theia
+  fi
+  # If not running interactively, don't do anything
+  case $- in
+      *i*) ;;
+        *) return;;
+  esac
+  # enable color support
+  if [ -x /usr/bin/dircolors ]; then
+      test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
+      alias ls='ls --color=auto'
+  fi
+  # common ls aliases
+  alias ll='ls -l'
+  alias la='ls -A'
+  alias l='ls -CF'
+
+[promises]
+recipe =
+instance-promises =
+  $${theia-listen-promise:path}
+  $${frontend-listen-promise:path}
+
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${directory:promises}/$${:_buildout_section_name_}
+
+[theia-listen-promise]
+<= check-port-listening-promise
+hostname= $${theia-instance:ip}
+port = $${theia-instance:port}
+
+[frontend-listen-promise]
+<= check-port-listening-promise
+hostname= $${frontend-instance:ip}
+port = $${frontend-instance:port}
+
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = $${frontend-instance:url}
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+pidfiles = $${:var}/run
+
+services = $${:etc}/service
+promises = $${:etc}/promise
+framebuffer = $${:srv}/framebuffer
+fonts = $${:srv}/fonts
+home = $${:srv}/home
\ No newline at end of file

software/theia/python-language-server-requirements.txt

+autopep8==1.4.2
+flake8==3.6.0
+future==0.16.0
+jedi==0.13.1
+mccabe==0.6.1
+parso==0.3.1
+pluggy==0.8.0
+pycodestyle==2.4.0
+pydocstyle==3.0.0
+pyflakes==2.0.0
+python-language-server==0.19.0
+rope==0.11.0
+six==1.11.0
+snowballstemmer==1.2.1
+yapf==0.24.0

software/theia/software.cfg

+[buildout]
+extends =
+   ../../component/python3/buildout.cfg
+   ../../component/python-2.7/buildout.cfg
+   ../../component/nodejs/buildout.cfg
+   ../../component/caddy/buildout.cfg
+   ../../component/git/buildout.cfg
+   ../../component/bash/buildout.cfg
+   ../../component/coreutils/buildout.cfg
+   ../../stack/slapos.cfg
+   ./gowork.cfg
+   ./buildout.hash.cfg
+# this gowork.cfg includes the one from caddy, because they share the only gowork
+# workspace (not intentionnaly, as far as I see there's only one gowork per SR)
+# it is included after caddy, otherwise only caddy is installed. The problem of this
+# approach is that caddy's version will be the one pinned here, so we have to update
+# here as well.
+
+parts =
+  theia-wrapper
+  slapos-cookbook
+  instance
+
+
+[nodejs]
+<= nodejs-8.9.4
+
+[python3]
+<= python3.6.6
+
+
+[yarn]
+# this could become a component, but it needs to be invoked from nodejs explicitly,
+# otherwise it uses system's nodejs
+# XXX why don't we build a wrapper ?
+version = 1.11.0
+recipe = slapos.recipe.build:download-unpacked
+url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
+md5sum = d4f05075f534dd9a0a8c18c650b55f0d
+
+
+[python-language-server]
+version = 0.19.0
+recipe = plone.recipe.command
+command =
+  bash -c "${python3:executable} -m venv ${:location} && \
+    . ${:location}/bin/activate && \
+    pip install -r ${python-language-server-requirements.txt:output}"
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+
+[python-language-server-requirements.txt]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}
+mode = 0644
+
+
+[theia]
+recipe = plone.recipe.command
+command = ${bash:location}/bin/bash -c "
+  export PATH=${nodejs:location}/bin/:${python2.7:location}/bin/:$PATH &&
+  mkdir -p ${:location} && \
+  cd ${:location} && \
+  cp ${package.json:rendered} . &&
+  cp ${yarn.lock:output} . &&
+  ${yarn:location}/bin/yarn && \
+  ${yarn:location}/bin/yarn theia build"
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+
+[yarn.lock]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}
+mode = 0644
+
+[package.json]
+recipe = slapos.recipe.template:jinja2
+# this comes from https://github.com/theia-ide/theia-apps/blob/598d4dc9d4f9e0514869273c069f734a41f20207/theia-full-docker/next.package.json
+# but "@theia/php": "next" was removed, because we don't have php/composer component so installation failed
+template =
+  inline:
+  {
+    "private": true,
+    "dependencies": {
+      "typescript": "latest",
+      "@theia/callhierarchy": "next",
+      "@theia/core": "next",
+      "@theia/cpp": "next",
+      "@theia/docker": "next",
+      "@theia/editor": "next",
+      "@theia/editorconfig": "next",
+      "@theia/extension-manager": "next",
+      "@theia/file-search": "next",
+      "@theia/filesystem": "next",
+      "@theia/git": "next",
+      "@theia/go": "next",
+      "@theia/java": "next",
+      "@theia/json": "next",
+      "@theia/keymaps": "next",
+      "@theia/languages": "next",
+      "@theia/markers": "next",
+      "@theia/merge-conflicts": "next",
+      "@theia/messages": "next",
+      "@theia/metrics": "next",
+      "@theia/mini-browser": "next",
+      "@theia/monaco": "next",
+      "@theia/navigator": "next",
+      "@theia/outline-view": "next",
+      "@theia/output": "next",
+      "@theia/plantuml": "next",
+      "@theia/preferences": "next",
+      "@theia/preview": "next",
+      "@theia/process": "next",
+      "@theia/python": "next",
+      "@theia/ruby": "next",
+      "@theia/rust": "next",
+      "@theia/search-in-workspace": "next",
+      "@theia/task": "next",
+      "@theia/terminal": "next",
+      "@theia/textmate-grammars": "next",
+      "@theia/tslint": "next",
+      "@theia/typescript": "next",
+      "@theia/userstorage": "next",
+      "@theia/variable-resolver": "next",
+      "@theia/workspace": "next",
+      "theia-yang-extension": "next"
+    },
+    "devDependencies": {
+      "@theia/cli": "next"
+    }
+  }
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+mode = 0644
+
+
+[gowork]
+# Install go-language-server in workspace
+# Note that this is the same workspace as caddy.
+# install list comes from https://github.com/theia-ide/go-language-server/blob/d259749c8f263c4d845055833b03b1d2dbefa5b3/README.md#prerequisites
+install +=
+  github.com/ramya-rao-a/go-outline
+  github.com/acroca/go-symbols
+  github.com/nsf/gocode
+  github.com/rogpeppe/godef
+  golang.org/x/tools/cmd/godoc
+  github.com/zmb3/gogetdoc
+  golang.org/x/lint/golint
+  github.com/fatih/gomodifytags
+  github.com/uudashr/gopkgs/cmd/gopkgs
+  golang.org/x/tools/cmd/gorename
+  sourcegraph.com/sqs/goreturns
+  github.com/cweill/gotests/...
+  golang.org/x/tools/cmd/guru
+  github.com/josharian/impl
+  github.com/haya14busa/goplay/cmd/goplay
+  github.com/davidrjenni/reftools/cmd/fillstruct
+
+
+[theia-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode = 0777
+template =
+  inline:
+  #!/bin/bash
+  export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
+  . ${gowork:env.sh}
+  export SHELL=bash
+  cd ${theia:location}
+  exec ${yarn:location}/bin/yarn theia start $@
+
+
+[instance]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+mode = 0644
+output = ${buildout:directory}/instance.cfg
+
+[versions]
+slapos.recipe.template = 4.3

stack/caucase/buildout.cfg

@@ -92,10 +92,8 @@ apache-libcloud = 2.1.0
 bcrypt = 3.1.3
 caucase = 0.1.4
 futures = 3.1.1
-gitdb2 = 2.0.2
 gunicorn = 19.7.1
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.81
 smmap2 = 2.0.3
 PyJWT = 1.6.4
 
@@ -124,14 +122,6 @@ Flask-Script = 2.0.6
 # Flask-User==0.6.19
 Flask-WTF = 0.14.2
 
-# Required by:
-# slapos.toolbox==0.71
-GitPython = 2.1.5
-
-# Required by:
-# slapos.toolbox==0.71
-PyRSS2Gen = 1.1
-
 # Required by:
 # Flask-AlchemyDumps==0.0.10
 SQLAlchemy = 1.1.15
@@ -144,34 +134,10 @@ Unipath = 1.1
 # Flask-WTF==0.14.2
 WTForms = 2.1
 
-# Required by:
-# slapos.toolbox==0.71
-atomize = 0.2.0
-
 # Required by:
 # Flask-Mail==0.9.1
 blinker = 1.4
 
-# Required by:
-# slapos.toolbox==0.71
-dnspython = 1.15.0
-
-# Required by:
-# slapos.toolbox==0.71
-erp5.util = 0.4.51
-
-# Required by:
-# slapos.toolbox==0.71
-feedparser = 5.2.1
-
-# Required by:
-# slapos.toolbox==0.71
-lockfile = 0.12.2
-
-# Required by:
-# slapos.toolbox==0.71
-passlib = 1.7.1
-
 # Required by:
 # caucase==0.1.4
 pem = 17.1.0

stack/caucase/buildout.hash.cfg

@@ -27,7 +27,7 @@ md5sum = c005cfef03a0c2e504fcfa075e59934a
 
 [template-caucase]
 filename = instance-caucase.cfg.jinja2.in
-md5sum = bab4cf56121f964eaad1abfba695d307
+md5sum = 589a0ad37abc28cf6e34e406e7b83eec
 
 [instance-caucase]
 filename = instance.cfg.in

stack/caucase/instance-caucase.cfg.jinja2.in

@@ -222,6 +222,17 @@ init-user = admin
 {% do part_list.append('publish-connection-parameter') -%}
 {% endif -%}
 
+# Used for ERP5 resiliency or (more probably)
+# webrunner resiliency with erp5 inside.
+[resiliency-exclude-file]
+# Generate rdiff exclude file
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = inline:
+  ${directory:log}/**
+rendered = ${directory:srv}/exporter.exclude
+{% do part_list.append('resiliency-exclude-file') -%}
+
 [buildout]
 
 extends =

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = f59c9c313147e437637c2c5ef00438d0
+md5sum = fc814dc35ee1a970d5f092cc0bb3a7a7
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -50,7 +50,7 @@ md5sum = dec33a617fa1b307c8ddb883efcfe3ce
 
 [template-postfix]
 filename = instance-postfix.cfg.in
-md5sum = 0d4938bc36c3829646d6f93fced9a3e0
+md5sum = 805d997fcac266b0958c1c385729b32d
 
 [template-postfix-master-cf]
 filename = postfix_master.cf.in
@@ -74,19 +74,19 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 83e69244728a0d323f535dd3e8b734b6
+md5sum = f723acac5c081c7f9e5eff268d05cc54
 
 [template-zeo]
 filename = instance-zeo.cfg.in
-md5sum = d8f70f5826315a402bf6d72744b80bd9
+md5sum = f9151066df96527d4c7f75c3079b2c10
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = d1257e7e942307be0a79e34aa4320e9f
+md5sum = b513678ffccb927d147fc4e0dda30dd5
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = f0fd49c7d6d9f7c6936afba0d18b7691
+md5sum = 016c56374566b8ea2a1cdb7c8ada3ea7
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -130,7 +130,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%       set test_runner_apache_url_list = [] %}
 {%       set test_runner_external_port = next_port() %}
 {%       for i, (test_runner_internal_ip, test_runner_internal_port) in
-             enumerate(slapparameter_dict[parameter_id ~ '-test-runner-address-list']) %}
+             enumerate(slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', [])) %}
 {%         do test_runner_backend_mapping.__setitem__(
                 'unit_test_' ~ i,
                 'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
@@ -310,6 +310,12 @@ newcerts = ${:ca-dir}/newcerts
 crl = ${:ca-dir}/crl
 apachedex = ${monitor-directory:private}/apachedex
 
+[{{ section('resiliency-exclude-file') }}]
+# Generate rdiff exclude file in case of resiliency
+< = jinja2-template-base
+template = {{ 'inline:{{ "${directory:log}/**\\n" }}' }}
+rendered = ${directory:srv}/exporter.exclude
+
 [{{ section('monitor-generate-apachedex-report') }}]
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}

stack/erp5/instance-erp5.cfg.in

@@ -194,7 +194,9 @@ config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longreques
 config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
+{%   if test_runner_enabled -%}
 config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list}
+{%   endif -%}
 {% endfor -%}
 
 {# if not explicitly configured, connect jupyter to first zope family, which  -#}
@@ -322,9 +324,11 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor -%}
+{% if test_runner_enabled -%}
 {%   for zope_family_name in zope_family_name_list -%}
 {{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
 {%   endfor -%}
+{% endif -%}
 
 
 [publish-early]
@@ -335,9 +339,11 @@ recipe = slapos.cookbook:publish-early
 {%- if has_posftix %}
   smtpd-sasl-password gen-smtpd-sasl-password:passwd
 {%- endif %}
+{% if test_runner_enabled -%}
 {%   for zope_family_name in zope_family_name_list %}
   {{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default
 {%   endfor -%}
+{% endif -%}
 {%- if neo %}
   neo-cluster gen-neo-cluster:name
 {%-  if neo[0] %}

stack/erp5/instance-mariadb.cfg.in

@@ -273,9 +273,18 @@ slowquery = ${monitor-directory:private}/slowquery_digest
 [{{ section('resiliency-exclude-file') }}]
 # Generate rdiff exclude file in case of resiliency
 < = jinja2-template-base
-template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n" }}' }}
+template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n${directory:log}/**\\n${directory:tmp}/**\\n" }}' }}
 rendered = ${directory:srv}/exporter.exclude
 
+[{{ section("resiliency-identity-signature-script")}}]
+# Generate identity script used by webrunner to check data integrity
+# It excludes repozo files as they already include a hash function
+# used to check backups when rebuilding the datafs
+recipe = slapos.cookbook:wrapper
+command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate"
+wrapper-path = ${directory:srv}/.backup_identity_script
+mode = 770
+
 [dash]
 dash = {{ dumps(dash) }}
 

stack/erp5/instance-postfix.cfg.in

@@ -56,6 +56,15 @@ var-spool-postfix-public = ${:var-spool-postfix}/public
 var-spool-postfix-saved = ${:var-spool-postfix}/saved
 var-spool-postfix-trace = ${:var-spool-postfix}/trace
 
+# Used for ERP5 resiliency or (more probably)
+# webrunner resiliency with erp5 inside.
+[{{ section("resiliency-exclude-file") }}]
+# Generate rdiff exclude file
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = {{ 'inline:{{ "${directory:log}/**\\n" }}' }}
+rendered = ${directory:srv}/exporter.exclude
+
 {% if divert -%}
 {%   set milter_port = tcpv4_port + 1 -%}
 {%   set socket = 'inet:' ~ ip ~ ':' ~ milter_port -%}

stack/erp5/instance-zeo.cfg.in

@@ -109,9 +109,18 @@ command = ${tidstorage:repozo-wrapper}
 # Generate rdiff exclude file
 recipe = slapos.recipe.template:jinja2
 mode = 644
-template = {{ 'inline:{{ "${directory:zodb}/**\\n" }}' }}
+template = {{ 'inline:{{ "${directory:zodb}/**\\n${directory:log}/**\\n" }}' }}
 rendered = ${directory:srv}/exporter.exclude
 
+[{{ section("resiliency-identity-signature-script")}}]
+# Generate identity script used by webrunner to check data integrity
+# It excludes repozo files as they already include a hash function
+# used to check backups when rebuilding the datafs
+recipe = slapos.cookbook:wrapper
+command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate" --exclude-path "srv/backup/zodb"
+wrapper-path = ${directory:srv}/.backup_identity_script
+mode = 770
+
 [{{ section("resiliency-after-import-script") }}]
 # Generate after import script used by importer instance of webrunner
 recipe = collective.recipe.template
@@ -144,6 +153,12 @@ input = inline: #!/bin/sh
   fi
   echo "Removing $zodb_path..."
   echo "Restoring $storage_name into $zodb_path..."
+  $repozo --verify --repository="$zodb_backup_directory/$storage_name"
+  CURRENT_EXIT_CODE=$?
+  if [ ! "$CURRENT_EXIT_CODE"="0" ]; then
+    echo "$storage_name Backup verification failed. Backup data is inconsistent."
+    exit "$CURRENT_EXIT_CODE"
+  fi
   $repozo --recover --output="$zodb_directory/$zodb_path" --repository="$zodb_backup_directory/$storage_name"
   CURRENT_EXIT_CODE=$?
   if [ ! "$CURRENT_EXIT_CODE"="0" ]; then

stack/erp5/instance-zope.cfg.in

@@ -109,6 +109,24 @@ ca-certs = ${:ca-dir}/certs
 ca-newcerts = ${:ca-dir}/newcerts
 ca-crl = ${:ca-dir}/crl
 
+# Used for ERP5 resiliency or (more probably)
+# webrunner resiliency with erp5 inside.
+[{{ section("resiliency-exclude-file") }}]
+# Generate rdiff exclude file
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = {{ 'inline:{{ "${directory:log}/**\\n" }}' }}
+rendered = ${directory:srv}/exporter.exclude
+
+[{{ section("resiliency-identity-signature-script")}}]
+# Generate identity script used by webrunner to check data integrity
+# It excludes repozo files as they already include a hash function
+# used to check backups when rebuilding the datafs
+recipe = slapos.cookbook:wrapper
+command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate"
+wrapper-path = ${directory:srv}/.backup_identity_script
+mode = 770
+
 [binary-link]
 recipe = slapos.cookbook:symbolic.link
 target-directory = ${directory:bin}

stack/monitor/buildout.cfg

@@ -118,9 +118,6 @@ depends =
   ${monitor-eggs:eggs}
 
 [versions]
-PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
-pycurl = 7.43.0
-slapos.toolbox = 0.81
 pyasn1 = 0.3.7
 

stack/monitor/buildout.hash.cfg

@@ -14,4 +14,4 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = 2d92f6c50569acbedbc075a84fadbed0
+md5sum = a8ae4a8685fd6d7263bf6f6db02fbdc5

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -65,7 +65,7 @@ recipe = slapos.cookbook:certificate_authority.request
 key-file = ${monitor-httpd-conf-parameter:key-file}
 cert-file = ${monitor-httpd-conf-parameter:cert-file}
 executable = ${monitor-httpd-wrapper:wrapper-path}
-wrapper = ${directory:bin}/monitor-httpd
+wrapper = ${directory:bin}/ca-monitor-httpd
 
 [ca-monitor-httpd-service]
 recipe = slapos.cookbook:wrapper

stack/resilient/buildout.cfg

@@ -130,6 +130,6 @@ eggs =
   ${rdiff-backup-build-1.3.4:egg}
 
 [versions]
-# 1.3.4nxd5 is invalid version string, thus pached version string is not '1.3.4nxd5+SlapOSPatched001'
-# but '1.3.4nxd5-SlapOSPatched001'.
-rdiff-backup = 1.3.4nxd5-SlapOSPatched001
+# 1.3.4nxdX is invalid version string, thus pached version string is not '1.3.4nxdX+SlapOSPatched001'
+# but '1.3.4nxdX-SlapOSPatched001'.
+rdiff-backup = 1.3.4nxd6-SlapOSPatched001

stack/resilient/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = c6c11db5372150019debb1ce519b907d
 
 [template-pull-backup]
 filename = instance-pull-backup.cfg.in
-md5sum = cda4bbedb3ec014ba0311629dd003b3a
+md5sum = 98d7e7c60e090dd3a460ca0410a194ec
 
 [template-replicated]
 filename = template-replicated.cfg.in

stack/resilient/instance-pull-backup.cfg.in

@@ -9,10 +9,8 @@ parts =
   cron-entry-logrotate
   pbs-sshkeys-authority
   sshkeys-openssh
-  backup-checksum-integrity-promise
   resilient-genstatrss-wrapper
   pbs-push-history-log
-  backup-signature-link
   cron-pbs-status-feed
   pull-push-stalled-promise
   notifier-feed-status-promise
@@ -273,41 +271,6 @@ monitor-username = admin
 #--
 #-- Resiliency promises.
 
-[backup-checksum-integrity-promise]
-recipe = slapos.recipe.template:jinja2
-template = inline:
-  #!${dash:location}/bin/dash
-  # only check integrity if pull is not running
-  latest_item=$(ls -t $${pbs:status-item-directory} | head -n1)
-  if [ ! -z "$latest_item" ]; then
-    pbs_result=$(cat "$${pbs:status-item-directory}/$latest_item" | python -c "import sys, json; print json.load(sys.stdin)['title']" 2>/dev/null)
-    if [ "$?" -eq 0 ]; then
-      echo $pbs_result | egrep "pull_raw\s*:\s*STARTED" > /dev/null
-      if [ "$?" -eq 0 ]; then
-        echo "Skipped, PBS pull is running.";
-        exit 0;
-      fi
-    fi
-  fi
-  # Raise an error if signatures are different
-  # Error cannot be deduced if files do not exist
-  cd $${directory:pbs-backup}
-  if [ ! -f "proof.signature" ]; then exit 0; fi
-  backup_signature=$(find . -maxdepth 2 -name backup.signature -not -path "./tmp/*")
-  if [ -z "$backup_signature" ]; then
-    exit 0;
-  else
-    diff -q "proof.signature" "$backup_signature";
-    if [ "$?" -eq 0 ]; then
-      exit 0;
-    else
-      echo "Signature file is not the same before and after transfer"
-      exit 1
-    fi
-  fi
-rendered = $${basedirectory:promises}/backup-checksum-integrity
-mode = 700
-
 [resilient-genstatrss-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX - hard-coded Urls
@@ -318,10 +281,6 @@ wrapper-path = $${directory:bin}/resilient-genstatrss.py
 recipe = cns.recipe.symlink
 symlink = $${pbs:rdiff-backup-data-folder}/restore.log = $${basedirectory:log}/pbs-push-history-log
 
-[backup-signature-link]
-recipe = cns.recipe.symlink
-symlink = $${directory:pbs-backup}/proof.signature = $${directory:monitor-resilient}/backup.signature
-
 [pull-push-stalled-promise]
 recipe = slapos.cookbook:wrapper
 # # time-buffer is 24h (+1h of latitude)

stack/slapos.cfg

@@ -100,7 +100,7 @@ eggs =
 [versions]
 setuptools = 33.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.5.2+slapos012
+zc.buildout = 2.5.2+slapos013
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -140,8 +140,9 @@ slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.15
 slapos.rebootstrap = 4.1
-slapos.recipe.build = 0.37
+slapos.recipe.build = 0.39
 slapos.recipe.cmmi = 0.8
+slapos.toolbox = 0.83
 stevedore = 1.21.0
 unicodecsv = 0.14.1
 xml-marshaller = 0.9.7
@@ -151,10 +152,46 @@ paramiko = 2.1.3
 # slapos.core==1.4.8
 Flask = 0.12
 
+# Required by:
+# slapos.toolbox==0.81
+GitPython = 2.1.11
+
+# Required by:
+# GitPython==2.1.11
+gitdb2 = 2.0.5
+
+# Required by:
+# gitdb==2.0.5
+smmap2 = 2.0.5
+
+# Required by:
+# slapos.toolbox==0.81
+PyRSS2Gen = 1.1
+
+# Required by:
+# slapos.toolbox==0.81
+atomize = 0.2.0
+
+# Required by:
+# slapos.toolbox==0.82
+croniter = 0.3.25
+
+# Required by:
+# slapos.toolbox==0.81
+dnspython = 1.15.0
+
 # Required by:
 # cryptography==1.8.1
 enum34 = 1.1.6
 
+# Required by:
+# slapos.toolbox==0.81
+erp5.util = 0.4.51
+
+# Required by:
+# slapos.toolbox==0.81
+feedparser = 5.2.1
+
 # Required by:
 # jsonschema==2.6.0
 functools32 = 3.2.3.post2
@@ -167,6 +204,10 @@ ipaddress = 1.0.18
 # slapos.cookbook==1.0.62
 jsonschema = 2.6.0
 
+# Required by:
+# slapos.toolbox==0.81
+lockfile = 0.12.2
+
 # Required by:
 # slapos.core==1.4.8
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
@@ -176,10 +217,26 @@ netifaces = 0.10.4
 # cryptography==1.8.1
 packaging = 16.8
 
+# Required by:
+# slapos.toolbox==0.81
+passlib = 1.7.1
+
 # Required by:
 # cffi==1.9.1
 pycparser = 2.17
 
+# Required by:
+# slapos.toolbox==0.81
+pycurl = 7.43.0
+
+# Required by:
+# slapos.toolbox==0.82
+python-dateutil = 2.7.3
+
+# Required by:
+# slapos.toolbox==0.81
+rpdb = 0.1.5
+
 # Required by:
 # slapos.core==1.4.8
 supervisor = 3.3.3