Update Release Candidate

component/apache-wsgi/buildout.cfg

-[buildout]
-
-extends = 
-  ../apache/buildout.cfg
-  ../python-2.7/buildout.cfg
-  ../bzip2/buildout.cfg
-  ../gdbm/buildout.cfg
-  ../gettext/buildout.cfg
-  ../libexpat/buildout.cfg
-  ../ncurses/buildout.cfg
-  ../openssl/buildout.cfg
-  ../readline/buildout.cfg
-  ../sqlite3/buildout.cfg
-  ../zlib/buildout.cfg
-  ../file/buildout.cfg
-
-parts =
-  apache-wsgi
-
-[python2.7]
-configure-options =
-  --enable-shared
-  --disable-static
-  --enable-ipv6
-  --enable-unicode=ucs4
-  --with-system-expat
-  --with-threads
-
-environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
-  LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
-  LD_RUN_PATH=${buildout:parts-directory}/${:_buildout_section_name_}/lib
-  
-[apache-wsgi]
-recipe = hexagonit.recipe.cmmi
-url = http://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz
-md5sum = f42d69190ea0c337ef259cbe8d94d985
-configure-options =
-  --with-apxs=${apache:location}/bin/apxs
-  --with-python=${python2.7:location}/bin/python2.7
-environment =
-  CPPFLAGS = -I${python2.7:location}/include/python2.7
-  LDFLAGS = -L${python2.7:location}/lib

component/bazel/buildout.cfg

@@ -72,48 +72,49 @@ gcc-lib = ${gcc:location}/lib
 gcc-lib64 = ${gcc:location}/lib64
 java_home = ${zulu:location}
 bazelrc = ${:_profile_base_location_}/bazelrc
-script =
-  extract_dir = self.extract(self.download(self.options['url'], self.options['md5sum']))
+install =
+  import os, shutil
+  extract_dir = self.extract(self.download(options['url'], options['md5sum']))
   crosstool_path = os.path.join(extract_dir, 'tools', 'cpp', 'CROSSTOOL')
   os.chmod(crosstool_path, 0644)
-  shutil.copy(self.options['bazel-crosstool-modified-file-path'],
+  shutil.copy(options['bazel-crosstool-modified-file-path'],
               crosstool_path)
   src_main_cpp_build_path = os.path.join(extract_dir, 'src', 'main', 'cpp', 'BUILD')
   os.chmod(src_main_cpp_build_path, 0644)
-  shutil.copy(self.options['bazel-src-main-cpp-build-path'],
+  shutil.copy(options['bazel-src-main-cpp-build-path'],
               src_main_cpp_build_path)
   src_main_tools_build_path = os.path.join(extract_dir, 'src', 'main', 'tools', 'BUILD')
   os.chmod(src_main_tools_build_path, 0644)
-  shutil.copy(self.options['bazel-src-main-tools-build-path'],
+  shutil.copy(options['bazel-src-main-tools-build-path'],
               src_main_tools_build_path)
   src_tools_singlejar_build_path = os.path.join(extract_dir, 'src', 'tools', 'singlejar', 'BUILD')
   os.chmod(src_tools_singlejar_build_path, 0644)
-  shutil.copy(self.options['bazel-src-tools-singlejar-build-path'],
+  shutil.copy(options['bazel-src-tools-singlejar-build-path'],
               src_tools_singlejar_build_path)
 
   target_path = extract_dir+'/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java'
   os.chmod(target_path, 0644)
   call([self.buildout['bazel']['patch-binary'], '-p1', '-d', extract_dir, '-i', self.buildout['bazel']['patch-file-path']])
   path = ':'.join((
-    self.options['unzip-bin'],
-    self.options['zip-bin'],
-    self.options['gcc-bin'],
+    options['unzip-bin'],
+    options['zip-bin'],
+    options['gcc-bin'],
     os.environ['PATH']
     ))
-  env = {'JAVA_HOME':self.options['java_home'],
+  env = {'JAVA_HOME':options['java_home'],
          'PATH':path,
          'LD_LIBRARY_PATH':':'.join((
-           self.options['gcc-lib'],
-           self.options['gcc-lib64'],
+           options['gcc-lib'],
+           options['gcc-lib64'],
            os.environ.get('LD_LIBRARY_PATH', '')
            )),
-         'LDFLAGS':'-Wl,-rpath='+self.options['gcc-lib64'],
-         'CC':self.options['gcc-bin']+'/gcc',
-         'CXX':self.options['gcc-bin']+'/g++',
-         'BAZELRC':self.options['bazelrc'],
+         'LDFLAGS':'-Wl,-rpath='+options['gcc-lib64'],
+         'CC':options['gcc-bin']+'/gcc',
+         'CXX':options['gcc-bin']+'/g++',
+         'BAZELRC':options['bazelrc'],
          'VERBOSE':'yes',
         }
-  bin_dir = os.path.join(self.options['location'], 'bin')
+  bin_dir = os.path.join(location, 'bin')
   os.makedirs(bin_dir)
   call(['bash', 'compile.sh', 'compile'], cwd=extract_dir, env=env)
   shutil.copy(os.path.join(extract_dir, 'output', 'bazel'),

component/boa/buildout.cfg

@@ -19,11 +19,7 @@ url = http://www.boa.org/boa-0.94.14rc21.tar.gz
 md5sum = e24b570bd767a124fcfb40a34d148ba9
 patches =
     ${boa-patch-ENOSYS:location}/${boa-patch-ENOSYS:filename}
-slapos_promise =
-    directory:bin/
-    file:bin/boa
-    file:bin/boa_indexer
-script = 
+install =
     import shutil
     import os
     url = self.download(self.options['url'], self.options['md5sum'])
@@ -33,10 +29,11 @@ script =
     call(['./configure'], cwd=workdir)
     call(['make'], cwd=workdir)
     # Installation of boa. Manually, no make install
-    os.makedirs('%(location)s/bin/')
-    shutil.copyfile(workdir + '/src/boa', '%(location)s/bin/boa')
-    os.chmod('%(location)s/bin/boa', 0755)
-    shutil.copyfile(workdir + '/src/boa_indexer', '%(location)s/bin/boa_indexer')
-    os.chmod('%(location)s/bin/boa_indexer', 0755)
+    bindir = os.path.join(location, 'bin')
+    os.makedirs(bindir)
+    for name in 'boa', 'boa_indexer':
+      path = os.path.join(bindir, name)
+      shutil.copyfile(os.path.join(workdir, 'src', name), path)
+      os.chmod(path, 0755)
 environment =
   PATH=${patch:location}/bin:%(PATH)s

component/boost-lib/buildout.cfg

@@ -9,13 +9,14 @@ extends =
 [boost-lib]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://downloads.sourceforge.net/sourceforge/boost/boost_1_58_0.tar.bz2
-md5sum = b8839650e61e9c1c0a89f371dd475546
+url = http://downloads.sourceforge.net/sourceforge/boost/boost_1_67_0.tar.bz2
+md5sum = ced776cb19428ab8488774e1415535ab
 location = @@LOCATION@@
 configure-command = ./bootstrap.sh --prefix=${:location} --with-python=${python2.7:location}/bin/python2.7
 make-binary =
 make-options =
-make-targets = ./b2 link=shared dll-path=${:location}/lib:${bzip2:location}/lib:${gcc:location}/lib:${gcc:location}/lib64:${zlib:location}/lib install
+make-targets = for a in $MAKEFLAGS; do case $a in -j*) j=$a; break;; esac; done;
+  ./b2 $j link=shared dll-path=${:location}/lib:${bzip2:location}/lib:${gcc:location}/lib:${gcc:location}/lib64:${zlib:location}/lib install
 environment =
   PATH=${gcc:location}/bin:%(PATH)s
   BZIP2_INCLUDE=${bzip2:location}/include

component/busybox/buildout.cfg

@@ -10,16 +10,17 @@ url = http://git.busybox.net/busybox/snapshot/busybox-1_20_2.tar.gz
 md5sum = 025acebb48040ef62dd635d416d317e8
 patches =
     ${:_profile_base_location_}/busybox-1_20_2.patch#292498db86c46e101bb14bf2c74c36f0
-script =
-    extract_dir = self.extract(self.download(%(url)r, %(md5sum)r))
+install =
+    env = self.environ
+    extract_dir = self.extract(self.download(options['url'], options['md5sum']))
     workdir = guessworkdir(extract_dir)
-    self.applyPatchList(self.options.get('patches'), '-p1', cwd=workdir)
+    self.applyPatchList(options.get('patches'), '-p1', cwd=workdir)
     self.logger.info("Creating default configuration")
     call(['make', 'defconfig'], cwd=workdir, env=env)
     self.logger.info("Building")
     call(['make'], cwd=workdir, env=env)
     self.logger.info("Installing")
-    call(['make', 'CONFIG_PREFIX=%(location)s', 'install'], cwd=workdir, env=env)
+    call(['make', 'CONFIG_PREFIX=' + location, 'install'], cwd=workdir, env=env)
     self.logger.info("Installation finished")
 environment =
   PATH=${patch:location}/bin:%(PATH)s

component/chromedriver/buildout.cfg

@@ -26,10 +26,11 @@ part = ${chromedriver:location}
 
 recipe = slapos.recipe.build
 location = ${buildout:bin-directory}/${:wrapper-name}
-script =
-  chromedriver = self.options['part']
+install =
+  import os
+  chromedriver = options['part']
   part = self.buildout[os.path.basename(chromedriver)]
-  with open(%(location)r, 'w') as f:
+  with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
   cd {}
   export LD_LIBRARY_PATH=$PWD:{}
@@ -65,10 +66,9 @@ library =
   ${nss:location}/lib
   ${nspr:location}/lib
 path =
-
-script =
-  url, md5sum = self.options[guessPlatform()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  os.mkdir(%(location)r)
-  shutil.copy(extract_dir + '/chromedriver', %(location)r)
+install =
+  import os, shutil
+  extract_dir = self.extract(self.download(*options[guessPlatform()].split()))
+  os.mkdir(location)
+  shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
 

component/chromium/buildout.cfg

@@ -46,10 +46,11 @@ part = ${chromium:location}
 
 recipe = slapos.recipe.build
 location = ${buildout:bin-directory}/${:wrapper-name}
-script =
-  chromeium = self.options['part']
-  part = self.buildout[os.path.basename(chromeium)]
-  with open(%(location)r, 'w') as f:
+install =
+  import os
+  chromium = options['part']
+  part = self.buildout[os.path.basename(chromium)]
+  with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
   cd {}
   # this also needs a $FONTCONFIG_FILE set, otherwise system fonts will be used and if
@@ -59,7 +60,7 @@ script =
   export PATH={}:$PATH
   exec ./chrome --disable-setuid-sandbox --no-sandbox --disable-gpu $@
   """.format(
-    chromeium,
+    chromium,
     ':'.join(part['library'].split()),
     ':'.join(part['path'].split()),
   ))
@@ -103,9 +104,6 @@ md5sum-x86_64 =
 
 
 recipe = slapos.recipe.build
-slapos_promise =
-  file:chrome
-
 
 x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
 
@@ -153,13 +151,12 @@ library =
   ${zlib:location}/lib
 path =
   ${fontconfig:location}/bin
-
-script =
+install =
   url, md5sum = self.options[guessPlatform()].split()
   extract_dir = self.extract(self.download(url, md5sum))
-  self.copyTree(guessworkdir(extract_dir), %(location)r)
+  self.copyTree(guessworkdir(extract_dir), location)
   # XXX adjust some permissions
   import os
-  os.system('"${findutils:location}/bin/find" "%(location)s" -type d -exec "${coreutils:location}/bin/chmod" a+rx {} \;')
-  os.system('"${findutils:location}/bin/find" "%(location)s" -type f -executable -exec "${coreutils:location}/bin/chmod" a+rx {} \;')
-  os.system('"${findutils:location}/bin/find" "%(location)s" -type f -exec "${coreutils:location}/bin/chmod" a+r {} \;')
+  os.system("'${findutils:location}/bin/find' '%s' -type d -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
+  os.system("'${findutils:location}/bin/find' '%s' -type f -executable -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
+  os.system("'${findutils:location}/bin/find' '%s' -type f -exec '${coreutils:location}/bin/chmod' a+r {} \\;" % location)

component/condor/buildout.cfg

@@ -38,9 +38,8 @@ depends =
 x86 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/28b36a94ad1a405bac689a76b6c353a7/condor-7.9.5-x86_Debian6-unstripped.tar.gz 227059bb9bebc9033665bb246d90a7bb
 x86-64 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/7000a12f9a22765b457e2958762871d5/condor-7.9.5-x86_64_Debian6-stripped.tar.gz 30fa4cebabb8ff4971c769f4ef74ba68
 
-script =
-  if not self.options.get('path'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
+install =
+  url, md5sum = self.options[guessPlatform()].split()
+  extract_dir = self.extract(self.download(url, md5sum))
   workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, "%(location)s")
-  
+  self.copyTree(workdir, location)
\ No newline at end of file

component/consul/buildout.cfg

@@ -15,15 +15,10 @@ url_x86 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_386.zip
 md5sum_x86 = dfdc0eedd79baab7e6bc56c1582fd02e
 md5sum_x86-64 = d6bc0898ea37ae2198370a9e1978d1bb
 
-# script to install.
-script =
-  location = %(location)r
-  self.failIfPathExists(location)
-  import sys
-  ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
-  WK_SUFIX_MAP = { 'x86': 'i386', 'x86-64': 'amd64' }
+install =
+  import shutil
   platform = guessPlatform()
-  url = self.options['url_' + platform]
-  md5sum = self.options['md5sum_' + platform]
+  url = options['url_' + platform]
+  md5sum = options['md5sum_' + platform]
   extract_dir = self.extract(self.download(url, md5sum))
   shutil.move(extract_dir, location)

component/dcron/buildout.cfg

@@ -20,7 +20,7 @@ make-options =
 post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#d7985eef1ceef01af00ffb1d38708c8d:post_make_hook
 environment =
   PATH=${patch:location}/bin:%(PATH)s
-randomsleep_install = ${randomsleep:location}
+dummy = ${randomsleep:recipe}
 
 [dcron-output]
 # Shared binary location to ease migration

component/firefox/buildout.cfg

@@ -31,10 +31,11 @@ part = ${firefox:location}
 
 recipe = slapos.recipe.build
 location = ${buildout:bin-directory}/${:wrapper-name}
-script =
-  firefox = self.options['part']
+install =
+  import os
+  firefox = options['part']
   part = self.buildout[os.path.basename(firefox)]
-  with open(%(location)r, 'w') as f:
+  with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
   cd {}
   export LD_LIBRARY_PATH=$PWD:{}
@@ -157,10 +158,10 @@ library =
 path =
   ${fontconfig:location}/bin
 
-script =
-  url, md5sum = self.options[guessPlatform()].split()
+install =
+  url, md5sum = options[guessPlatform()].split()
   extract_dir = self.extract(self.download(url, md5sum))
-  self.copyTree(guessworkdir(extract_dir), %(location)r)
+  self.copyTree(guessworkdir(extract_dir), location)
 
 
 [geckodriver]
@@ -218,7 +219,8 @@ location = ${buildout:bin-directory}/${:_buildout_section_name_}
 x86 = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
 x86-64 = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
 
-script =
-  url, md5sum = self.options[guessPlatform()].split()
+install =
+  import shutil
+  url, md5sum = options[guessPlatform()].split()
   extract_dir = self.extract(self.download(url, md5sum))
-  shutil.copy(extract_dir + '/geckodriver', %(location)r)
+  shutil.copy(extract_dir + '/geckodriver', location)

component/flare/buildout.cfg

-[buildout]
-extends =
-  ../boost-lib/buildout.cfg
-  ../tokyocabinet/buildout.cfg
-
-parts =
-  flare
-
-[flare]
-recipe = slapos.recipe.cmmi
-url = http://labs.gree.jp/data/source/flare-1.0.9.tgz
-md5sum = e59ccce1ba29e7edf6f665561678d5c8
-configure-options =
-  --with-tokyocabinet=${tokyocabinet:location}
-  --with-boost=${boost-lib:location}
-environment =
-  LDFLAGS =-Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${boost-lib:location}/lib
-

component/fonts/buildout.cfg

@@ -63,19 +63,25 @@ md5sum = d0efec10b9f110a32e9b8f796e21782c
 [msttcore-fonts]
 location = ${fonts:location}/${:_buildout_section_name_}
 recipe = slapos.recipe.build
-script =
+install =
+  import os, subprocess
   from zc.buildout.download import Download
-  d = self.options['location']
+  d = location
   fonts = []
   download = lambda x, dl=Download(self.buildout['buildout']): (
-    dl("http://downloads.sf.net/corefonts/%%s32.exe" %% name, md5sum=md5sum)
+    dl("http://downloads.sf.net/corefonts/%s32.exe" % name, md5sum=md5sum)
     for md5sum, name in (x.split() for x in x.splitlines() if x))
   extract = lambda x, d=d, p7z="${p7zip:location}/bin/7z": any(
     subprocess.check_call((p7z, "x", "-ssc-", path, "*.ttf"), cwd=d)
     for path, is_temp in x)
-  try: fonts += download(self.options['fonts']); os.makedirs(d); extract(fonts)
-  except: shutil.rmtree(d, ignore_errors=True); raise
-  finally: any(os.remove(path) for path, is_temp in fonts if is_temp)
+  try:
+    fonts += download(options['fonts'])
+    os.makedirs(d)
+    extract(fonts)
+  finally:
+    for path, is_temp in fonts:
+      if is_temp:
+        os.remove(path)
 slapos_promise =
 slapos_update_promise = ${:slapos_promise}
 fonts =

component/gtk-2/buildout.cfg

@@ -25,8 +25,8 @@ parts =
 
 [cairo]
 recipe = slapos.recipe.cmmi
-url = http://cairographics.org/releases/cairo-1.14.2.tar.xz
-md5sum = e1cdfaf1c6c995c4d4c54e07215b0118
+url = https://cairographics.org/releases/cairo-1.16.0.tar.xz
+md5sum = f19e0353828269c22bd72e271243a552
 pkg_config_depends = ${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${glib:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libpng:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
 configure-options =
   --disable-static
@@ -46,8 +46,8 @@ environment =
 
 [harfbuzz]
 recipe = slapos.recipe.cmmi
-url = http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-2.3.1.tar.bz2
-md5sum = 531de9df7c8a5405dd9c6a873fcee8c2
+url = https://www.freedesktop.org/software/harfbuzz/release/harfbuzz-2.6.4.tar.xz
+md5sum = 2b3a4dfdb3e5e50055f941978944da9f
 pkg_config_depends = ${cairo:location}/lib/pkgconfig:${cairo:pkg_config_depends}:${icu4c:location}/lib/pkgconfig
 configure-options =
   --disable-static
@@ -63,15 +63,15 @@ environment =
 [fribidi]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://github.com/fribidi/fribidi/releases/download/v1.0.5/fribidi-1.0.5.tar.bz2
-md5sum = 0f97f65038545340316ec0f4bd53df0b
+url = https://github.com/fribidi/fribidi/releases/download/v1.0.9/fribidi-1.0.9.tar.xz
+md5sum = 1b767c259c3cd8e0c8496970f63c22dc
 configure-options =
   --enable-shared
   --disable-static
 
 [pango]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.32/3.32.2/sources/pango-1.42.4.tar.xz
+url = https://download.gnome.org/sources/pango/1.42/pango-1.42.4.tar.xz
 md5sum = deb171a31a3ad76342d5195a1b5bbc7c
 pkg_config_depends = ${fribidi:location}/lib/pkgconfig:${harfbuzz:location}/lib/pkgconfig:${harfbuzz:pkg_config_depends}
 configure-options =
@@ -86,8 +86,8 @@ environment =
 [gdk-pixbuf]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/gdk-pixbuf-2.36.0.tar.xz
-md5sum = 1a3baf91956c7923dab49ee3de100ce1
+url = https://download.gnome.org/sources/gdk-pixbuf/2.36/gdk-pixbuf-2.36.7.tar.xz
+md5sum = 04ca1535f3fab7070db48009ed1e93f6
 pkg_config_depends = ${glib:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libX11:pkg_config_depends}:${pcre:location}/lib/pkgconfig
 configure-options =
   --disable-static
@@ -104,8 +104,8 @@ environment =
 [atk]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/atk-2.22.0.tar.xz
-md5sum = c7f2adcf75e4058727174cde970e9129
+url = https://download.gnome.org/sources/atk/2.28/atk-2.28.1.tar.xz
+md5sum = dfb5e7474220afa3f4ca7e45af9f3a11
 configure-options =
   --with-python=${python2.7:location}/bin/python2.7
   --disable-gtk-doc-html
@@ -117,8 +117,8 @@ environment =
 
 [gtk-2]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/gtk+-2.24.31.tar.xz
-md5sum = 526a1008586094a2cbb4592fd3f9ee10
+url = https://download.gnome.org/sources/gtk+/2.24/gtk+-2.24.32.tar.xz
+md5sum = d5742aa42275203a499b59b4c382a784
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig
 configure-options =
   --disable-static

component/inkscape/buildout.cfg

@@ -27,8 +27,8 @@ extends =
 
 [gsl]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/gsl/gsl-2.3.tar.gz
-md5sum = 905fcbbb97bc552d1037e34d200931a0
+url = ftp://ftp.gnu.org/gnu/gsl/gsl-2.6.tar.gz
+md5sum = bda73a3dd5ff2f30b5956764399db6e7
 configure-options =
   --disable-static
 environment =
@@ -39,8 +39,8 @@ environment =
 
 [inkscape]
 recipe = slapos.recipe.cmmi
-url = https://inkscape.org/gallery/item/10682/inkscape-0.92.1.tar_XlpI7qT.bz2
-md5sum = db2eb2a566cf35ff949fc9ccb172889a
+url = https://inkscape.org/gallery/item/13330/inkscape-0.92.4_A6N0YOn.tar.bz2
+md5sum = ac30f6d5747fd9c620c00dad500f414f
 location = @@LOCATION@@
 pkg_config_depends = ${freetype:location}/lib/pkgconfig:${gtkmm:location}/lib/pkgconfig:${gtkmm:pkg_config_depends}:${gsl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig
 configure-command = ${cmake:location}/bin/cmake

component/java/buildout.cfg

@@ -21,11 +21,11 @@ slapos_promisee =
 # http://java.com/en/download/manual_java7.jsp
 x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97798 90a6b9e2a32d06c18a3f16b485f0d1ea
 x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97800 7605134662f6c87131eca5745895fe84
-script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
+install =
+  url, md5sum = options[guessPlatform()].split()
+  extract_dir = self.extract(self.download(url, md5sum))
   workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, "%(location)s")
+  self.copyTree(workdir, location)
 
 [java-re-8]
 recipe = slapos.recipe.build
@@ -40,11 +40,11 @@ slapos_promisee =
 # Update 161
 x86 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230530_2f38c3b165be4555a1fa6e98c45e0808 32db95dd417fd7949922206b2a61aa19
 x86-64 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230532_2f38c3b165be4555a1fa6e98c45e0808 4385bc121b085862be623f4a31e7e0b4
-script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
+install =
+  url, md5sum = options[guessPlatform()].split()
+  extract_dir = self.extract(self.download(url, md5sum))
   workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, "%(location)s")
+  self.copyTree(workdir, location)
 
 [java-re-8-output]
 # Shared binary location to ease migration

component/jsl/buildout.cfg

@@ -8,13 +8,12 @@ parts = jsl
 recipe = slapos.recipe.build
 url = http://www.javascriptlint.com/download/jsl-0.3.0-src.tar.gz
 md5sum = 2b94ffa4fab07acabe0c5e73cd49bcdf
-script =
-  location = %(location)r
-  self.failIfPathExists(location)
+install =
+  import os
   import sys
-  url = self.options['url']
-  md5sum = self.options['md5sum']
+  url = options['url']
+  md5sum = options['md5sum']
   extract_dir = self.extract(self.download(url, md5sum))
   workdir = guessworkdir(extract_dir)
   os.chdir(os.path.join(workdir, 'src'))
-  os.system('make -f Makefile.ref -j 1 DIST="%(location)s" all export')
+  os.system("make -f Makefile.ref -j 1 DIST='%s' all export" % location)

component/jupyter/buildout.cfg

@@ -70,7 +70,6 @@ scripts =
 [jupyter-notebook-initialized-scripts]
 recipe = zc.recipe.egg:scripts
 eggs = ${jupyter:eggs}
-interpreter = pythonjupyter
 environment = jupyter-env
 
 scripts = 

component/kerberos/buildout.cfg

@@ -18,6 +18,6 @@ url = ${kerberos-package:location}/krb5-1.11.1.tar.gz
 configure-command = src/configure
 configure-options =
     --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-
+make-options = -j1
 environment =
   PATH=${bison:location}/bin:%(PATH)s
\ No newline at end of file

component/libcap/buildout.cfg

@@ -7,7 +7,6 @@ parts = libcap
 
 [libcap2]
 recipe = slapos.recipe.build
-format = no
 url = http://pkgs.fedoraproject.org/lookaside/pkgs/libcap/libcap-2.22.tar.bz2/ce64058bdb3f086ddbfca8ce6c919845/libcap-2.22.tar.bz2
 md5sum = ce64058bdb3f086ddbfca8ce6c919845
 attr-include = ${attr:location}/include/
@@ -19,17 +18,17 @@ slapos_promise =
   file:lib/libcap.so
   file:sbin/getcap
   file:sbin/setcap
-script =
+install =
   import os
-  url = self.download(self.options['url'], self.options['md5sum'])
+  url = self.download(options['url'], options['md5sum'])
   extract_dir = self.extract(url)
   workdir = guessworkdir(extract_dir)
-  cflags = '-I%(attr)s' % {'attr': self.options['attr-include']}
-  ldflags = '-L%(attr)s -Wl,-rpath=%(attr)s' % {'attr': self.options['attr-lib']}
-  call(['make', 'CFLAGS=%s' % cflags, 'LDFLAGS=%s' % ldflags, 'DESTDIR=%s' % self.options['location'], 'RAISE_SETFCAP=no', 'prefix=', 'install'],
-       cwd=workdir, env=env)
-  lib64 = os.path.join(self.options['location'], 'lib64')
-  lib = os.path.join(self.options['location'], 'lib')
+  cflags = '-I%(attr-include)s' % options
+  ldflags = '-L%(attr-lib)s -Wl,-rpath=%(attr-lib)s' % options
+  call(['make', 'CFLAGS=' + cflags, 'LDFLAGS=' + ldflags, 'DESTDIR=' + location, 'RAISE_SETFCAP=no', 'prefix=', 'install'],
+       cwd=workdir, env=self.environ)
+  lib64 = os.path.join(location, 'lib64')
+  lib = os.path.join(location, 'lib')
   # XXX: Dirty if case
   #      if lib64 exists, then create a symlink from lib to lib64
   os.path.exists(lib64) and os.symlink(lib64, lib)

component/libjpeg/buildout.cfg

@@ -5,7 +5,7 @@ parts =
 [libjpeg]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.ijg.org/files/jpegsrc.v8d.tar.gz
-md5sum = 52654eb3b2e60c35731ea8fc87f1bd29
+url = http://www.ijg.org/files/jpegsrc.v9d.tar.gz
+md5sum = 693a4e10906e66467ca21f045547fe15
 configure-options =
   --disable-static

component/libreoffice-bin/buildout.cfg

@@ -17,28 +17,31 @@ md5sum_x86-64 = cbea6cd17063b5bcbe0cb32f7819f0cf
 
 # where office code can be found?
 officedir = libreoffice5.2
-
-# script to install
-script =
-  location = %(location)r
-  self.failIfPathExists(location)
+install =
+  import os
   import sys
   ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
   platform = guessPlatform()
-  url = self.options['url'] %% (ARCH_DIR_MAP[platform], platform)
-  md5sum = self.options['md5sum_' + platform]
+  url = options['url'] % (ARCH_DIR_MAP[platform], platform)
+  md5sum = options['md5sum_' + platform]
   extract_dir = self.extract(self.download(url, md5sum))
   workdir = guessworkdir(extract_dir)
   storagedir = os.path.join(workdir, 'storage')
   os.mkdir(storagedir)
-  rpmsdir = os.path.join(workdir, [q for q in os.listdir(workdir) if q == 'RPMS'][0])
-  rpmlist = [os.path.join(rpmsdir, q) for q in os.listdir(rpmsdir) if q.endswith('.rpm') and 'javafilter' not in q and 'xsltfilter' not in q]
-  [self.pipeCommand([[sys.executable, '${:rpm2cpio}', rpm], ['${:cpio}', '-idum']], cwd=storagedir) for rpm in rpmlist]
-  self.copyTree(os.path.join(storagedir, 'opt', '${:officedir}'), location, ['ure-link'])
+  rpmsdir = os.path.join(workdir, 'RPMS')
+  for q in os.listdir(rpmsdir):
+    if q.endswith('.rpm') and 'javafilter' not in q and 'xsltfilter' not in q:
+      self.pipeCommand((
+        (sys.executable, options['rpm2cpio'], os.path.join(rpmsdir, q)),
+        (options['cpio'], '-idum'),
+      ), cwd=storagedir)
+  self.copyTree(os.path.join(storagedir, 'opt', options['officedir']),
+                location, ['ure-link'])
   os.symlink('ure', os.path.join(location, 'ure-link'))
   # backward compatibility for cloudooo configuration
   os.mkdir(os.path.join(location, 'basis-link'))
-  os.symlink(os.path.join('..', 'program'), os.path.join(location, 'basis-link', 'program'))
+  os.symlink(os.path.join('..', 'program'),
+             os.path.join(location, 'basis-link', 'program'))
 
 # helper binaries
 cpio = ${cpio:location}/bin/cpio

component/nghttp2/buildout.cfg

@@ -4,7 +4,6 @@ extends =
   ../libtool/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../python-2.7/buildout.cfg
-  ../python-setuptools/buildout.cfg
 
 parts =
   nghttp2
@@ -12,10 +11,8 @@ parts =
 [nghttp2]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://github.com/nghttp2/nghttp2/archive/v1.20.0.zip 
-md5sum = e77319851b27c2366aa1cc8a8eabe73e
-location = @@LOCATION@@
-
+url = https://github.com/nghttp2/nghttp2/archive/v1.40.0.tar.gz
+md5sum = ef3c6983e0de2f16b523460c812fe598
 pre-configure =
   autoreconf -fisv -I ${libtool:location}/share/aclocal -I ${pkgconfig:location}/share/aclocal
   automake

component/packer/buildout.cfg

@@ -7,22 +7,18 @@ parts =
 recipe = slapos.recipe.build
 
 # here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
-url_x86-64 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_amd64.zip
-url_x86 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_386.zip
-  
+url_x86-64 = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_linux_amd64.zip
+url_x86 = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_linux_386.zip
+version = 0.7.5
+
 # supported architectures md5sums
 md5sum_x86 = a545108a0ccfde7c1e74de6c4e6fdded
 md5sum_x86-64 = f343d709b84db494e8d6ec38259aa4a6
 
-# script to install.
-script =
-  location = %(location)r
-  self.failIfPathExists(location)
-  import sys
-  ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
-  WK_SUFIX_MAP = { 'x86': '386', 'x86-64': 'amd64' }
+install =
+  import shutil
   platform = guessPlatform()
-  url = self.options['url_' + platform]
-  md5sum = self.options['md5sum_' + platform]
+  url = options['url_' + platform]
+  md5sum = options['md5sum_' + platform]
   extract_dir = self.extract(self.download(url, md5sum))
   shutil.move(extract_dir, location)

component/phantomjs/buildout.cfg

@@ -17,20 +17,17 @@ depends =
 x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686.tar.bz2 9c1426eef5b04679d65198b1bdd6ef88
 x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
 
-script =
-  if not self.options.get('url'):
-    self.options['url'], self.options['md5sum'] = \
-      self.options[guessPlatform()].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'],
-                             self.options.get('md5sum')))
+install =
+  import os
+  url, md5sum = options[guessPlatform()].split()
+  extract_dir = self.extract(self.download(url, md5sum))
   workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, "%(location)s")
-  wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
-  with open(wrapper_location, 'w') as wrapper:
-    wrapper.write("""#!/bin/sh
+  self.copyTree(workdir, location)
+  with open(os.path.join(location, "phantomjs-slapos"), 'w') as wrapper:
+    wrapper.write("""#!/bin/sh -e
   cd %(location)s
   export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
   export PATH=${fontconfig:location}/bin:$PATH
   exec %(location)s/bin/phantomjs "$@"
-  """)
-  os.chmod(wrapper_location, 0o755)
+  """ % options)
+    os.fchmod(wrapper.fileno(), 0o755)

component/pyodide/buildout.cfg

@@ -69,8 +69,8 @@ filename = get-pip.py
 mode = 0755
 
 [pyodide-script]
+# BUG: This section does FS changes outside location.
 recipe = slapos.recipe.build
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 build_dir =  ${buildout:parts-directory}/pyodide-script-test
 part_dir = ${buildout:parts-directory}
 git-executable = ${git:location}/bin/git
@@ -87,28 +87,26 @@ gcc_bin_dir = ${gcc-8.2:location}/bin
 python_bin_dir = ${python3.7:location}/bin
 md5sum = 47ec6a091e503349d948760dc03bed51
 pip_script = ${get-pip:target}
-script =
+install =
   import subprocess, os, shutil
 
   env = {
-    'PKG_CONFIG_PATH':self.options['PKG_CONFIG_PATH'],
-    'PATH':self.options['PATH'] + ':' + os.environ['PATH'],
-    'CPATH':self.options['CPATH'],
-    'CPPFLAGS':self.options['CPPFLAGS'],
-    'LDFLAGS':self.options['LDFLAGS'],
+    'PKG_CONFIG_PATH':options['PKG_CONFIG_PATH'],
+    'PATH':options['PATH'] + ':' + os.environ['PATH'],
+    'CPATH':options['CPATH'],
+    'CPPFLAGS':options['CPPFLAGS'],
+    'LDFLAGS':options['LDFLAGS'],
   }
 
-  if not os.path.isdir(self.options['build_dir']):
-    command_list_0 = [{'cmd': self.options['git-executable'] + ' clone ' + self.options['repository'] + ' pyodide-script-test', 'cwd': self.options['part_dir']}]
-    # put the Popen in the loop to let the pipe.wait() work.
+  if not os.path.isdir(options['build_dir']):
+    command_list_0 = [{'cmd': options['git-executable'] + ' clone ' + options['repository'] + ' pyodide-script-test', 'cwd': options['part_dir']}]
     for command_0 in command_list_0:
-      pipe = subprocess.Popen(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
-      pipe.wait()
+      subprocess.check_call(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
   # else:
-    # shutil.rmtree(self.options['build_dir'], ignore_errors=True)
+    # shutil.rmtree(options['build_dir'], ignore_errors=True)
 
-  cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + self.options['openssl_location'] + ' --prefix@" ' + self.options['build_dir'] + '/cpython/Makefile'
-  pip_script = self.options['pip_script']
+  cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + options['openssl_location'] + ' --prefix@" ' + options['build_dir'] + '/cpython/Makefile'
+  pip_script = options['pip_script']
   install_pip = "python3 " + pip_script
   install_pytest = "pip install pytest selenium pytest-instafail pytest-timeout"
   # Hack: create a gfortran symlink inside the python bin dir.
@@ -119,28 +117,27 @@ script =
   # So we are not going to provide the custom gcc bin path in the PATH environment variable.
   # But scipy needs gfortran. The workaround is just give a symlink in the cpython bin dir, which points to the custom gcc gfortran.
   # Note: the parts/cpython is for run the pyodide building script. The pyodide/cpython is for compile to webassembly module which will loads to the browser.
-  create_gfortran_symlink = 'ln -s ' + self.options['gcc_bin_dir'] + '/gfortran ' + self.options['python_bin_dir'] + '/gfortran'
+  create_gfortran_symlink = 'ln -s ' + options['gcc_bin_dir'] + '/gfortran ' + options['python_bin_dir'] + '/gfortran'
 
   command_list = [
-    {'cmd': cpython_makefile_command, 'cwd': self.options['build_dir']},
-    {'cmd': install_pip, 'cwd': self.options['build_dir']},
-    {'cmd': install_pytest, 'cwd': self.options['build_dir']},
-    {'cmd': create_gfortran_symlink, 'cwd': self.options['build_dir']},
-    {'cmd': 'make -C emsdk', 'cwd': self.options['build_dir']},
-    {'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': self.options['build_dir']},
-    {'cmd': 'make -C lz4', 'cwd': self.options['build_dir']},
-    {'cmd': 'make -C cpython', 'cwd': self.options['build_dir']},
-    {'cmd': 'make -C CLAPACK', 'cwd': self.options['build_dir']},
-    {'cmd': 'make -C packages', 'cwd': self.options['build_dir']},
+    {'cmd': cpython_makefile_command, 'cwd': options['build_dir']},
+    {'cmd': install_pip, 'cwd': options['build_dir']},
+    {'cmd': install_pytest, 'cwd': options['build_dir']},
+    {'cmd': create_gfortran_symlink, 'cwd': options['build_dir']},
+    {'cmd': 'make -C emsdk', 'cwd': options['build_dir']},
+    {'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': options['build_dir']},
+    {'cmd': 'make -C lz4', 'cwd': options['build_dir']},
+    {'cmd': 'make -C cpython', 'cwd': options['build_dir']},
+    {'cmd': 'make -C CLAPACK', 'cwd': options['build_dir']},
+    {'cmd': 'make -C packages', 'cwd': options['build_dir']},
   ]
 
   for command in command_list:
-    pipe = subprocess.Popen(command['cmd'], cwd=command['cwd'], env=env, shell=True)
-    pipe.wait()
+    subprocess.check_call(command['cmd'], cwd=command['cwd'], env=env, shell=True)
   
   # We need to execute the command "./emsdk/emsdk/emsdk_env.sh" to add the emsdk related environment variables.
   # Run "emsdk_env.sh" first, then read the content of "emsdk_set_env.sh" to get the environment variables
-  emsdk_env_content = open(self.options['emsdk_set_env_script'])
+  emsdk_env_content = open(options['emsdk_set_env_script'])
   for content in emsdk_env_content:
     # remove "export " prefix
     content = content[7:]
@@ -153,13 +150,12 @@ script =
   env['CPATH'] += env['EMSCRIPTEN'] + '/system/include/'
 
   command_list_2 = [
-    {'cmd': 'make build/pyodide.asm.js', 'cwd': self.options['build_dir']},
-    {'cmd': 'make -e', 'cwd': self.options['build_dir']},
+    {'cmd': 'make build/pyodide.asm.js', 'cwd': options['build_dir']},
+    {'cmd': 'make -e', 'cwd': options['build_dir']},
   ]
   
   for command_2 in command_list_2:
-    pipe = subprocess.Popen(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
-    pipe.wait()
+    subprocess.check_call(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
 
 [python3.7-PyYAML]
 recipe = slapos.recipe.cmmi

component/python-cocoapi/buildout.cfg

@@ -32,15 +32,15 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [python-cocoapi-build]
 recipe = slapos.recipe.build
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 workdir = ${python-cocoapi-repository:location}/PythonAPI
 python-bin = ${buildout:bin-directory}/${python-cocoapi-build-interpreter:interpreter}
 gcc-location = ${gcc:location}
-script =
+install =
+  import os
   os.makedirs(location)
-  workdir = self.options['workdir']
-  python_bin = self.options['python-bin']
-  gcc_location = self.options['gcc-location']
+  workdir = options['workdir']
+  python_bin = options['python-bin']
+  gcc_location = options['gcc-location']
   env = {'PATH':':'.join([gcc_location+'/bin',
                           os.environ['PATH']]),
          'CC':gcc_location+'/bin/gcc',
@@ -50,16 +50,17 @@ script =
   call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
 
 [python-cocoapi-build-install-egg]
+# BUG: This section does FS changes outside location.
 recipe = slapos.recipe.build
 unzip-binary = ${unzip:location}/bin/unzip
 python-cocoapi-repository-path = ${python-cocoapi-repository:location}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 need-python-cocoapi-build = ${python-cocoapi-build:location}
 egg = pycocotools
-script =
+install =
+  import os
   os.makedirs(location)
-  workdir = self.options['python-cocoapi-repository-path']+'/PythonAPI'
+  workdir = options['python-cocoapi-repository-path']+'/PythonAPI'
   egg_name = 'pycocotools-2.0-py2.7-linux-x86_64.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
-  call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+  call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])

component/python-setuptools/buildout.cfg

-[buildout]
-parts = python-setuptools
-
-[setuptools-download]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-download-only = true
-package_suffix = setuptools-18.4
-filename = ${:package_suffix}.tar.gz
-url = https://pypi.python.org/packages/source/s/setuptools/${:filename}
-md5sum = 214c6c43bd7035e870c1beab402c48e7
-mode = 0644
-
-[python-setuptools]
-# Hack to manually install setuptools outside of the Buildout environment.
-# It is useful in case of using things like "python setup.py test" in eggs
-# downloaded from source (so not using bildout magic).
-# To be able to use it, you should in your instance do an:
-# export PYTHONPATH=${:location} (defined in ${:environment})
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-pythonpath = ${:location}/${setuptools-download:package_suffix}
-environment = export PATH=${python2.7:location}/bin:$PATH; export PYTHONPATH=${:pythonpath};
-stop-on-error = true
-recipe = plone.recipe.command
-update = true
-# chmod is a hack, but the mode of hexagonit.recipe.download above didn't worked yet (there was new version recently)
-command = ${:environment} (mkdir ${:location}; cd ${:location} && tar -xzvf ${setuptools-download:location}/${setuptools-download:filename} && cd ${:pythonpath} && ${buildout:executable} setup.py build) || (rm -fr ${:location}; exit 1)

component/pytorch/buildout.cfg

@@ -44,7 +44,6 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [pytorch-build]
 recipe = slapos.recipe.build
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 workdir = ${pytorch-repository:location}
 python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter}
 no-cuda = 0
@@ -53,16 +52,17 @@ git-bin = ${git:location}/bin
 binutils-location = ${binutils:location}
 gcc-location = ${gcc:location}
 openblas-location = ${openblas:location}
-script =
+install =
+  import os
   os.makedirs(location)
-  workdir = self.options['workdir']
-  python_bin = self.options['python-bin']
-  binutils_location = self.options['binutils-location']
-  gcc_location = self.options['gcc-location']
-  openblas_location = self.options['openblas-location']
+  workdir = options['workdir']
+  python_bin = options['python-bin']
+  binutils_location = options['binutils-location']
+  gcc_location = options['gcc-location']
+  openblas_location = options['openblas-location']
   env = {'PYTHONPATH':workdir,
-         'PATH':':'.join([self.options['cmake-bin'],
-                          self.options['git-bin'],
+         'PATH':':'.join([options['cmake-bin'],
+                          options['git-bin'],
                           binutils_location+'/bin',
                           gcc_location+'/bin',
                           os.environ['PATH']]),
@@ -84,26 +84,27 @@ script =
                                          '-Wl,-rpath,'+openblas_location+'/lib',
                                          ])
          }
-  if self.options.get('no-cuda') == '1':
+  if options.get('no-cuda') == '1':
     env['NO_CUDA'] = '1'
-  import os.path
   call([python_bin, 'setup.py', 'build'], cwd=workdir, env=env)
   call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
 
 [pytorch-build-install-egg]
+# BUG: This section does FS changes outside location.
 recipe = slapos.recipe.build
 unzip-binary = ${unzip:location}/bin/unzip
 pytorch-repository-path = ${pytorch-repository:location}
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 need-pytorch-build = ${pytorch-build:location}
 egg = torch
-script =
+install =
+  import os
   os.makedirs(location)
-  workdir = self.options['pytorch-repository-path']
+  workdir = options['pytorch-repository-path']
   egg_name = 'torch-0.2.0+4af66c4-py2.7-linux-x86_64.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
-  call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+  call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
 
 [pytorch-egg]
 recipe = zc.recipe.egg

component/quic_client-bin/buildout.cfg

@@ -33,13 +33,9 @@ library =
   ${libffi:location}/lib
   ${zlib:location}/lib
   ${libuuid:location}/lib
-script =
-  import glob
-  location = %(location)r
-  self.failIfPathExists(location)
-  url = self.options['url']
-  md5sum = self.options['md5sum']
-  extract_dir = self.extract(self.download(url, md5sum))
+install =
+  import glob, os, shutil
+  extract_dir = self.extract(self.download(options['url'], options['md5sum']))
   os.mkdir(location)
   source_file = glob.glob(os.path.join(extract_dir, '*', 'client-linux-debug'))[0]
   shutil.copy(source_file, location)
@@ -49,7 +45,7 @@ script =
   export LD_LIBRARY_PATH={}
   exec {}/client-linux-debug "$@"
   """.format(
-    ':'.join(self.options['library'].split()),
+    ':'.join(options['library'].split()),
     location
   ))
     os.fchmod(f.fileno(), 0o755)

component/randomsleep/buildout.cfg

@@ -4,26 +4,12 @@ extends =
   ../bash/buildout.cfg
 
 [randomsleep]
-recipe = slapos.recipe.build
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-bin_dir = ${buildout:bin-directory}
-bash_script_code =
-  if [ "$#" -ne 1 ]; then
-    echo "usage: randomsleep maxseconds"
-    exit
-  fi
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+template =
+  inline:#!${bash:location}/bin/bash
+  [ $# = 1 ] || {
+    echo "usage: ${:_buildout_section_name_} maxseconds"
+    exit 1
+  }
   exec ${coreutils:location}/bin/sleep $((RANDOM * $1 >> 15)).$((RANDOM * 100 >> 15))
-wrapper_script_code =
-  #!${bash:location}/bin/bash
-  . ${:location}/randomsleep.bash
-script =
-  os.makedirs(self.options['location'])
-
-  bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')
-  with open(bash_script_path, 'w') as f:
-    f.write(self.options['bash_script_code'])
-
-  wrapper_script_path = os.path.join(self.options['bin_dir'], 'randomsleep')
-  with open(wrapper_script_path, 'w') as f:
-    f.write(self.options['wrapper_script_code'])
-  os.chmod(wrapper_script_path, 0o750)

component/spidermonkey/buildout.cfg

@@ -4,20 +4,16 @@ parts = spidermonkey
 
 [spidermonkey-1.7]
 recipe = slapos.recipe.build
-url = ftp://ftp.mozilla.org/pub/mozilla.org/js/js-1.7.0.tar.gz
+url = http://ftp.mozilla.org/pub/mozilla.org/js/js-1.7.0.tar.gz
 md5sum = 5571134c3863686b623ebe4e6b1f6fe6
-slapos_promise =
-  directory:include/
-  directory:bin/
-  file:bin/js
-  statlib:lib/libjs.a
-  file:lib/libjs.so
-script =
-  url = self.download(self.options['url'], self.options['md5sum'])
+install =
+  import os
+  url = self.download(options['url'], options['md5sum'])
   extract_dir = self.extract(url)
   workdir = os.path.join(guessworkdir(extract_dir), 'src')
-  env.update(JS_DIST=self.options['location'])
-  call(['make', '-f', 'Makefile.ref'], cwd=workdir, env=env)
+  env = self.environ
+  env['JS_DIST'] = location
+  call(['make', '-f', 'Makefile.ref', '-j1'], cwd=workdir, env=env)
   call(['make', '-f', 'Makefile.ref', 'export'], cwd=workdir, env=env)
 
 [spidermonkey]

component/surykatka/buildout.cfg

-[buildout]
-extends =
-  buildout.hash.cfg
-  ../python3/buildout.cfg
-
-parts =
-  surykatka
-
-[surykatka-requirements]
-recipe = slapos.recipe.build:download 
-url = ${:_profile_base_location_}/${:_update_hash_filename_}
-
-[surykatka]
-recipe = plone.recipe.command
-command =
-  bash -c "${python3.7:executable} -m venv ${:location} && \
-    . ${:location}/bin/activate && \
-    pip install -r ${surykatka-requirements:target}"
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-stop-on-error = true
-executable = ${:location}/bin/surykatka

component/surykatka/buildout.hash.cfg

-# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
-# The only allowed lines here are (regexes):
-# - "^#" comments, copied verbatim
-# - "^[" section beginings, copied verbatim
-# - lines containing an "=" sign which must fit in the following categorie.
-#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
-#     Copied verbatim.
-#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
-#     by the re-generation script.
-#     Re-generated.
-# - other lines are copied verbatim
-# Substitution (${...:...}), extension ([buildout] extends = ...) and
-# section inheritance (< = ...) are NOT supported (but you should really
-# not need these here).
-[surykatka-requirements]
-_update_hash_filename_ = requirements.txt
-md5sum = c4bd35bcc5c9c25efe1edff5dd022605

component/surykatka/requirements.txt

-Click==7.0
-certifi==2019.11.28
-chardet==3.0.4
-dnspython==1.16.0
-forcediphttpsadapter==1.0.1
-idna==2.8
-miniupnpc==2.0.2
-peewee==3.13.1
-requests==2.22.0
-surykatka==0.4.2
-urllib3==1.25.8

component/tensorboard/buildout.cfg

@@ -19,7 +19,6 @@ command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:
 
 [tensorboard-build]
 recipe = slapos.recipe.build
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 workdir = ${tensorboard-repository:location}
 gcc-bin = ${gcc:location}/bin
 gcc-lib = ${gcc:location}/lib
@@ -28,22 +27,22 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
 python27-lib = ${python2.7:location}/lib
 java_home_bin = ${bazel:java_home}/bin
 bazel-bin = ${bazel:location}/bin
-script =
+install =
+  import os
   os.makedirs(location)
-  workdir = self.options['workdir']
-  env = {'PATH':':'.join([self.options['gcc-bin'],
-                          self.options['java_home_bin'],
-                          self.options['bazel-bin'],
+  workdir = options['workdir']
+  env = {'PATH':':'.join([options['gcc-bin'],
+                          options['java_home_bin'],
+                          options['bazel-bin'],
                           os.environ['PATH']]),
-         'COMPILER_PATH':':'.join([self.options['gcc-bin'],
+         'COMPILER_PATH':':'.join([options['gcc-bin'],
                                    os.environ.get('COMPILER_PATH') or '']),
-         'LIBRARY_PATH':':'.join([self.options['gcc-lib'],
-                                  self.options['gcc-lib64'],
+         'LIBRARY_PATH':':'.join([options['gcc-lib'],
+                                  options['gcc-lib64'],
                                   os.environ.get('LIBRARY_PATH') or '']),
-         'PYTHON_BIN_PATH':self.options['numpy-python-command'],
-         'PYTHON_LIB_PATH':self.options['python27-lib'],
+         'PYTHON_BIN_PATH':options['numpy-python-command'],
+         'PYTHON_LIB_PATH':options['python27-lib'],
          }
-  import os.path
   env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
   bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package']
   call(bazel_command, cwd=workdir, env=env)
@@ -52,24 +51,24 @@ script =
 recipe = slapos.recipe.build
 unzip-binary = ${unzip:location}/bin/unzip
 tensorboard-repository-path = ${tensorboard-repository:location}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 need-tensorboard-build = ${tensorboard-build:location}
 egg = tensorflow-tensorboard
 bazel-bin = ${bazel:location}/bin
 java_home_bin = ${bazel:java_home}/bin
 numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
-script =
+install =
+  import os
   os.makedirs(location)
-  workdir = self.options['tensorboard-repository-path']
+  workdir = options['tensorboard-repository-path']
   egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
-  env = {'PATH':':'.join([self.options['bazel-bin'],
-                          self.options['java_home_bin'],
+  env = {'PATH':':'.join([options['bazel-bin'],
+                          options['java_home_bin'],
                           os.environ['PATH']]),
-         'PYTHON_BIN_PATH':self.options['numpy-python-command'],
+         'PYTHON_BIN_PATH':options['numpy-python-command'],
          }
   call(['tensorboard/pip_package/build_pip_package.sh', dist_dir], cwd=workdir, env=env)
-  call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+  call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
 
 [versions]

component/tensorflow/buildout.cfg

-[buildout]
+# [buildout]
 extends =
   ../../stack/slapos.cfg
   ../scipy/buildout.cfg
@@ -43,7 +43,6 @@ cudnn_install_path = /usr/local/cuda
 
 [tensorflow-build]
 recipe = slapos.recipe.build
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 workdir = ${tensorflow-repository:location}
 gcc-bin = ${gcc:location}/bin
 gcc-lib = ${gcc:location}/lib
@@ -53,20 +52,21 @@ python27-lib = ${python2.7:location}/lib
 java_home_bin = ${bazel:java_home}/bin
 bazel_bin = ${bazel:location}/bin
 need_cuda = ${cuda:cuda_toolkit_path}
-script =
+install =
+  import os
   os.makedirs(location)
-  workdir = self.options['workdir']
-  env = {'PATH':':'.join([self.options['gcc-bin'],
-                          self.options['java_home_bin'],
-                          self.options['bazel_bin'],
+  workdir = options['workdir']
+  env = {'PATH':':'.join([options['gcc-bin'],
+                          options['java_home_bin'],
+                          options['bazel_bin'],
                           os.environ['PATH']]),
-         'COMPILER_PATH':':'.join([self.options['gcc-bin'],
+         'COMPILER_PATH':':'.join([options['gcc-bin'],
                                    os.environ.get('COMPILER_PATH') or '']),
-         'LIBRARY_PATH':':'.join([self.options['gcc-lib'],
-                                  self.options['gcc-lib64'],
+         'LIBRARY_PATH':':'.join([options['gcc-lib'],
+                                  options['gcc-lib64'],
                                   os.environ.get('LIBRARY_PATH') or '']),
-         'PYTHON_BIN_PATH':self.options['numpy-python-command'],
-         'PYTHON_LIB_PATH':self.options['python27-lib'],
+         'PYTHON_BIN_PATH':options['numpy-python-command'],
+         'PYTHON_LIB_PATH':options['python27-lib'],
          'CC_OPT_FLAGS':'-march=native',
          'TF_NEED_JEMALLOC':'1',
          'TF_ENABLE_XLA':'0',
@@ -79,7 +79,7 @@ script =
          'TF_NEED_MPI':'0',
          'TF_NEED_CUDA':self.buildout['cuda']['tf_need_cuda'],
          ##### FOR CUDA #####
-         'GCC_HOST_COMPILER_PATH':os.path.join(self.options['gcc-bin'], 'gcc'),
+         'GCC_HOST_COMPILER_PATH':os.path.join(options['gcc-bin'], 'gcc'),
          'TF_CUDA_VERSION':self.buildout['cuda']['tf_cuda_version'],
          'CUDA_TOOLKIT_PATH':self.buildout['cuda']['cuda_toolkit_path'],
          'TF_CUDNN_VERSION':self.buildout['cuda']['tf_cudnn_version'],
@@ -87,10 +87,9 @@ script =
          'TF_CUDA_COMPUTE_CAPABILITIES':self.buildout['cuda']['tf_cuda_compute_capabilities'],
          ####################
          }
-  import os.path
   if not os.path.exists(env['CUDA_TOOLKIT_PATH']): env['TF_NEED_CUDA'] = '0'
   env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
-  env['LDFLAGS'] = '-lW,-rpath='+self.options['gcc-lib64']
+  env['LDFLAGS'] = '-lW,-rpath='+options['gcc-lib64']
   call(['./configure'], cwd=workdir, env=env)
   bazel_command = (env['TF_NEED_CUDA'] == '1' and
     ['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '--config', 'cuda', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package']
@@ -99,21 +98,20 @@ script =
   call(bazel_command, cwd=workdir, env=env)
 
 [tensorflow-build-install-egg]
+# BUG: This section does FS changes outside location.
 recipe = slapos.recipe.build
 unzip-binary = ${unzip:location}/bin/unzip
 tensorflow-repository-path = ${tensorflow-repository:location}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 need-tensorboard-build = ${tensorboard-build:location}
 need-tensorflow-build = ${tensorflow-build:location}
 need-protobuf-python = ${protobuf-python:egg}
 egg = tensorflow
-script =
+install =
+  import os
   os.makedirs(location)
-  workdir = self.options['tensorflow-repository-path']
+  workdir = options['tensorflow-repository-path']
   egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
   call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
-  call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
-
-[versions]
+  call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])

component/tomcat/buildout.cfg

@@ -7,27 +7,22 @@ parts =
   tomcat
 
 [tomcat]
-<= tomcat6
-
-[tomcat6]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-strip-top-level-dir = true
-url = http://apache.multidist.com/tomcat/tomcat-6/v6.0.36/bin/apache-tomcat-6.0.36.tar.gz
-md5sum = 3dde098fd0b3a08d3f2867e4a95591ba
+<= tomcat9
 
 [tomcat7]
 recipe = hexagonit.recipe.download
 ignore-existing = true
 strip-top-level-dir = true
-url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.100/bin/apache-tomcat-7.0.100.tar.gz
+url = https://archive.apache.org/dist/tomcat/tomcat-7/v${:version}/bin/apache-tomcat-${:version}.tar.gz
+version = 7.0.100
 md5sum = 79be4ba5a6e770730a4be3d5cb3c7862
 
 [tomcat9]
 recipe = hexagonit.recipe.download
 ignore-existing = true
 strip-top-level-dir = true
-url = https://archive.apache.org/dist/tomcat/tomcat-9/v9.0.12/bin/apache-tomcat-9.0.12.tar.gz
+url = https://archive.apache.org/dist/tomcat/tomcat-9/v${:version}/bin/apache-tomcat-${:version}.tar.gz
+version = 9.0.12
 md5sum = 7283da4a3a6e939adcd8f919be4ba41a
 
 [tomcat7-output]

component/wkhtmltopdf/buildout.cfg

@@ -23,33 +23,24 @@ url_x86 = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux
 md5sum_x86 = ce1a2c0b2cf786ccc5d5828c42c99ddd
 md5sum_x86-64 = 96b7306cebb9e65355f69f7ab63df68b
 
-# script to install.
-script =
-  location = %(location)r
-  self.failIfPathExists(location)
-  import sys
-  import os
-  ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
-  WK_SUFIX_MAP = { 'x86': 'i386', 'x86-64': 'amd64' }
+install =
+  import os,shutil, sys, tempfile
   platform = guessPlatform()
-  url = self.options['url_' + platform]
-  md5sum = self.options['md5sum_' + platform]
+  url = options['url_' + platform]
+  md5sum = options['md5sum_' + platform]
   path = self.download(url, md5sum)
-  import tempfile
   extract_dir = tempfile.mkdtemp(self.name)
   self.cleanup_dir_list.append(extract_dir)
-  self.logger.debug('Created working directory ' + repr(extract_dir))
+  self.logger.debug('Created working directory %s', extract_dir)
   env = os.environ.copy()
   env["PATH"] = "${tar:location}/bin:${xz-utils:location}/bin" + (":" + env["PATH"] if env.get("PATH") else "")
   env["LD_LIBRARY_PATH"] = "${xz-utils:location}/lib" + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "")
   call(["tar", "xJf", path, "-C", extract_dir], env=env)
   shutil.move(os.path.join(extract_dir, "wkhtmltox"), location)
-  wrapper_location = os.path.join("%(location)s", "wkhtmltopdf")
-  wrapper = open(wrapper_location, 'w')
-  wrapper.write("""#!${dash:location}/bin/dash
+  with open(os.path.join(location, "wkhtmltopdf"), 'w') as wrapper:
+    wrapper.write("""#!/bin/sh
   export LD_LIBRARY_PATH=%(location)s/lib:${fontconfig:location}/lib:${freetype:location}/lib:${libX11:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libpng12:location}/lib:${zlib:location}/lib
   export PATH=${fontconfig:location}/bin:$PATH
   exec %(location)s/bin/wkhtmltopdf "$@"
-  """)
-  wrapper.close()
-  os.chmod(wrapper_location, 0755)
+  """ % options)
+    os.fchmod(wrapper.fileno(), 0o755)

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.139'
+version = '1.0.143'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 
@@ -103,7 +103,6 @@ setup(name=name,
           'dropbear = slapos.recipe.dropbear:Recipe',
           'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
           'dropbear.client = slapos.recipe.dropbear:Client',
-          'duplicity = slapos.recipe.duplicity:Recipe',
           'equeue = slapos.recipe.equeue:Recipe',
           'erp5.promise = slapos.recipe.erp5_promise:Recipe',
           'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
@@ -129,7 +128,6 @@ setup(name=name,
           'logrotate.d = slapos.recipe.logrotate:Part',
           'mkdirectory = slapos.recipe.mkdirectory:Recipe',
           'mioga.instantiate = slapos.recipe.mioga.instantiate:Recipe',
-          'mydumper = slapos.recipe.mydumper:Recipe',
           'nbdserver = slapos.recipe.nbdserver:Recipe',
           'neoppod.cluster = slapos.recipe.neoppod:Cluster',
           'neoppod.admin = slapos.recipe.neoppod:Admin',

slapos/recipe/librecipe/generic.py

@@ -27,7 +27,6 @@
 #
 ##############################################################################
 import errno
-import io
 import logging
 import os
 import sys
@@ -124,21 +123,6 @@ class GenericBaseRecipe(object):
   def createExecutable(self, name, content, mode=0o700):
     return self.createFile(name, content, mode)
 
-  def addLineToFile(self, filepath, line, encoding='utf8'):
-    """Append a single line to a text file, if the line does not exist yet.
-
-    line must be unicode."""
-
-    if os.path.exists(filepath):
-      lines = [l.rstrip('\n') for l in io.open(filepath, 'r', encoding=encoding)]
-    else:
-      lines = []
-
-    if not line in lines:
-      lines.append(line)
-      with io.open(filepath, 'w+', encoding=encoding) as f:
-        f.write(u'\n'.join(lines))
-
   def createPythonScript(self, name, absolute_function, args=(), kw={}):
     """Create a python script using zc.buildout.easy_install.scripts
 

slapos/recipe/mydumper.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-import os
-
-from slapos.recipe.librecipe import GenericBaseRecipe
-
-
-def _mydumper_base_cmd(mydumper, database, user, password,
-                       socket=None, host=None, port=None, **kw):
-  cmd = [mydumper]
-  cmd.extend(['-B', database])
-
-  if socket:
-    cmd.extend(['-S', socket])
-  else:
-    cmd.extend(['-h', host])
-    cmd.extend(['-P', port])
-
-  cmd.extend(['-u', user])
-  if password:
-    cmd.extend(['-p', password])
-
-  return cmd
-
-
-def do_export(args):
-  cmd = _mydumper_base_cmd(**args)
-
-  if args['compression']:
-    cmd.append('--compress')
-
-  if args['rows'] is not None:
-    cmd.extend(['-r', args['rows']])
-
-  cmd.extend(['-o', args['directory']])
-
-  os.execv(cmd[0], cmd)
-
-
-def do_import(args):
-  cmd = _mydumper_base_cmd(**args)
-  cmd.append('--overwrite-tables')
-  cmd.extend(['-d', args['directory']])
-  os.execv(cmd[0], cmd)
-
-
-class Recipe(GenericBaseRecipe):
-
-  def install(self):
-    config = {
-      'database': self.options['database'],
-      'directory': self.options['backup-directory'],
-      'user': self.options['user'],
-      'password': self.options.get('password'),
-    }
-
-    if self.options.get('host'):
-      config['host'] = self.options['host']
-      config['port'] = self.options.get('port', 3306)
-    elif self.options.get('socket'):
-      config['socket'] = self.options['socket']
-    else:
-      raise ValueError("host or socket must be defined")
-
-    if self.optionIsTrue('import', False):
-      function = do_import
-      config['mydumper'] = self.options['myloader-binary']
-    else:
-      function = do_export
-      config['mydumper'] = self.options['mydumper-binary']
-      config['compression'] = self.optionIsTrue('compression', default=False)
-      config['rows'] = self.options.get('rows')
-
-    return self.createPythonScript(self.options['wrapper'],
-      '%s.%s' % (function.__module__, function.__name__), (config,))

slapos/recipe/notifier.py

@@ -46,20 +46,16 @@ class Recipe(GenericBaseRecipe):
 
 class Callback(GenericBaseRecipe):
 
-  def createCallback(self, notification_id, callback):
+  def install(self):
+    options = self.options
+    notification_id = options['on-notification-id']
     # XXX: hashing the name here and in
     # slapos.toolbox/slapos/pubsub/__init__.py is completely messed up and
     # prevent any debug.
     callback_id = sha512(str2bytes(notification_id)).hexdigest()
 
-    filepath = os.path.join(self.options['callbacks'], callback_id)
-    self.addLineToFile(filepath, callback)
-    return filepath
-
-  def install(self):
-    # XXX this path is returned multiple times, one for each callback that has been added.
-    return [self.createCallback(self.options['on-notification-id'],
-                                self.options['callback'])]
+    return self.createFile(os.path.join(options['directory'], callback_id),
+                           options['callbacks'])
 
 class Notify(GenericBaseRecipe):
 

slapos/recipe/pbs.py

@@ -43,7 +43,7 @@ from slapos.recipe.librecipe import shlex
 
 def promise(ssh_client, user, host, port):
   # Redirect output to /dev/null
-  with open(os.devnull) as _dev_null:
+  with open(os.devnull, 'wb') as _dev_null:
     ssh = subprocess.Popen(
         (ssh_client, '%s@%s' % (user, host), '-p', str(port)),
         stdin=subprocess.PIPE, stdout=_dev_null, universal_newlines=True)

software/backupserver/buildout.hash.cfg

@@ -42,7 +42,7 @@ md5sum = 432d22bb0f67df5203bbc5d1134a952b
 
 [template-update-rss-script]
 filename = template-update-rss.sh.in
-md5sum = 80b3746568bc8e308a1f337bdaa2441c
+md5sum = ae4a0043414336a521b524d9c95f1c68
 
 [template-pullrdiffbackup]
 filename = instance-pullrdiffbackup.cfg.in

software/backupserver/software.cfg

@@ -16,8 +16,6 @@ extends =
   ../../stack/monitor/buildout.cfg
 
 parts =
-  extra-eggs
-  monitor-extra-eggs
   rdiff-backup
   dcron
   logrotate
@@ -30,17 +28,12 @@ parts =
   template-crontab-line
   slapos-cookbook
 
-[monitor-extra-eggs]
+[rssgen-eggs]
 recipe = zc.recipe.egg
-eggs =
-  ${monitor-eggs:eggs}
-
-[extra-eggs]
-recipe = zc.recipe.egg
-interpreter = pythonforrssgen
+interpreter = python-${:_buildout_section_name_}
 eggs =
   PyRSS2Gen
-  python_dateutil
+  python-dateutil
 
 ##########################################################
 # Service startup scripts and configuration files
@@ -112,7 +105,7 @@ mode = 0644
 rdiff-backup = 1.0.5+SlapOSPatched001
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.4.2
+slapos.recipe.template = 2.4.3
 PyRSS2Gen = 1.1
 gitdb2 = 2.0.5
 smmap2 = 2.0.5

software/backupserver/template-update-rss.sh.in

@@ -4,7 +4,7 @@ STATUS_DIR=$${directory:status}
 RSS_DIR=$${directory:www}
 
 TAIL=${coreutils-output:tail}
-PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
+PYTHON=${buildout:bin-directory}/${rssgen-eggs:interpreter}
 STATUS2RSS=${status2rss:output}
 BASENAME=${coreutils-output:basename}
 

software/caddy-frontend/buildout.hash.cfg

@@ -118,4 +118,4 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
 
 [template-kedifa]
 filename = instance-kedifa.cfg.in
-md5sum = 7814a48cc05d25c8f2d7b527ef2485b5
+md5sum = 9d6111a5d6bc07e708116ca331925241

software/caddy-frontend/instance-kedifa.cfg.in

@@ -210,7 +210,7 @@ ca-certificate = ${directory:etc-kedifa}/ca-certificate.pem
 crl = ${directory:etc-kedifa}/crl.pem
 template-csr = ${directory:etc-kedifa}/template-csr.pem
 pidfile = ${directory:run}/kedifa.pid
-logfile = ${directory:log}/kedfia.log
+logfile = ${directory:log}/kedifa.log
 
 [kedifa-reloader]
 recipe = slapos.cookbook:wrapper

software/caddy-frontend/test/test_data/test.TestDefaultMonitorHttpdPort.test_file_list_log-CADDY.txt

 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestDuplicateSiteKeyProtection.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
 T-2/var/log/httpd-cache-direct/_replicate_access_log

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestSlaveBadParameters.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
-T-1/var/log/kedfia.log
+T-1/var/log/kedifa.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log

software/cloudooo/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 6e4431cf4b0a0d034402604b1e2844c0
 
 [template-cloudooo-instance]
 filename = instance-cloudooo.cfg.in
-md5sum = 79f2740f7e28ad9fcd2f823d076e790e
+md5sum = 5fc0919cc3eab365f773c6eb73f9c6c3

software/cloudooo/instance-cloudooo.cfg.in

@@ -117,10 +117,12 @@ crl = ${apache-ssl-client:crl}
 [apache-promise]
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
 <= monitor-promise-base
-module = check_port_listening
+module = check_url_available
 name = apache.py
-config-hostname = {{ ipv4 }}
-config-port = {{ apache_dict.values()[0][0] }}
+config-url = https://{{ ipv4 }}:{{ apache_dict.values()[0][0] }}
+# XXX cloudooo replies "400 Bad Request" for GET on / but what we want to check
+# is that we don't have a "503 Service Unavailable" from apache or haproxy.
+config-http_code = 400
 
 [apache-conf-ssl]
 cert = ${directory:apache-conf}/apache.crt

software/cloudooo/test/README.md

+Tests for Cloudooo software release

slapos/recipe/duplicity.py → software/cloudooo/test/setup.py

 ##############################################################################
 #
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -24,26 +24,29 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
+from setuptools import setup, find_packages
 
-class Recipe(GenericBaseRecipe):
+version = '0.0.1.dev0'
+name = 'slapos.test.cloudooo'
+with open("README.md") as f:
+  long_description = f.read()
 
-  def install(self):
-
-    remote_url = self.options['remote-backup']
-    backup_directory = self.options['local-directory']
-
-    cmd = [self.options['duplicity-binary'],]
-    options = ['--no-encryption', '--archive-dir', self.options['cache']]
-
-    if self.optionIsTrue('recover', False):
-      options.append('--force')
-      # duplicity [options] remote backup
-      cmd.extend(options)
-      cmd.extend([remote_url, backup_directory])
-    else:
-      # duplicity [options] local remote
-      cmd.extend(options)
-      cmd.extend([backup_directory, remote_url])
-
-    return self.createWrapper(self.options['wrapper'], cmd)
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' cloudooo",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.cookbook',
+        'slapos.libnetworkcache',
+        'six',
+        'PyPDF2',
+      ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/cloudooo/test/test.py

+##############################################################################
+# coding: utf-8
+#
+# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import json
+import six.moves.xmlrpc_client as xmlrpclib
+import ssl
+import base64
+import io
+
+import PyPDF2
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+# Cloudooo needs a lot of time before being available.
+CloudOooTestCase.instance_max_retry = 30
+
+
+def normalizeFontName(font_name):
+  if '+' in font_name:
+    return font_name.split('+')[1]
+  if font_name.startswith('/'):
+    return font_name[1:]
+
+
+def getReferencedFonts(pdf_file_reader):
+  """Return fonts referenced in this pdf
+  """
+  fonts = set()
+
+  def collectFonts(obj):
+    """Recursively visit PDF objects and collect referenced fonts in `fonts`
+    """
+    if hasattr(obj, 'keys'):
+      if '/BaseFont' in obj:
+        fonts.add(obj['/BaseFont'])
+      for k in obj.keys():
+        collectFonts(obj[k])
+
+  for page in pdf_file_reader.pages:
+    collectFonts(page.getObject()['/Resources'])
+
+  return {normalizeFontName(font) for font in fonts}
+
+
+class HTMLtoPDFConversionFontTestMixin:
+  """Mix-In class to test how fonts are selected during
+  HTML to PDF conversions.
+
+  This needs to be mixed with a test case defining:
+
+  * pdf_producer : the name of /Producer in PDF metadata
+  * expected_font_mapping : a mapping of resulting font name in pdf,
+    keyed by font-family in the input html
+  * _convert_html_to_pdf: a method to to convert html to pdf
+  """
+  def _convert_html_to_pdf(self, src_html):
+    # type: (str) -> bytes
+    """Convert the HTML source to pdf bytes.
+    """
+
+  def setUp(self):
+    self.url = json.loads(
+        self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
+    # XXX ignore certificate errors
+    ssl_context = ssl.create_default_context()
+    ssl_context.check_hostname = False
+    ssl_context.verify_mode = ssl.CERT_NONE
+    self.server = xmlrpclib.ServerProxy(
+        self.url,
+        context=ssl_context,
+        allow_none=True,
+    )
+
+  def test(self):
+    actual_font_mapping_mapping = {}
+    for font, expected_substitution in sorted(
+        self.expected_font_mapping.items()):
+      src_html = '''
+      <style>
+          p {{ font-family: "{font}"; font-size: 20pt; }}
+      </style>
+      <p>the quick brown fox jumps over the lazy dog.</p>
+      <p>THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG.</p>
+      '''.format(**locals())
+
+      pdf_data = self._convert_html_to_pdf(src_html)
+      pdf_reader = PyPDF2.PdfFileReader(io.BytesIO((pdf_data)))
+      self.assertEqual(
+          self.pdf_producer,
+          pdf_reader.getDocumentInfo()['/Producer'])
+      fonts_in_pdf = getReferencedFonts(pdf_reader)
+
+      if len(fonts_in_pdf) == 1:
+        actual_font_mapping_mapping[font] = fonts_in_pdf.pop()
+      else:
+        actual_font_mapping_mapping[font] = fonts_in_pdf
+
+    self.maxDiff = None
+    self.assertEqual(self.expected_font_mapping, actual_font_mapping_mapping)
+
+
+class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
+  __partition_reference__ = 'wk'
+  pdf_producer = 'Qt 4.8.7'
+  expected_font_mapping = {
+      'Arial Black': 'Roboto-Medium',
+      'Arial': 'Roboto-Medium',
+      'Avant Garde': 'Roboto-Medium',
+      'Bookman': 'Roboto-Medium',
+      'Carlito': 'Roboto-Medium',
+      'Comic Sans MS': 'Roboto-Medium',
+      'Courier New': 'Roboto-Medium',
+      'DejaVu Sans Condensed': 'Roboto-Medium',
+      'DejaVu Sans ExtraLight': 'Roboto-Medium',
+      'DejaVu Sans Mono': 'Roboto-Medium',
+      'DejaVu Sans': 'Roboto-Medium',
+      'DejaVu Serif Condensed': 'Roboto-Medium',
+      'DejaVu Serif': 'Roboto-Medium',
+      'Garamond': 'Roboto-Medium',
+      'Gentium Basic': 'Roboto-Medium',
+      'Gentium Book Basic': 'Roboto-Medium',
+      'Georgia': 'Roboto-Medium',
+      'Helvetica': 'Roboto-Medium',
+      'Impact': 'Roboto-Medium',
+      'IPAex Gothic': 'Roboto-Medium',
+      'IPAex Mincho': 'Roboto-Medium',
+      'Liberation Mono': 'LiberationMono',
+      'Liberation Sans Narrow': 'Roboto-Medium',
+      'Liberation Sans': 'LiberationSans',
+      'Liberation Serif': 'LiberationSerif',
+      'Linux LibertineG': 'Roboto-Medium',
+      'OpenSymbol': 'Roboto-Medium',
+      'Palatino': 'Roboto-Medium',
+      'Roboto Black': 'Roboto-Medium',
+      'Roboto Condensed Light': 'Roboto-Medium',
+      'Roboto Condensed Regular': 'Roboto-Medium',
+      'Roboto Light': 'Roboto-Medium',
+      'Roboto Medium': 'Roboto-Medium',
+      'Roboto Thin': 'Roboto-Medium',
+      'Times New Roman': 'Roboto-Medium',
+      'Trebuchet MS': 'Roboto-Medium',
+      'Verdana': 'Roboto-Medium',
+      'ZZZdefault fonts when no match': 'Roboto-Medium',
+  }
+
+  def _convert_html_to_pdf(self, src_html):
+    return base64.decodestring(
+        self.server.convertFile(
+            base64.encodestring(src_html.encode()).decode(),
+            'html',
+            'pdf',
+            False,
+            False,
+            {
+                'encoding': 'utf-8'
+            },
+        ).encode())
+
+
+class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
+  __partition_reference__ = 'lo'
+  pdf_producer = 'LibreOffice 5.2'
+  expected_font_mapping = {
+      'Arial Black': 'LinuxLibertineG',
+      'Arial': 'LinuxLibertineG',
+      'Avant Garde': 'LinuxLibertineG',
+      'Bookman': 'LinuxLibertineG',
+      'Carlito': 'Carlito',
+      'Comic Sans MS': 'LinuxLibertineG',
+      'Courier New': 'LinuxLibertineG',
+      'DejaVu Sans Condensed': 'DejaVuSansCondensed',
+      'DejaVu Sans ExtraLight': 'LinuxLibertineG',
+      'DejaVu Sans Mono': 'DejaVuSansMono',
+      'DejaVu Sans': 'DejaVuSans',
+      'DejaVu Serif Condensed': 'DejaVuSerifCondensed',
+      'DejaVu Serif': 'DejaVuSerif',
+      'Garamond': 'LinuxLibertineG',
+      'Gentium Basic': 'GentiumBasic',
+      'Gentium Book Basic': 'GentiumBookBasic',
+      'Georgia': 'LinuxLibertineG',
+      'Helvetica': 'LinuxLibertineG',
+      'Impact': 'LinuxLibertineG',
+      'IPAex Gothic': 'IPAexGothic',
+      'IPAex Mincho': 'IPAexMincho',
+      'Liberation Mono': 'LiberationMono',
+      'Liberation Sans Narrow': 'LiberationSansNarrow',
+      'Liberation Sans': 'LiberationSans',
+      'Liberation Serif': 'LiberationSerif',
+      'Linux LibertineG': 'LinuxLibertineG',
+      'OpenSymbol': 'OpenSymbol',
+      'Palatino': 'LinuxLibertineG',
+      'Roboto Black': 'Roboto-Black',
+      'Roboto Condensed Light': 'RobotoCondensed-Light',
+      'Roboto Condensed Regular': 'LinuxLibertineG',
+      'Roboto Light': 'Roboto-Light',
+      'Roboto Medium': 'Roboto-Medium',
+      'Roboto Thin': 'Roboto-Thin',
+      'Times New Roman': 'LinuxLibertineG',
+      'Trebuchet MS': 'LinuxLibertineG',
+      'Verdana': 'LinuxLibertineG',
+      'ZZZdefault fonts when no match': 'LinuxLibertineG',
+  }
+
+  def _convert_html_to_pdf(self, src_html):
+    return base64.decodestring(
+        self.server.convertFile(
+            base64.encodestring(src_html.encode()).decode(),
+            'html',
+            'pdf',
+        ).encode())

software/erp5testnode/testsuite/deploy-test/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 8288e59eb442c662544daffbf446a033
 
 [template-deploy-test]
 filename = instance.cfg.in
-md5sum = 616abb7fb4608321e11ade0a43b0ce4b
+md5sum = b673c497abe5b3e7de4c7fc8b01aa3d0

software/erp5testnode/testsuite/deploy-test/instance-deploy-test.cfg.jinja2

@@ -47,7 +47,7 @@ config-data-to-vm =
 config-enable-http-server = true
 
 # VM options
-config-ram-size = 2048
+config-ram-size = 4096
 config-cpu-count = 2
 
 # await for system to be ready

software/erp5testnode/testsuite/deploy-test/instance.cfg.in

@@ -29,7 +29,7 @@ rendered = $${buildout:directory}/bin/$${:_buildout_section_name_}
 template = inline:
   #!/bin/sh
   export PATH=${python-with-eggs:location}:$PATH
-  exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory')}}" --python_interpreter=${buildout:bin-directory}/${runTestSuite_py:interpreter} "$@"
+  exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory', 'playbook/roles/standalone-shared') }}" --python_interpreter=${buildout:bin-directory}/${runTestSuite_py:interpreter} "$@"
 mode = 0755
 context =
     key slapparameter_dict slap-configuration:configuration

software/html5ide/software.cfg

@@ -8,7 +8,6 @@ extends =
   ../../component/nodejs/buildout.cfg
 
 parts =
-  monitor-eggs
   slapos-cookbook
   nodejs
   template

software/htmlvalidatorserver/software.cfg

@@ -50,7 +50,7 @@ mode = 0644
 # but '1.3.4nxd2-SlapOSPatched001'.
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.4.2
+slapos.recipe.template = 2.4.3
 inotifyx = 0.2.2
 gitdb2 = 2.0.3
 smmap2 = 2.0.3

software/kvm/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = c9d4356b5148ed8ff8c4f3da63c137ce
+md5sum = cad7d490b84a484b089e34b27a5675ff
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2

software/kvm/instance.cfg.in

@@ -87,7 +87,7 @@ extra-context =
   raw novnc_location ${noVNC:location}
   raw netcat_bin ${netcat:location}/bin/netcat
   raw python_executable ${buildout:executable}
-  raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
+  raw python_eggs_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
   raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
   raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
   raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}

software/kvm/software.cfg

@@ -19,7 +19,6 @@ extends =
 # to avoid versioning issues
 common-parts =
   template
-  eggs
 
 # XXX: we have to manually add this for resilience
   rdiff-backup
@@ -41,22 +40,12 @@ parts = ${:common-parts}
 
 #XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu
 
-[eggs]
-recipe = zc.recipe.egg
-interpreter = python.eggs
-eggs =
+[extra-eggs]
+eggs +=
   ${python-cffi:egg}
-  ${python-cryptography:egg}
-  ${lxml-python:egg}
   websockify
   slapos.cookbook
-  slapos.toolbox
   erp5.util
-  cns.recipe.symlink
-  collective.recipe.template
-  plone.recipe.command
-  ${pycurl:egg}
-  collective.recipe.shelloutput
 
 [http-proxy]
 # https://github.com/nodejitsu/node-http-proxy

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = dc9770bacea2c504b92ad2162e58d222
+md5sum = 07873511a15df74f0be761947819ac9b
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
 
 [template-monitor-edgetest]
 _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
-md5sum = 9e237dbdda59e788202f0da194a57d41
+md5sum = a57106ee88ff3295b9ffce84105da79b
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = f28a329e830ed737d468abcb4e89e1a2
+md5sum = c535f4df6388cdc3f6c1df2a91d6dd53
 
 [network-bench-cfg]
 filename = network_bench.cfg.in
@@ -38,8 +38,8 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720
 
 [monitor-collect-csv-dump]
 filename = script/collect_csv_dump.py
-md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
+md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
 
 [template-surykatka-ini]
 _update_hash_filename_ = surykatka.ini.jinja2
-md5sum = a2de719a5a65438c8c3ee5195442beb6
+md5sum = 89545501f0e5bf11608978886429da3d

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -9,7 +9,7 @@
 {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
 {%- if extra_slave_instance_list %}
 {#- Create slaves to process with setting up defaults #}
-{%-   for slave in sorted(json_module.loads(extra_slave_instance_list)) %}
+{%-   for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %}
 {%-   if 'check-status-code' not in slave %}
 {%-     do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
 {%-   endif %}
@@ -37,7 +37,7 @@
 {%- set part_list = [] %}
 {%- for class, slave_instance_list in slave_instance_dict.items() %}
 {#- class is used to separate surykatka with different timeouts #}
-{%-   for slave in sorted(slave_instance_list) %}
+{%-   for slave in slave_instance_list | sort(attribute='slave_title') %}
 {%-     set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
 {%-     do part_list.append(part_id) %}
 {%-     set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}

software/monitor/instance-monitor-edgetest.cfg.jinja2

@@ -27,7 +27,7 @@ extra-context =
 {%- set namebase = "edgebot" %}
 {%- set authorized_slave_list = [] %}
 {%- set monitor_base_url_dict = {} -%}
-{%- for slave in sorted(slave_instance_list) %}
+{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
 {%-   do authorized_slave_list.append(slave) %}
 {%- endfor %}
 {%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
@@ -56,7 +56,7 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{edgebot_software_type}}
 return = monitor-base-url
 
-{% for section, edgebot_request in request_dict.iteritems() %}
+{% for section, edgebot_request in request_dict.items() %}
 [{{section}}]
 <= replicate
 name = {{ edgebot_request.get('name') }}
@@ -66,11 +66,11 @@ state = {{ edgebot_request.get('state') }}
 {%-   set slave_configuration_dict = slapparameter_dict %}
 {%-   do slave_configuration_dict.update(edgebot_request.get('config')) %}
 {%-   do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
-{%-   for config_key, config_value in slave_configuration_dict.iteritems() %}
+{%-   for config_key, config_value in slave_configuration_dict.items() %}
 config-{{ config_key }} = {{ dumps(config_value) }}
 {%   endfor -%}
 {%-   if edgebot_request.get('sla') %}
-{%-     for parameter, value in edgebot_request.get('sla').iteritems() %}
+{%-     for parameter, value in edgebot_request.get('sla').items() %}
 sla-{{ parameter }} = {{ value }}
 {%-     endfor %}
 {%-   else %}

software/monitor/instance.cfg

@@ -44,17 +44,12 @@ context = import json_module json
           raw template_json_edgetest_test ${json-test-template:target}
 mode = 0644
 
-[surykatka-wrapped]
-recipe = slapos.cookbook:wrapper
-wrapper-path = $${buildout:bin-directory}/surykatka-wrapped
-command-line = ${surykatka:location}/bin/python3.7 ${surykatka:executable}
-
 [instance-base-edgebot]
 recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-edgebot:target}
 rendered = $${buildout:directory}/template-monitor-edgebot.cfg
 extensions = jinja2.ext.do
-surykatka-binary = ${surykatka:executable}
+surykatka-binary = ${buildout:bin-directory}/surykatka
 template-surykatka-ini = ${template-surykatka-ini:target}
 
 context = import json_module json
@@ -64,7 +59,7 @@ context = import json_module json
           key slapparameter_dict slap-configuration:configuration
           key slap_software_type slap-configuration:slap-software-type
           raw software_type edgebot
-          key surykatka_binary surykatka-wrapped:wrapper-path
+          key surykatka_binary :surykatka-binary
           key template_surykatka_ini :template-surykatka-ini
           raw buildout_bin ${buildout:bin-directory}
           raw monitor_template_output ${monitor-template:output}

software/monitor/script/collect_csv_dump.py

@@ -27,6 +27,7 @@
 #
 ##############################################################################
 
+from __future__ import print_function
 import os
 import argparse
 import csv
@@ -53,15 +54,14 @@ def parseArguments():
   return parser.parse_args()
 
 def writeFile(name, folder, date_scope, rows):
-  if os.path.exists(
-      os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))):
+  folder = os.path.join(folder, date_scope)
+  f = os.path.join(folder, "dump_%s.csv" % name)
+  if os.path.exists(f):
     # File already exists, no reason to recreate it.
     return
-  mkdir_p(os.path.join(folder, date_scope), 0o755)
-  file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
-  csv_output = csv.writer(file_io)
-  csv_output.writerows(rows)
-  file_io.close()
+  mkdir_p(folder, 0o755)
+  with open(f, "w") as file_io:
+    csv.writer(file_io).writerows(rows)
 
 def dump_table_into_csv(db, folder):
     db.connect()
@@ -96,6 +96,6 @@ if __name__ == "__main__":
 
   
   if not os.path.exists(parser.collector_db):
-    print "Collector database not found..."
+    print("Collector database not found...")
 
   dump_table_into_csv(Database(parser.collector_db), parser.output_folder)

software/monitor/software.cfg

@@ -4,9 +4,11 @@ extends =
   buildout.hash.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
-  ../../component/surykatka/buildout.cfg
   ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
+  ../../component/python3/buildout.cfg
+
+python = python3
 
 parts = 
   slapos-cookbook
@@ -68,6 +70,7 @@ eggs +=
   ${python-cryptography:egg}
   slapos.toolbox
   slapos.core
+  surykatka
 
 scripts =
   networkbench
@@ -78,13 +81,30 @@ scripts =
   monitor.genstatus
   monitor.configwrite
   check-computer-memory
+  surykatka
 
 [monitor-eggs]
+recipe = zc.recipe.egg
 eggs += 
   slapos.cookbook
   slapos.recipe.cmmi
   hexagonit.recipe.download
   plone.recipe.command
+scripts =
 
 [versions]
 slapos.recipe.template = 4.4
+
+surykatka = 0.4.2
+
+# For surykatka 0.4.2
+click = 7.0
+certifi = 2019.11.28
+chardet = 3.0.4
+dnspython = 1.16.0
+forcediphttpsadapter = 1.0.1
+idna = 2.8
+miniupnpc = 2.0.2
+peewee = 3.13.1
+requests = 2.22.0
+urllib3 = 1.25.8

software/monitor/surykatka.ini.jinja2

@@ -10,7 +10,7 @@ NAMESERVER =
 {%-   endfor %}
 {% endif %}
 URL =
-{%- for slave in sorted(slave_instance_list) %}
+{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
 {%-   if 'url' in slave %}
   {{ slave['url'] }}
 {%-   endif -%}

software/monitor/test/test.py

@@ -65,7 +65,7 @@ class ServicesTestCase(SlapOSInstanceTestCase):
       self.assertIn(expected_process_name, process_names)
 
 
-class MonitorTestMixin(object):
+class MonitorTestMixin:
   monitor_setup_url_key = 'monitor-setup-url'
 
   def test_monitor_setup(self):
@@ -152,8 +152,7 @@ class EdgeSlaveMixin(MonitorTestMixin):
     )
 
   def updateSurykatkaDict(self):
-    class_list = self.surykatka_dict.keys()
-    for class_ in class_list:
+    for class_ in self.surykatka_dict:
       update_dict = {}
       update_dict['ini-file'] = os.path.join(
         self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,))
@@ -200,7 +199,7 @@ class EdgeSlaveMixin(MonitorTestMixin):
       set(
         glob.glob(
           os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))),
-      set([q['ini-file'] for q in self.surykatka_dict.values()])
+      {q['ini-file'] for q in self.surykatka_dict.values()}
     )
     for info_dict in self.surykatka_dict.values():
       self.assertEqual(
@@ -243,14 +242,11 @@ class EdgeSlaveMixin(MonitorTestMixin):
     for info_dict in self.surykatka_dict.values():
       if os.path.exists(info_dict['json-file']):
         os.unlink(info_dict['json-file'])
-      env = os.environ.copy()
-      env.pop('PYTHONPATH', None)
       try:
-        subprocess.check_call(info_dict['status-json'], shell=True, env=env)
+        subprocess.check_call(info_dict['status-json'])
       except subprocess.CalledProcessError as e:
         self.fail('%s failed with code %s and message %s' % (
           info_dict['status-json'], e.returncode, e.output))
-      self.assertTrue(os.path.exists(info_dict['json-file']))
       with open(info_dict['json-file']) as fh:
         status_json = json.load(fh)
       self.assertIn('bot_status', status_json)
@@ -680,8 +676,8 @@ INTERVAL = 120
 TIMEOUT = 4
 SQLITE = %(db_file)s
 URL =
-  https://www.erp5.org/
-  https://www.erp5.com/"""}
+  https://www.erp5.com/
+  https://www.erp5.org/"""}
   }
 
   @classmethod

software/neoppod/buildout.hash.cfg

@@ -18,11 +18,11 @@ md5sum = 80599fcc6e5d07270d7900aebfd62139
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = c03fbfc9df9edc1ef60be970e0627c5e
+md5sum = 8e28f599247ad604ec6e32df410412a8
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
-md5sum = 87670ddc6b5d2007dac1b6d2ba86d168
+md5sum = dabc1e50475055b3ee9184dcace5e8d2
 
 [instance-neo-master]
 filename = instance-neo-master.cfg.in
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = 512383220488335ac186013c2ffdc7c1
+md5sum = 7642c760a2c5af3e3e81c2c54486d1a8
 
 [template-neo-my-cnf]
 filename = my.cnf.in

software/neoppod/instance-neo-admin.cfg.in

 {%- if slapparameter_dict.get('admin') != 0 %}
-{%- set monitor_dict = slapparameter_dict['monitor'] %}
+{%- set monitor_dict = slapparameter_dict.get('monitor', {}) %}
 {%- set bang_on_problem = monitor_dict.pop('bang-on-problem', None) %}
 {%- set periodicity = monitor_dict.pop('periodicity', None) %}
+{%- set monitor_passwd = slapparameter_dict.get('monitor-passwd') %}
 
 [buildout]
-extends =
-  {{ template_monitor }}
 parts +=
   logrotate-admin
+  neoctl
+{%- if monitor_passwd != None %}
   monitor-neo-health
+extends =
+  {{ template_monitor }}
+
+[directory]
+plugin =  ${:etc}/plugin
+
+[monitor-neo-health]
+<= monitor-promise-base
+module = check_neo_health
+name = ${:_buildout_section_name_}.py
+config-neoctl = ${neoctl:wrapper-path}
+{%- if bang_on_problem != None %}
+config-bang-on-problem = {{ dumps(bang_on_problem) }}
+{%- endif %}
+{%- if periodicity != None %}
+config-periodicity = {{ dumps(periodicity) }}
+{%- endif %}
+
+[monitor-instance-parameter]
+monitor-httpd-port = {{ slapparameter_dict.get('monitor-port', 2052) }}
+monitor-title = neo
+password = {{ monitor_passwd }}
+{%- endif %}
 
 [neo-admin]
 recipe = slapos.cookbook:neoppod.admin
@@ -33,26 +57,6 @@ extra-options =
 {%-   endfor %}
 {%- endfor %}
 
-[directory]
-plugin =  ${:etc}/plugin
-
-[monitor-neo-health]
-<= monitor-promise-base
-module = check_neo_health
-name = ${:_buildout_section_name_}.py
-config-neoctl = ${neoctl:wrapper-path}
-{%- if bang_on_problem != None %}
-config-bang-on-problem = {{ dumps(bang_on_problem) }}
-{%- endif %}
-{%- if periodicity != None %}
-config-periodicity = {{ dumps(periodicity) }}
-{%- endif %}
-
-[monitor-instance-parameter]
-monitor-httpd-port = {{ slapparameter_dict.get('admin', 2050) + 1 }}
-monitor-title = neo
-password = {{ slapparameter_dict['monitor-passwd'] }}
-
 [logrotate-admin]
 < = logrotate-entry-base
 name = neo-admin

software/neoppod/instance-neo-input-schema.json

@@ -82,15 +82,18 @@
           "additionalProperties": false,
           "properties": {
             "admin": {
-              "description": "Port of admin node. 0 to disable.",
-              "default": 2050,
+              "description": "Admin port, 0 to disable. Default to 2050 if the node has a master, 0 otherwise.",
               "type": "integer"
             },
             "master": {
-              "description": "Port of master node. 0 to disable.",
+              "description": "Master port, 0 to disable.",
               "default": 2051,
               "type": "integer"
             },
+            "monitor": {
+              "description": "Monitor port, 0 to disable. Unless explicitly set by a node, the first one with an admin gets a monitor at port 2052.",
+              "type": "integer"
+            },
             "storage-count": {
               "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
               "default": 1,

software/neoppod/instance-neo.cfg.in

@@ -65,7 +65,7 @@ recipe = slapos.cookbook:publish.serialised
 # TODO: make port a partition parameter
 ip = {{ "[%s]" % list(ipv6_set)[0] if ipv6_set else list(ipv4_set)[0] }}
 {% set admin = slapparameter_dict.get('admin', 2050) -%}
-{% set master = slapparameter_dict.get('master', 2052) -%}
+{% set master = slapparameter_dict.get('master', 2051) -%}
 {% if master -%}
 port-master = {{ master }}
 master = ${:ip}:${:port-master}
@@ -75,10 +75,12 @@ master =
 {% if admin -%}
 port-admin = {{ admin }}
 admin = ${:ip}:${:port-admin}
+{%-   if 'monitor-passwd' in slapparameter_dict %}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
-{% else -%}
+{%-   endif %}
+{%- else %}
 admin =
-{% endif -%}
+{%- endif %}
 
 {#- Hack to deploy SSL certs via instance parameters #}
 {%- for name, pem in zip(('ca.crt', 'neo.crt', 'neo.key'),

software/neoppod/root-common.cfg.in

@@ -88,13 +88,26 @@ software-type = {{ software_type }}
 {% endfor -%}
 config-autostart = {{ dumps(sum(storage_count)) }}
 {%- do assert(replicas < len(node_list)) %}
-{%- set admin_list = [] %}
+{%- set monitor = set() %}
+{%- for i, node in enumerate(node_list) %}
+{%-   set port = node.get('monitor') %}
+{%-   if port %}
+{%-     do monitor.add(node.get('admin') != 0 and port) %}
+{%-   endif %}
+{%- endfor %}
+{%- if False in monitor %}
+{%-   do monitor.remove(False) %}
+{%-   do assert(monitor) %}
+{%- endif %}
 {%- for i, node in enumerate(node_list) %}
 {%-   set section_id =  prefix ~ i %}
 {%-   do section_id_list.append(section_id) %}
-{%-   if admin_list %}
+{%-   if node.get('master') == 0 and not node.get('monitor') %}
 {%-     do node.setdefault('admin', 0) %}
 {%-   endif %}
+{%-   if monitor or node.get('admin') == 0 %}
+{%-     do node.setdefault('monitor', 0) %}
+{%-   endif %}
 
 [{{section_id}}]
 <= {{ prefix }}request-common
@@ -102,14 +115,21 @@ name = {{ section_id }}
 return =
   master
   admin
-{%-   if node.get('admin') != 0 %}
+{%-   set port = node.pop('monitor', None) %}
+{%-   if port != 0 or node.get('admin') != 0 and not monitor %}
+{%-     if port != 0 %}
   monitor-base-url
-{%-     if monitor_base_url_dict != None %}
-{%-       do monitor_base_url_dict.__setitem__('neo',
-            '${' ~ section_id ~ ':connection-monitor-base-url}') %}
-{%-     endif %}
-{%-     do admin_list.append(section_id) %}
+{%-       if monitor_base_url_dict != None %}
+{%-         do monitor_base_url_dict.__setitem__('neo',
+              '${' ~ section_id ~ ':connection-monitor-base-url}') %}
+{%-       endif %}
 config-monitor-passwd = ${monitor-htpasswd:passwd}
+{%-     endif %}
+{%-     if port %}
+config-monitor-port = {{ dumps(port) }}
+{%-     else %}
+{%-       do monitor.add(None) %}
+{%-     endif %}
 config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
 {%-   endif %}
 {%-   for k, v in node.iteritems() %}
@@ -117,7 +137,7 @@ config-{{ k }} = {{ dumps(v) }}
 {%-   endfor %}
 {{ sla(section_id) }}
 {%- endfor %}
-{%- do assert(len(admin_list) == 1, admin_list) %}
+{%- do assert(len(monitor) == 1, monitor) %}
 
 [{{section(prefix ~ 'cluster')}}]
 recipe = slapos.cookbook:neoppod.cluster

software/neoppod/software-common.cfg

@@ -25,7 +25,6 @@ parts =
 # before any other section that would depend on it
     neoppod-develop
     neoppod
-    slapos-deps-eggs
     slapos-cookbook
 
 [neoppod-repository]
@@ -72,18 +71,6 @@ egg = ${:_buildout_section_name_}
 setup-eggs = ${python-cffi:egg}
 ###
 
-[slapos-deps-eggs]
-recipe = zc.recipe.egg
-eggs =
-  ${lxml-python:egg}
-  ${pycurl:egg}
-  ${python-PyYAML:egg}
-  ${python-cryptography:egg}
-  ${python-cliff:egg}
-  slapos.toolbox
-scripts =
-  slapos-kill
-
 [download-base-neo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:filename}

software/neotest/software.cfg

@@ -39,7 +39,6 @@ parts =
     neotest-runTestSuite
 
 #   for instance
-    slapos-deps-eggs
     slapos-cookbook
     instance.cfg
 

software/powerdns/software.cfg

@@ -8,7 +8,6 @@ extends =
 parts =
   template
   slapos-cookbook
-  monitor-eggs
   eggs
 
 [eggs]

software/re6stnet/software.cfg

@@ -17,30 +17,13 @@ extends =
 
 parts +=
   slapos-cookbook
-  eggs
   dash
   template
 
-[eggs]
+[re6st-eggs]
 recipe = zc.recipe.egg
+interpreter = python-${:_buildout_section_name_}
 eggs =
-  ${lxml-python:egg}
-  ${pycurl:egg}
-  ${python-cryptography:egg}
-  slapos.toolbox
-scripts =
-  slapos-kill
-  monitor.bootstrap
-  monitor.collect
-  monitor.genstatus
-  monitor.configwrite
-  is-process-older-than-dependency-set
-
-[extra-eggs]
-recipe = zc.recipe.egg
-interpreter = pythonwitheggs
-eggs =
-  ${lxml-python:egg}
   ${python-cffi:egg}
   ${python-cryptography:egg}
   pyOpenSSL
@@ -70,7 +53,7 @@ context =
     key template_wrapper template-wrapper:target
     key template_registry_run template-registry-run:target
     key monitor2_template_rendered monitor2-template:rendered
-    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
+    raw python_with_eggs ${buildout:bin-directory}/${re6st-eggs:interpreter}
 
 [template-re6stnet]
 < = download-base

software/slapos-sr-testing/software-py3.cfg

@@ -13,3 +13,4 @@ eggs -=
 
 [template]
 extra =
+  ${slapos.test.monitor-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -114,7 +114,7 @@ setup = ${slapos-repository:location}/software/nextcloud/test/
 
 [slapos.test.turnserver-setup]
 <= setup-develop-egg
-egg = slapos.test.nextcloud
+egg = slapos.test.turnserver
 setup = ${slapos-repository:location}/software/turnserver/test/
 
 [slapos.test.theia-setup]
@@ -132,6 +132,11 @@ setup = ${slapos-repository:location}/software/grafana/test/
 egg = slapos.test.gitlab
 setup = ${slapos-repository:location}/software/gitlab/test/
 
+[slapos.test.cloudooo-setup]
+<= setup-develop-egg
+egg = slapos.test.cloudooo
+setup = ${slapos-repository:location}/software/cloudooo/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -172,6 +177,7 @@ eggs =
   ${slapos.test.jupyter-setup:egg}
   ${slapos.test.nextcloud-setup:egg}
   ${slapos.test.turnserver-setup:egg}
+  ${slapos.test.cloudooo-setup:egg}
   ${backports.lzma:egg}
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
@@ -217,7 +223,6 @@ extra =
   ${slapos.test.erp5-setup:setup}
   ${slapos.test.htmlvalidatorserver-setup:setup}
   ${slapos.test.slapos-master-setup:setup}
-  ${slapos.test.monitor-setup:setup}
   ${slapos.test.plantuml-setup:setup}
   ${slapos.test.powerdns-setup:setup}
   ${slapos.test.proftpd-setup:setup}
@@ -231,6 +236,7 @@ extra =
   ${slapos.test.theia-setup:setup}
   ${slapos.test.grafana-setup:setup}
   ${slapos.test.gitlab-setup:setup}
+  ${slapos.test.cloudooo-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always
@@ -275,3 +281,4 @@ backports.lzma = 0.0.13
 mock = 2.0.0
 testfixtures = 6.11
 funcsigs = 1.0.2
+PyPDF2 = 1.26.0

software/slaprunner/buildout.hash.cfg

@@ -18,15 +18,15 @@ md5sum = ec70348dd71b319590a5c5837f3d2e45
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = b20894f378530e79c847ddfb61782cc5
+md5sum = dc3a7734b08977f438b35d441a12f628
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
-md5sum = fc22e2d2f03ce58631f157a5b4943e15
+md5sum = 2852689640b632fc713d7a603ce66207
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = 918fb2984cb2ed7afba9200167f98a0f
+md5sum = a582be15f0fb797fc75cfc39cd1d12ab
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in

software/slaprunner/instance-runner-import.cfg.in

@@ -19,7 +19,6 @@ parts +=
   runtestsuite
   shellinabox
   shellinabox-service
-  symlinks
   slapos-cfg
   cron-entry-prepare-software
   deploy-instance-parameters
@@ -30,8 +29,6 @@ parts +=
   software-release-deployment-promise
 
   resilient-software-release-information
-# have to repeat the next one, as it's not inherited from pbsready-import
-  import-on-notification
 
   monitor-base
 

software/slaprunner/instance-runner.cfg

@@ -18,7 +18,6 @@ common-runner-parts =
   runner-sshd-promise
   runner-sshd-service
   runtestsuite
-  symlinks
   shellinabox
   shellinabox-service
   slapos-cfg
@@ -43,8 +42,6 @@ common-runner-parts =
 ## Monitoring part
   monitor-base
   monitor-check-webrunner-internal-instance
-## Usability part
-  template-slapuser-script
 
 parts = $${:common-runner-parts}
 extends =
@@ -158,10 +155,10 @@ private-project = $${:home}/.git-private
 public-project = $${:home}/.git-public
 
 [slaprunner]
-slaprunner = ${buildout:directory}/bin/slaprunner
-slapos = ${buildout:directory}/bin/slapos
-slapproxy = ${buildout:directory}/bin/slapproxy
-supervisor = ${buildout:directory}/bin/slapgrid-supervisorctl
+slaprunner = ${buildout:bin-directory}/slaprunner
+slapos = ${buildout:bin-directory}/slapos
+slapproxy = ${buildout:bin-directory}/slapproxy
+supervisor = ${buildout:bin-directory}/slapgrid-supervisorctl
 git-binary = ${git:location}/bin/git
 root_check = false
 slapos.cfg = $${directory:etc}/slapos.cfg
@@ -214,11 +211,10 @@ default_repository_branch = $${slap-parameter:slapos-reference}
 [slaprunner-supervisord-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX hardcoded locations
-command-line = $${directory:bin}/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
+command-line = $${template-slapuser-script:rendered} node supervisord -n
 wrapper-path = $${directory:services}/slaprunner-supervisord
 hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
-
 [test-runner]
 <= slaprunner
 slapos.cfg = $${directory:etc}/slapos-test.cfg
@@ -234,7 +230,7 @@ auto_deploy = True
 [runtestsuite]
 recipe = slapos.cookbook:wrapper
 arguments = --server_url=$${slap-connection:server-url} --key_file=$${slap-connection:key-file} --cert_file=$${slap-connection:cert-file} --computer_id=$${slap-connection:computer-id} --partition_id=$${slap-connection:partition-id}
-command-line = ${buildout:directory}/bin/slaprunnertest $${:arguments}
+command-line = ${buildout:bin-directory}/slaprunnertest $${:arguments}
 wrapper-path = $${directory:bin}/runTestSuite
 environment = PATH=$${shell-environment:path}
   RUNNER_CONFIG=$${slapos-cfg:rendered}
@@ -422,7 +418,7 @@ hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 <= logrotate-entry-base
 name = apache
 log = $${directory:log}/httpd-access.log $${directory:log}/httpd-error.log
-post = test ! -s $${buildout:directory}/var/run/httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/httpd.pid -s USR1
+post = test ! -s $${buildout:directory}/var/run/httpd.pid || ${buildout:bin-directory}/slapos-kill --pidfile $${buildout:directory}/var/run/httpd.pid -s USR1
 
 [httpd-graceful-wrapper]
 recipe = collective.recipe.template
@@ -452,7 +448,7 @@ stop-on-error = true
 #-- WSGI
 
 [gunicorn]
-bin_gunicorn = $${directory:bin}/gunicorn
+bin_gunicorn = ${buildout:bin-directory}/gunicorn
 bin_launcher = $${directory:services}/gunicorn
 path_shell = ${dash:location}/bin/dash
 socket = $${directory:tmp}/flaskserver.sock
@@ -470,7 +466,7 @@ hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [gunicorn-graceful]
 recipe = slapos.cookbook:wrapper
-command-line = $${directory:bin}/killpidfromfile $${gunicorn:path_pid} SIGHUP
+command-line = ${buildout:bin-directory}/killpidfromfile $${gunicorn:path_pid} SIGHUP
 wrapper-path = $${directory:scripts}/gunicorn-graceful
 
 #--------------------
@@ -634,11 +630,6 @@ name = runner-sshd.py
 config-hostname = $${slap-network-information:global-ipv6}
 config-port = $${runner-sshd-port:port}
 
-[symlinks]
-recipe = cns.recipe.symlink
-symlink_target = $${directory:bin}
-symlink_base = ${buildout:directory}/bin
-
 [slap-parameter]
 # Default value if no ssh key is specified
 user-authorized-key =
@@ -695,13 +686,13 @@ template = inline:
 
 [shellinabox-service]
 recipe = slapos.cookbook:wrapper
-command-line = $${directory:bin}/shellinaboxd
+command-line = $${shellinabox:rendered}
 wrapper-path = $${directory:services}/shellinaboxd
 hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [shell-environment]
 shell = ${bash:location}/bin/bash
-path = ${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin:${mosh:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:$${buildout:directory}/bin/:/usr/bin:/bin/
+path = ${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin:${mosh:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:$${directory:bin}:/usr/bin:/bin/
 
 [prepare-software]
 recipe = slapos.cookbook:wrapper
@@ -820,11 +811,11 @@ mode = 0744
 context =
     section supervisord supervisord
     section slaprunner slaprunner
-    raw python_executable ${buildout:directory}/bin/${extra-eggs:interpreter}
+    raw python_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
 
 [supervisord-wrapper]
 recipe = slapos.cookbook:wrapper
-command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
+command-line = ${buildout:bin-directory}/supervisord -c $${supervisord-conf:rendered} --nodaemon
 wrapper-path = $${directory:services}/supervisord
 hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 

software/slaprunner/software-py3.cfg

@@ -9,7 +9,7 @@ python = python3
 common-parts -=
   rdiff-backup
 
-[eggs]
+[extra-eggs]
 eggs -=
 # futures is a backport of Py3's concurrent.futures module
   futures

software/slaprunner/software.cfg

@@ -37,8 +37,8 @@ shared-part-list =
 # to avoid versioning issues
 
 common-parts =
+  slapos-cookbook
   template
-  eggs
   instance-runner-import
   instance-runner-export
   template-slapos-cfg
@@ -141,32 +141,19 @@ filename = slapos-slapuser-script.in
 < = template-base
 output = ${buildout:directory}/buildout-shared-part-list
 
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-  ${pycurl:egg}
-  collective.recipe.template
-  cns.recipe.symlink
+[extra-eggs]
+eggs +=
   erp5.util
   lock-file
-  plone.recipe.command
   slapos.recipe.build
   slapos.toolbox[flask_auth]
   gunicorn
 # for gunicorn[gthread]
   futures
-  ${slapos-cookbook:eggs}
-  slapos.core # listed explicitly for scripts generation
-
-[extra-eggs]
-recipe = zc.recipe.egg
-interpreter = pythonwitheggs
-eggs +=
   supervisor
 
 [versions]
 Flask-Auth = 0.85
-cns.recipe.symlink = 0.2.3
 futures = 3.0.5
 gitdb = 0.6.4
 gunicorn = 19.10.0

software/slaprunner/template/runner-import.sh.jinja2

@@ -161,6 +161,12 @@ DB_SOFTWARE_TABLE=$(db_query ".table software__")
 # XXX hardcoded
 PARTITION=$(basename $HOME)
 OLD_SOFTWARE_RELEASE=$(db_query "select software_release from $DB_PARTITION_TABLE where reference='slappart0';")
+if [ "$OLD_SOFTWARE_RELEASE" == "" ];
+then
+   echo "No instance configured"
+   echo 0 > $RESTORE_EXIT_CODE_FILE
+   exit 0
+fi
 SOFTWARE_RELEASE=$({{ sys.executable }} - $OLD_SOFTWARE_RELEASE $PARTITION <<EOF
 if 1:
   import os, re, sys

software/wendelin-scalability/test-common.cfg

@@ -8,9 +8,9 @@ list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
 [patch-template]
 recipe = slapos.recipe.build
 location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
-script =
-  with open(self.options['location'], 'w') as dst, \
-       open(self.options['base']) as src:
+install =
+  with open(options['base']) as src:
     src = src.read()
-    i = src.index('[buildout]')
-    dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
+  i = src.index('[buildout]')
+  with open(location, 'w') as dst:
+    dst.write(src[:i] + options['extra'] + '\n' + src[i:])

software/wendelin-scalability/test-much-deduplication-high-compressibility.cfg

@@ -37,11 +37,10 @@ md5sum = 374e00dfbd28f27819ccfff31d169c9c
 [gpg-decrypt]
 recipe = slapos.recipe.build
 gpg = ${gnupg:location}/bin/gpg
-script =
-  import subprocess, zc.buildout
-  options = self.options
+install =
+  import subprocess
   args = (options['gpg'], '-d', '--batch', '--passphrase-fd', '0',
-          '-o', options['location'], options['input'])
+          '-o', location, options['input'])
   p = subprocess.Popen(args, stdin=subprocess.PIPE)
   p.communicate(options['password'])
   retcode = p.poll()

stack/caucase/buildout.cfg

@@ -26,6 +26,7 @@ eggs =
   ${slapos-cookbook:eggs}
   plone.recipe.command
   slapos.recipe.template
+scripts =
 
 [caucase-jinja2-library]
 recipe = slapos.recipe.build:download

stack/erp5/buildout.cfg

@@ -86,6 +86,7 @@ parts +=
 
 # some additional utils
   zodbanalyze
+  zodbpack
 
 # Create instance template
   template
@@ -518,7 +519,6 @@ entry-points =
 scripts =
   apachedex
   performance_tester_erp5
-  repozo
   runwsgi
   runzope
   runzeo
@@ -556,22 +556,21 @@ scripts = zodbanalyze
 recipe = zc.recipe.egg:develop
 setup = ${erp5:location}
 
-[slapos-deps-eggs]
+[zodbpack]
 recipe = zc.recipe.egg
-eggs +=
+eggs =
+  ${lxml-python:egg}
+  ${pycurl:egg}
+  ${python-PyYAML:egg}
+  ${python-cryptography:egg}
+  ${python-cliff:egg}
   slapos.toolbox[zodbpack]
-scripts +=
-  is-local-tcp-port-opened
-  is-process-older-than-dependency-set
-  check-slow-queries-digest-result
-  check-apachedex-result
-  check-computer-memory
-  runApacheDex
+scripts =
   zodbpack
 
 [versions]
 # See ../../software/neoppod/software-common.cfg for versions common with NEO:
-#  neoppod, mysqlclient, slapos.recipe.template & [slapos-deps-eggs]
+#  neoppod, mysqlclient, slapos.recipe.template
 
 # patched eggs
 Acquisition = 2.13.12+SlapOSPatched001

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 0eb2f45a145dde0c59d3488bfdbe66ff
+md5sum = 1c332e1a3388258572fcc6528aca5725
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in

stack/erp5/instance-mariadb.cfg.in

@@ -213,10 +213,6 @@ recipe = slapos.cookbook:symbolic.link
 target-directory = ${directory:bin}
 link-binary = {{ dumps(parameter_dict['link-binary']) }}
 
-[{{ section("binary-link-mysqlbinlog") }}]
-< = binary-link
-link-binary = {{ parameter_dict['mariadb-location'] }}/bin/mysqlbinlog
-
 [binary-wrap-base]
 recipe = slapos.cookbook:wrapper
 # Note: --defaults-file must be the first argument, otherwise wrapped binary

stack/monitor/buildout.cfg

@@ -13,7 +13,6 @@ extends =
 
 parts =
   slapos-cookbook
-  monitor-eggs
   extra-eggs
   monitor2-template
 
@@ -31,7 +30,7 @@ destination = ${buildout:parts-directory}/monitor-scripts
 on-update = true
 
 [monitor-eggs]
-recipe = zc.recipe.egg
+# XXX: merge with extra-eggs?
 eggs =
   ${lxml-python:egg}
   ${pycurl:egg}
@@ -42,9 +41,6 @@ eggs =
   slapos.toolbox
   slapos.core
 
-# Do no generate any scripts here as all of them are generated by extraeggs
-scripts =
-
 [extra-eggs]
 recipe = zc.recipe.egg
 interpreter = pythonwitheggs
@@ -105,8 +101,6 @@ context =
     raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
     raw check_disk_space ${buildout:directory}/bin/check-free-disk
     raw bin_directory ${buildout:directory}/bin
-depends = 
-  ${monitor-eggs:eggs}
 
 [versions]
 cns.recipe.symlink = 0.2.3

stack/resilient/README.rst

@@ -46,7 +46,7 @@ The pull-backup, upon receiving the notification, will make a copy of the data a
 
 You should provide the bin/{mysoftware}-exporter script, see for instance
   https://lab.nexedi.com/nexedi/slapos/blob/HEAD/slapos/recipe/postgres/__init__.py#L207
-  https://lab.nexedi.com/nexedi/slapos/blob/HEAD/slapos/recipe/mydumper.py#L71
+  https://lab.nexedi.com/nexedi/slapos/blob/1.0.142/slapos/recipe/mydumper.py#L71
 
 By default, as defined in
   https://lab.nexedi.com/nexedi/slapos/blob/HEAD/stack/resilient/pbsready-export.cfg.in#L27
@@ -68,7 +68,7 @@ or through a monitoring + election script.
 You should provide the bin/{mysoftware}-importer script, see for instance
 
   https://lab.nexedi.com/nexedi/slapos/blob/HEAD/slapos/recipe/postgres/__init__.py#L233
-  https://lab.nexedi.com/nexedi/slapos/blob/HEAD/slapos/recipe/mydumper.py#L71
+  https://lab.nexedi.com/nexedi/slapos/blob/1.0.142/slapos/recipe/mydumper.py#L71
 
 
 
@@ -94,7 +94,7 @@ extends = ${instance-mysoftware:output}
 
 parts +=
     mysoftware
-    import-on-notification
+    notify-callback
 
 [importer]
 recipe = YourImportRecipe

stack/resilient/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 66331047b7dbf2513c5726d5d1647320
 
 [pbsready-import]
 filename = pbsready-import.cfg.in
-md5sum = d813c43ed00eff868fb13bc75b045336
+md5sum = 6f32cede10cf0d6430c017f49fe5aa30
 
 [pbsready-export]
 filename = pbsready-export.cfg.in
@@ -30,7 +30,7 @@ md5sum = 555d528b198564f0ce1e94db1160ebf3
 
 [template-replicated]
 filename = template-replicated.cfg.in
-md5sum = 815fd8f7c42b9cf59b286b0fe77fa76d
+md5sum = 1b25afe215e45a92e0988ea2ca9484f4
 
 [template-parts]
 filename = template-parts.cfg.in

stack/resilient/pbsready-import.cfg.in

@@ -7,8 +7,7 @@ parts +=
   resilient-web-takeover-httpd-wrapper
   resilient-web-takeover-httpd-promise
 
-  check-backup-integrity-on-notification
-  import-on-notification
+  notify-callback
   backup-checksum-integrity-promise
   resilient-publish-connection-parameter
 
@@ -39,13 +38,17 @@ minimum = 65516
 maximum = 65525
 ip = $${notifier:host}
 
-[import-on-notification]
+[notify-callback]
 # notifier.callback runs a script when a notification (sent by a parent PBS)
 # is received
-<= notifier
 recipe = slapos.cookbook:notifier.callback
+directory = $${notifier:callbacks}
 on-notification-id = $${slap-parameter:on-notification}
-callback = $${importer:wrapper}
+callbacks =
+# import on notification
+  $${importer:wrapper}
+# check backup integrity on notification
+  $${post-notification-run:output}
 
 [post-notification-run]
 recipe = collective.recipe.template
@@ -59,12 +62,6 @@ input = inline:
 output = $${rootdirectory:bin}/post-notification-run
 mode = 0700
 
-[check-backup-integrity-on-notification]
-<= notifier
-recipe = slapos.cookbook:notifier.callback
-on-notification-id = $${slap-parameter:on-notification}
-callback = $${post-notification-run:output}
-
 [backup-checksum-integrity-promise-bin]
 recipe = slapos.recipe.template:jinja2
 template = inline:

stack/resilient/template-replicated.cfg.in

@@ -237,13 +237,13 @@ input = inline:#!/bin/bash
   if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
-output = ${resilient-directory:bin}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
+output = ${resilient-directory:bin}/resilient-request-pbs-{{namebase}}-{{id}}-public-key
 mode = 700
 
 [resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise]
 <= monitor-promise-base
 module = check_command_execute
-name = resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
+name = resilient-request-pbs-{{namebase}}-{{id}}-public-key.py
 config-command = ${resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise-bin:output}
 
 [request-pull-backup-server-{{namebase}}-{{id}}]

stack/slapos.cfg

@@ -100,7 +100,7 @@ eggs =
   slapos.libnetworkcache
 
 [versions]
-setuptools = 40.4.3
+setuptools = 44.0.0
 # Use SlapOS patched zc.buildout
 zc.buildout = 2.7.1+slapos001
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
@@ -137,13 +137,13 @@ pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
 six = 1.12.0
-slapos.cookbook = 1.0.139
+slapos.cookbook = 1.0.143
 slapos.core = 1.5.9
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.4
-slapos.recipe.build = 0.43
+slapos.recipe.build = 0.44
 slapos.recipe.cmmi = 0.12
 slapos.toolbox = 0.109
 stevedore = 1.21.0

update-hash

-#!/usr/bin/env python
+#!/usr/bin/env python2
 """
 Suggested installation:
 $ cp update-hash .git/hooks/