Update Release Candidate

component/6tunnel/buildout.cfg

@@ -10,12 +10,10 @@ extends =
 [6tunnel]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://toxygen.net/6tunnel/6tunnel-0.11rc2.tar.gz
+url = https://github.com/wojtekka/6tunnel/releases/download/0.11rc2/6tunnel-0.11rc2.tar.gz
 md5sum = 74e02d4f0704b3083a01feda66033449
 pre-configure =
   aclocal
-  autoheader
-  automake --foreign
   autoconf
 environment =
   PATH=${autoconf:location}/bin:${automake:location}/bin:%(PATH)s

component/attr/buildout.cfg

@@ -4,6 +4,7 @@ parts = attr
 
 [attr]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://download.savannah.gnu.org/releases/attr/attr-2.4.47.src.tar.gz
 md5sum = 84f58dec00b60f2dc8fd1c9709291cc7
 configure-options =

component/boost-lib/buildout.cfg

@@ -5,6 +5,7 @@ extends =
   ../bzip2/buildout.cfg
   ../gcc/buildout.cfg
   ../zlib/buildout.cfg
+  ../xz-utils/buildout.cfg
 
 [boost-lib]
 recipe = slapos.recipe.cmmi
@@ -12,14 +13,16 @@ shared = true
 url = http://downloads.sourceforge.net/sourceforge/boost/boost_1_67_0.tar.bz2
 md5sum = ced776cb19428ab8488774e1415535ab
 location = @@LOCATION@@
-configure-command = ./bootstrap.sh --prefix=${:location} --with-python=${python2.7:location}/bin/python2.7
+configure-command = ./bootstrap.sh --prefix=${:location} --with-python=${python2.7:location}/bin/python2.7 --without-icu
 make-binary =
 make-options =
 make-targets = for a in $MAKEFLAGS; do case $a in -j*) j=$a; break;; esac; done;
-  ./b2 $j link=shared dll-path=${:location}/lib:${bzip2:location}/lib:${gcc:location}/lib:${gcc:location}/lib64:${zlib:location}/lib install
+  ./b2 $j link=shared dll-path=${:location}/lib:${bzip2:location}/lib:${gcc:location}/lib:${gcc:location}/lib64:${zlib:location}/lib:${xz-utils:location}/lib install
 environment =
   PATH=${gcc:location}/bin:%(PATH)s
   BZIP2_INCLUDE=${bzip2:location}/include
   BZIP2_LIBPATH=${bzip2:location}/lib
   ZLIB_INCLUDE=${zlib:location}/include
   ZLIB_LIBPATH=${zlib:location}/lib
+  LZMA_INCLUDE=${xz-utils:location}/include
+  LZMA_LIBRARY_PATH=${xz-utils:location}/lib
\ No newline at end of file

component/coturn/buildout.cfg

@@ -13,4 +13,4 @@ environment =
   PATH=${sqlite3:location}/bin:%(PATH)s
   CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
   CFLAGS=-I${libevent2:location}/include
-  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib

component/cython/buildout.cfg

@@ -4,3 +4,6 @@ parts = cython
 [cython]
 recipe = zc.recipe.egg:custom
 egg = cython
+
+[versions]
+Cython = 0.28.2

component/gcc/buildout.cfg

@@ -7,8 +7,8 @@ extends =
   ../patch/buildout.cfg
   ../perl/buildout.cfg
   ../tar/buildout.cfg
-  ../xz-utils/buildout.cfg
   ../binutils/buildout.cfg
+  ../zlib/buildout.cfg
 
 parts =
   gcc
@@ -16,16 +16,7 @@ parts =
 [gcc-common]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/gnu/gcc/gcc-5.5.0/gcc-5.5.0.tar.gz
-md5sum = 781bc0195edeb0ceaace8428f63ae63d
-# make install does not work when several core are used
-make-targets = install -j1
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/libsanitizer_Use_pre-computed_size_of_struct_ustat_for_Linux.patch#1e5f33e89f9fe1ca3e406eabcc621762
-
-[gcc]
-<= gcc-common
+url = http://ftp.gnu.org/gnu/gcc/gcc-${:version}/gcc-${:version}.tar.xz
 configure-options =
   --disable-bootstrap
   --disable-multilib
@@ -36,12 +27,32 @@ configure-options =
   --with-isl=${isl:location}
   --with-ld=${binutils:location}/bin/ld
   --with-as=${binutils:location}/bin/as
+post-install =
+  cd '@@LOCATION@@/bin'
+  ln -s gcc cc
 environment =
-  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
   PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
 
-[gcc-minimal]
+[gcc]
 <= gcc-common
+version = 5.5.0
+md5sum = 0f70424213b4a1113c04ba66ddda0c1f
+# make install does not work when several core are used
+make-targets = install -j1
+patch-binary = ${patch:location}/bin/patch
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/libsanitizer_Use_pre-computed_size_of_struct_ustat_for_Linux.patch#1e5f33e89f9fe1ca3e406eabcc621762
+
+[gcc-8.2]
+<= gcc-common
+version = 8.2.0
+md5sum = 4ab282f414676496483b3e1793d07862
+
+[gcc-minimal]
+<= gcc
 configure-options =
   --disable-bootstrap
   --disable-multilib
@@ -54,23 +65,3 @@ configure-options =
 environment =
   LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
   PATH=${perl:location}/bin:${tar:location}/bin:%(PATH)s
-
-
-[gcc-8.2]
-recipe = slapos.recipe.cmmi
-shared = true
-url = https://ftp.gnu.org/gnu/gcc/gcc-8.2.0/gcc-8.2.0.tar.gz
-md5sum = ee04f0c22a941f5f17d93809387f2729
-configure-options =
-  --disable-bootstrap
-  --disable-multilib
-  --with-gmp=${gmp:location}
-  --with-mpfr=${mpfr:location}
-  --with-mpc=${mpc:location}
-  --enable-languages="c,c++,fortran"
-  --with-isl=${isl:location}
-  --with-ld=${binutils:location}/bin/ld
-  --with-as=${binutils:location}/bin/as
-environment =
-  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
-  PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s

component/inkscape/buildout.cfg

@@ -46,8 +46,6 @@ pkg_config_depends = ${freetype:location}/lib/pkgconfig:${gtkmm:location}/lib/pk
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
   -DCMAKE_INSTALL_PREFIX=${:location}
-  -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-  -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
   -DENABLE_POPPLER=OFF
   -DWITH_GNOME_VFS=OFF
   -DWITH_IMAGE_MAGICK=OFF

component/keras/buildout.cfg

@@ -28,4 +28,3 @@ scripts = keras-python
 Keras = 2.1.0
 tensorflow = 1.4.0
 h5py = 2.7.0rc2
-Cython = 0.25.2

component/libcap/buildout.cfg

@@ -6,32 +6,23 @@ extends =
 parts = libcap
 
 [libcap2]
-recipe = slapos.recipe.build
+recipe = slapos.recipe.cmmi
+shared = true
 url = http://pkgs.fedoraproject.org/lookaside/pkgs/libcap/libcap-2.22.tar.bz2/ce64058bdb3f086ddbfca8ce6c919845/libcap-2.22.tar.bz2
 md5sum = ce64058bdb3f086ddbfca8ce6c919845
-attr-include = ${attr:location}/include/
-attr-lib = ${attr:location}/lib/
-slapos_promise =
-  directory:sbin
-  directory:include
-  statlib:lib/libcap.a
-  file:lib/libcap.so
-  file:sbin/getcap
-  file:sbin/setcap
-install =
-  import os
-  url = self.download(options['url'], options['md5sum'])
-  extract_dir = self.extract(url)
-  workdir = guessworkdir(extract_dir)
-  cflags = '-I%(attr-include)s' % options
-  ldflags = '-L%(attr-lib)s -Wl,-rpath=%(attr-lib)s' % options
-  call(['make', 'CFLAGS=' + cflags, 'LDFLAGS=' + ldflags, 'DESTDIR=' + location, 'RAISE_SETFCAP=no', 'prefix=', 'install'],
-       cwd=workdir, env=self.environ)
-  lib64 = os.path.join(location, 'lib64')
-  lib = os.path.join(location, 'lib')
-  # XXX: Dirty if case
-  #      if lib64 exists, then create a symlink from lib to lib64
-  os.path.exists(lib64) and os.symlink(lib64, lib)
+location = @@LOCATION@@
+configure-command = :
+make-options =
+  RAISE_SETFCAP=no
+  prefix=${:location}
+  CFLAGS=-I${attr:location}/include/
+  LDFLAGS="-L${attr:location}/lib/ -Wl,-rpath=${attr:location}/lib/ -Wl,-rpath=${:location}/lib/"
+  install
+
+# if lib64 exists, then create a symlink from lib to lib64
+post-install =
+  cd ${:location}
+  [ -d lib64 ] && ln -s lib64 lib
 
 [libcap]
 <= libcap2
\ No newline at end of file

component/libevent/buildout.cfg

@@ -7,6 +7,7 @@ extends =
 recipe = slapos.recipe.cmmi
 url = http://www.monkey.org/~provos/libevent-1.4.13-stable.tar.gz
 md5sum = 0b3ea18c634072d12b3c1ee734263664
+shared = true
 configure-options =
   --disable-static
 

component/libzip/buildout.cfg

@@ -26,4 +26,4 @@ environment =
   PATH=${cmake:location}/bin:%(PATH)s
   CMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
   CMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
-  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib

component/lz4/buildout.cfg

@@ -5,6 +5,7 @@ parts =
 
 [lz4]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://github.com/lz4/lz4/archive/v1.9.2.tar.gz
 md5sum = 3898c56c82fb3d9455aefd48db48eaad
 configure-command = true

component/mariadb/buildout.cfg

@@ -34,7 +34,7 @@ version = 10.4.12
 md5sum = 97d7c0f508c04a31c138fdb24e95dbc4
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 pre-configure =
-  set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
+  set '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
   grep -q "$@"
   sed -i "/$1/d" "$2"
 configure-command = ${cmake:location}/bin/cmake
@@ -68,8 +68,6 @@ configure-options =
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
   -DCMAKE_INCLUDE_PATH=${unixodbc:location}/include
   -DCMAKE_LIBRARY_PATH=${unixodbc:location}/lib
-  -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-  -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
 CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include -I${lz4:location}/include -I${snappy:location}/include -I${zstd:location}/include
 CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${lz4:location}/lib:${snappy:location}/lib:${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64
 environment =
@@ -77,11 +75,11 @@ environment =
   CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${lz4:location}/include:${snappy:location}/include:${zstd:location}/include
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
   LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib -L${lz4:location}/lib -L${snappy:location}/lib -L${zstd:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s
+  PATH=${gcc:location}/bin:${patch:location}/bin:%(PATH)s
 patch-options = -p1
 patches =
   https://sources.debian.org/data/main/m/mariadb-10.3/1:10.3.22-0+deb10u1/debian/patches/0024-Revert-to-using-system-pcre-library.patch#1c6a0f2634f5a56122299674b77b1131
-post-install = set -e
+post-install = 
   set -- wsrep-lib/wsrep-API/*/wsrep_api.h
   install -DpT $1 ${:location}/$1
   cp -a wsrep-lib/include ${:location}/wsrep-lib
@@ -92,7 +90,7 @@ post-install = set -e
 recipe = slapos.recipe.cmmi
 url = https://packages.groonga.org/source/mroonga/mroonga-9.12.tar.gz
 md5sum = d0af673f1bad3b9ccf33870bb2344a25
-pre-configure = set -e
+pre-configure = 
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source
   cd fake_mariadb_source

component/nano/buildout.cfg

@@ -4,6 +4,7 @@ parts =
 
 extends =
   ../file/buildout.cfg
+  ../zlib/buildout.cfg
   ../ncurses/buildout.cfg
 
 [nano]
@@ -17,8 +18,8 @@ location = @@LOCATION@@
 environment=
   PKG_CONFIG=false
   NCURSESW_CONFIG=${ncurses:location}/bin/ncursesw6-config
-  CPPFLAGS=-I${file:location}/include
-  LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/
+  CPPFLAGS=-I${file:location}/include -I${zlib:location}/include
+  LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/ -L${zlib:location}/lib/ -Wl,-rpath=${zlib:location}/lib/
 post-install =
   cd ${:location} && mkdir etc &&
   echo include "${:location}/share/nano/*.nanorc" > etc/nanorc

component/nodejs/buildout.cfg

@@ -15,6 +15,12 @@ parts =
 [nodejs]
 <= nodejs-8.9.4
 
+[nodejs-10.19.0]
+<= nodejs-base
+openssl_location = ${openssl:location}
+version = v10.19.0
+md5sum = 9e433c753d839d2d2a6341861501674f
+
 [nodejs-10.6.0]
 <= nodejs-base
 openssl_location = ${openssl:location}

component/onlyoffice-core/buildout.cfg

@@ -25,7 +25,7 @@ environment =
   CXXFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${icu4c-slaposgcc:location}/include -I${boost-lib:location}/include -Wno-comment -Wno-deprecated-declarations -Wno-endif-labels -Wno-parentheses -Wno-reorder -Wno-sign-compare -Wno-switch -Wno-unknown-pragmas -Wno-unused
   LDFLAGS=-L${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib -L${gcc:location}/lib64 -Wl,-rpath=${gcc:location}/lib64 -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${icu4c-slaposgcc:location}/lib -Wl,-rpath=${icu4c-slaposgcc:location}/lib -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -Wl,-rpath=${:location}/lib
 post-install =
-  set -e -x
+  set -x
   mkdir -p ${:location}/bin ${:location}/lib
   mv -t ${:location}/lib build/lib/*/*.so
   mv -t ${:location}/bin build/bin/*/*

component/pole/buildout.cfg

@@ -12,7 +12,7 @@ make-binary =
 make-options =
 make-targets = true
 post-install =
-  set -e -x
+  set -x
   mkdir bin
   cd pole
   g++ -o ../bin/poledump pole.cpp poledump.cpp

component/protobuf/buildout.cfg

 [buildout]
+extends =
+  ../zlib/buildout.cfg
+
 parts = protobuf
 
 [protobuf]
@@ -6,9 +9,13 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-python-3.4.0.tar.gz
 md5sum = 0820cc2e56d71aef8e99794fcbd184cd
+environment =
+  LDFLAGS=-Wl,-rpath=${zlib:location}/lib
 
 [protobuf-cpp]
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
 md5sum = 6d59dad503bea5ad420fd09ddad84481
+environment =
+  LDFLAGS=-Wl,-rpath=${zlib:location}/lib
\ No newline at end of file

component/qt/buildout.cfg

@@ -47,7 +47,7 @@ url = http://download.qt.io/archive/qt/4.8/4.8.7/qt-everywhere-opensource-src-4.
 md5sum = d990ee66bf7ab0c785589776f35ba6ad
 # see https://github.com/NixOS/nixpkgs/blob/3e387c3e005c87566b5403d24c86f71f4945a79b/pkgs/development/libraries/qt-4.x/4.8/default.nix#L101
 pre-configure =
-  set -e -x
+  set -x
   sed 's,/usr/X11R6/lib64,${libX11:location}/lib64 ${xproto:location}/lib64 ${libXext:location}/lib64,g' -i mkspecs/*/*.conf
   sed 's,/usr/X11R6/lib,${libX11:location}/lib ${xproto:location}/lib ${libXext:location}/lib,g' -i mkspecs/*/*.conf
   sed 's,/usr/X11R6/include,${libX11:location}/include ${xproto:location}/include ${libXext:location}/include,g' -i mkspecs/*/*.conf

component/rpy2/buildout.cfg

@@ -27,6 +27,7 @@ rpath =
   ${readline:location}/lib
   ${xz-utils:location}/lib
   ${r-language:location}/lib/R/lib
+  ${r-language:location}/lib64/R/lib
 library-dirs =
   ${pcre:location}/lib
   ${readline:location}/lib

component/screen/buildout.cfg

@@ -3,6 +3,7 @@ parts =
   screen
 extends =
   ../ncurses/buildout.cfg
+  ../patch/buildout.cfg
   ../texinfo/buildout.cfg
 
 [screen]
@@ -11,7 +12,10 @@ shared = true
 url = http://ftp.gnu.org/gnu/screen/screen-${:version}.tar.gz
 version = 4.8.0
 md5sum = d276213d3acd10339cd37848b8c4ab1e
+patches =
+  ${:_profile_base_location_}/dependencies.diff
+patch-options = -p2
 environment =
   CFLAGS=-I${ncurses:location}/include
   LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
-  PATH=${texinfo:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:${texinfo:location}/bin:%(PATH)s

component/sqlite3/buildout.cfg

 [buildout]
-extends = ../readline/buildout.cfg
+extends =
+  ../readline/buildout.cfg
+  ../zlib/buildout.cfg
 parts =
   sqlite3
 
@@ -14,5 +16,5 @@ configure-options =
 # Increase MAX_VARIABLE_NUMBER like many os. For example:
 # https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/sqlite
 environment =
-  CPPFLAGS=-I${readline:location}/include -I${ncurses:location}/include -DSQLITE_MAX_VARIABLE_NUMBER=250000
-  LDFLAGS=-L@@LOCATION@@ -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib
+  CPPFLAGS=-I${readline:location}/include -I${ncurses:location}/include -I${zlib:location}/include -DSQLITE_MAX_VARIABLE_NUMBER=250000
+  LDFLAGS=-L@@LOCATION@@ -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/tensorflow/buildout.cfg

-# [buildout]
+[buildout]
 extends =
   ../../stack/slapos.cfg
   ../scipy/buildout.cfg

component/tmux/buildout.cfg

@@ -10,8 +10,9 @@ parts +=
 recipe = slapos.recipe.cmmi
 url = https://github.com/tmux/tmux/releases/download/2.9a/tmux-2.9a.tar.gz
 md5sum = f0564dbf4cf6b301c4845219d27ed3ad
+shared = true
 environment =
   CFLAGS=-I${ncurses:location}/include -I${libevent2:location}/include/
   LDFLAGS=-L${ncurses:location}/lib/ -L${libevent2:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/ -Wl,-rpath=${libevent2:location}/lib/
 configure-options =
-  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --prefix=@@LOCATION@@

component/wendelin.core/buildout.cfg

@@ -14,7 +14,7 @@ egg = wendelin.core
 [wendelin.core-dev]
 recipe = zc.recipe.egg:develop
 egg = wendelin.core
-setup = ${wendelin.core-repository-submoduleinit:location}
+setup = ${wendelin.core-repository:location}
 environment = wendelin.core-dev-env
 
 [wendelin.core-dev-env]
@@ -28,13 +28,3 @@ repository = https://lab.nexedi.com/nexedi/wendelin.core.git
 # dir is pretty name as top-level -dev recipe
 location = ${buildout:parts-directory}/wendelin.core-dev
 git-executable = ${git:location}/bin/git
-
-
-# TODO add `git clone --recursive` to slapos.recipe.build:gitclone
-#      and this way merge this -submoduleinit into -repository part
-[wendelin.core-repository-submoduleinit]
-recipe = plone.recipe.command
-command = cd "${wendelin.core-repository:location}" && ${git:location}/bin/git submodule update --init
-stop-on-error = true
-# propagate location of main repo
-location= ${wendelin.core-repository:location}

component/zstd/buildout.cfg

 [buildout]
-extends = ../zlib/buildout.cfg
+extends =
+  ../zlib/buildout.cfg
+  ../xz-utils/buildout.cfg
+  ../lz4/buildout.cfg
 parts =
   zstd
 
 [zstd]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://github.com/facebook/zstd/releases/download/v${:version}/zstd-${:version}.tar.gz
 version = 1.4.4
 md5sum = 487f7ee1562dee7c1c8adf85e2a63df9
-shared = true
 location = @@LOCATION@@
 configure-command = :
 environment =
   PREFIX=${:location}
-  LD_FLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-  CFLAGS=-I${zlib:location}/include
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib -L${lz4:location}/lib -Wl,-rpath=${lz4:location}/lib
+  CFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${lz4:location}/include

release-cookbook

+#!/bin/bash
+
+git fetch
+
+git checkout master
+git reset origin/master --hard
+git clean -f
+
+# Remove any local tag
+git fetch --prune origin "+refs/tags/*:refs/tags/*"
+rm -rf dist/slapos.cookbook*
+
+# for now this script is hardcoded to release on 1.0.x versions intentionally
+# update to version 2 or 1.1 would require a major reorganisation on the release
+# process
+CURRENT_VERSION=`git tag | grep "^1\+\.0\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1`
+NEXT_MINOR_VERSION=$((`echo $CURRENT_VERSION | cut -f3 -d.`+1))
+NEXT_VERSION=1.0.$NEXT_MINOR_VERSION
+
+CURRENT_EGG_VERSION=`cat setup.py | grep ^version | cut -d\' -f2`
+
+sed -i "s/$CURRENT_EGG_VERSION/$NEXT_VERSION/g" setup.py
+git commit -m "Release slapos.cookbook ($NEXT_VERSION)" setup.py
+
+if [ ! $EGG_ONLY == "Y" ];
+then
+  sed -i "s/slapos.cookbook = $CURRENT_EGG_VERSION/slapos.cookbook = $NEXT_VERSION/g" stack/slapos.cfg
+  sed -i "s/slapos.cookbook==$CURRENT_EGG_VERSION/slapos.cookbook==$NEXT_VERSION/g" stack/slapos.cfg
+  git commit -m "stack/slapos: slapos.cookbook version up ($NEXT_VERSION)" stack/slapos.cfg
+fi
+
+echo "############### Merging into 1.0 and generating the tag ################"
+git checkout 1.0
+git reset --hard origin/1.0 
+git clean -f
+git merge master -m "Update Release Candidate"
+
+# Revert changes from master
+git diff HEAD..master | git apply 
+
+# Download patch to update revisions based on tests
+wget -O update-release.sh https://nexedi.erp5.net/portal_skins/custom/TestResultModule_getReleaseCandidateRevision 
+bash update-release.sh
+git commit -m 'Update git revisions' -a  
+git tag $NEXT_VERSION -m "Release $NEXT_VERSION"
+
+echo "############### Building sdist  ################"
+python setup.py sdist
+
+
+echo "###################################################################"
+echo "You are about to release a new tag, a new version of slapos.cookbook"
+echo " and a new version of SlapOS Software Release"
+echo ""
+echo "Lastest release: $CURRENT_VERSION"
+echo "Next Release to be Tagged: $NEXT_VERSION"
+echo "Current slapos.cookbook version: $CURRENT_EGG_VERSION"
+echo "Next slapos.cookbook to be released: $NEXT_VERSION"
+echo ""
+echo "###################################################################"
+echo ""
+echo ""
+echo "########################################################################"
+echo " Please review the current local changes before continue. Ensure that"
+echo " ALL changes and commits are correct before continue."
+echo ""
+echo " To review master use :: git log master"
+echo " To review 1.0 use    :: git log 1.0"
+echo " To review $NEXT_VERSION use :: git log $NEXT_VERSION"
+echo " To review individual commits :: git show HASH"
+echo ""
+echo ""
+echo "Once everything is ok, please upload slapos.cookbook egg /!\\ BEFORE /!\\" 
+echo "push your changes into the repository. Please use twine for it."
+echo ""
+echo "To upload :: python -m twine upload dist/slapos.cookbook-$NEXT_VERSION*"
+echo "To verify if upload succeed access: https://pypi.org/project/slapos.cookbook/$NEXT_VERSION/"
+echo ""
+echo " More info on twine: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives"
+echo ""
+echo "Once egg is updated please push your changes into main repository:"
+echo ""
+echo " git push origin master 1.0 $NEXT_VERSION"
+echo ""
+echo " !! DO NOT FORGET TO PUSH YOUR CHANGES IN THE END !!"
+echo ""
+echo "###################################################################"
+echo ""

release-sr

+#!/bin/bash
+
+git fetch
+
+git checkout 1.0
+git reset origin/1.0 --hard
+
+# for now this script is hardcoded to release on 1.0.x versions intentionally
+# update to version 2 or 1.1 would require a major reorganisation on the release
+# process
+CURRENT_VERSION=`git tag | grep "^1\+\.0\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1`
+
+NEXT_MINOR_VERSION=$((`echo $CURRENT_VERSION | cut -f3 -d.`+1))
+
+echo "###################################################################"
+echo "You are about to release a new version of SlapOS Software Release"
+echo "Lastest release: $CURRENT_VERSION"
+echo "Next Release to be Tagged: 1.0.$NEXT_MINOR_VERSION"
+echo "###################################################################"
+
+git tag 1.0.$NEXT_MINOR_VERSION -m "Release 1.0.$NEXT_MINOR_VERSION"
+
+
+echo " Please review current tag and them push it:"
+echo " To review use git log 1.0.$NEXT_MINOR_VERSION"
+
+echo ""
+echo " To push into main repository:"
+echo " git push origin 1.0.$NEXT_MINOR_VERSION"

setup.py

@@ -141,8 +141,6 @@ setup(name=name,
           'onetimeupload = slapos.recipe.onetimeupload:Recipe',
           'pbs = slapos.recipe.pbs:Recipe',
           'postgres = slapos.recipe.postgres:Recipe',
-          'postgres.export = slapos.recipe.postgres.backup:ExportRecipe',
-          'postgres.import = slapos.recipe.postgres.backup:ImportRecipe',
           'proactive = slapos.recipe.proactive:Recipe',
           'promise.plugin= slapos.recipe.promise_plugin:Recipe',
           'publish = slapos.recipe.publish:Recipe',

slapos/recipe/postgres/__init__.py → slapos/recipe/postgres.py

@@ -42,7 +42,8 @@ class Recipe(GenericBaseRecipe):
 
         - a Postgres cluster
         - configuration to allow connections from IPv4, IPv6 or unix socket.
-        - a superuser with provided name and generated password
+          IPv4 and IPv6 can be disabled, unix socket will always be available.
+        - a superuser with provided name and password
         - a database with provided name
         - a start script in the services directory
 
@@ -52,50 +53,59 @@ class Recipe(GenericBaseRecipe):
         dbname
             name of the database to be used by the application.
         ipv4
-            set of ipv4 to listen on.
+            ipv4 to listen on, can be multiple ips or can be empty.
         ipv6
-            set of ipv6 to listen on.
+            ipv6 to listen on, can be multiple ips or can be empty.
+        port
+            port to listen on, same for both IPv4 and IPv6.
         pgdata-directory
             path to postgres configuration and data.
         services
             must be ${buildout:directory}/etc/service.
         superuser
             name of the superuser to create.
+        password
+            password for the superuser.
 
     Exposed options:
-        password
-            generated password for the superuser.
         url
-            generated DBAPI connection string.
+            generated DBAPI connection string, on IPv6.
             it can be used as-is (ie. in sqlalchemy) or by the _urlparse.py recipe.
+            this is only available if at least one IPv6 was provided.
     """
 
     def _options(self, options):
-        options['url'] = 'postgresql://%(superuser)s:%(password)s@[%(ipv6-random)s]:%(port)s/%(dbname)s' % options
-
+        if options.get('ipv6'):
+            options['url'] = "postgresql://{superuser}:{password}@[{ipv6}]:{port}/{dbname}".format(
+                superuser=options['superuser'],
+                password=options['password'],
+                ipv6=options['ipv6'].splitlines()[0],
+                port=options['port'],
+                dbname=options['dbname'],
+            )
 
     def install(self):
         pgdata = self.options['pgdata-directory']
 
-        # if the pgdata already exists, skip all steps, we don't need to do anything.
-
+        paths = []
+        # if the pgdata already exists, we don't need to recreate databases.
         if not os.path.exists(pgdata):
             try:
                 self.createCluster()
-                self.createConfig()
+                paths.extend(self.createConfig())
                 self.createDatabase()
                 self.updateSuperuser()
-                self.createRunScript()
+                paths.extend(self.createRunScript())
             except:
                 # do not leave half-installed postgresql - else next time we
                 # run we won't update it.
                 shutil.rmtree(pgdata)
                 raise
         else:
-            self.createConfig()
-            self.createRunScript()
+            paths.extend(self.createConfig())
+            paths.extend(self.createRunScript())
 
-        return []
+        return paths
 
     update = install
 
@@ -129,10 +139,11 @@ class Recipe(GenericBaseRecipe):
 
     def createConfig(self):
         pgdata = self.options['pgdata-directory']
-        ipv4 = self.options['ipv4']
-        ipv6 = self.options['ipv6']
+        ipv4 = self.options['ipv4'].splitlines()
+        ipv6 = self.options['ipv6'].splitlines()
 
-        with open(os.path.join(pgdata, 'postgresql.conf'), 'wb') as cfg:
+        postgres_conf = os.path.join(pgdata, 'postgresql.conf')
+        with open(postgres_conf, 'w') as cfg:
             cfg.write(textwrap.dedent("""\
                     listen_addresses = '%s'
                     logging_collector = on
@@ -149,11 +160,12 @@ class Recipe(GenericBaseRecipe):
                     unix_socket_directories = '%s'
                     unix_socket_permissions = 0700
                     """ % (
-                        ','.join(ipv4.union(ipv6)),
+                        ','.join(set(ipv4).union(ipv6)),
                         pgdata,
                         )))
 
-        with open(os.path.join(pgdata, 'pg_hba.conf'), 'wb') as cfg:
+        pg_hba_conf = os.path.join(pgdata, 'pg_hba.conf')
+        with open(pg_hba_conf, 'w') as cfg:
             # see http://www.postgresql.org/docs/9.2/static/auth-pg-hba-conf.html
 
             cfg_lines = [
@@ -174,7 +186,7 @@ class Recipe(GenericBaseRecipe):
                 cfg_lines.append('host    all             all             %s/%s                   md5' % (ip, ipv6_netmask_bits))
 
             cfg.write('\n'.join(cfg_lines))
-
+        return postgres_conf, pg_hba_conf
 
     def createDatabase(self):
         self.runPostgresCommand(cmd='CREATE DATABASE "%s"' % self.options['dbname'])
@@ -232,6 +244,6 @@ class Recipe(GenericBaseRecipe):
                     -D %(pgdata-directory)s
                 """ % self.options)
         name = os.path.join(self.options['services'], 'postgres-start')
-        self.createExecutable(name, content=content)
+        return [self.createExecutable(name, content=content)]
 
 

slapos/test/recipe/test_postgres.py

+import unittest
+import tempfile
+import shutil
+import os.path
+import zc.buildout.testing
+
+
+class PostgresTest(unittest.TestCase):
+  def setUp(self):
+    self.buildout = buildout = zc.buildout.testing.Buildout()
+    self.pgdata_directory = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.pgdata_directory)
+    self.services_directory = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.services_directory)
+    buildout['postgres'] = {
+      'bin': 'software/parts/postgres/bin/',
+      'dbname': 'dbname',
+      'ipv4': '127.0.0.1',
+      'ipv6': '::1',
+      'port': '5443',
+      'pgdata-directory': self.pgdata_directory,
+      'services': self.services_directory,
+      'superuser': 'superuser',
+      'password': 'secret',
+    }
+    from slapos.recipe import postgres
+    self.recipe = postgres.Recipe(
+        buildout,
+        'postgres',
+        buildout['postgres'])
+
+  def test_options(self):
+    self.assertEqual(
+        'postgresql://superuser:secret@[::1]:5443/dbname',
+        self.buildout['postgres']['url'])
+
+  def test_install(self):
+    installed = self.recipe.install()
+
+    self.assertIn('postgresql.conf', os.listdir(self.pgdata_directory))
+    self.assertIn('pg_hba.conf', os.listdir(self.pgdata_directory))
+    self.assertIn('postgres-start', os.listdir(self.services_directory))
+
+    self.assertEqual(
+        sorted(installed),
+        sorted([
+            os.path.join(self.pgdata_directory, 'postgresql.conf'),
+            os.path.join(self.pgdata_directory, 'pg_hba.conf'),
+            os.path.join(self.services_directory, 'postgres-start')]))

software/apache-frontend/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = f686f765e55d1dce2e55a400f0714b3e
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg
-md5sum = eb2143422d824e96abe99619f11d3523
+md5sum = ec825cba3ea775452716f1da7cd2e549
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in

software/apache-frontend/common.cfg

@@ -38,9 +38,9 @@ parts +=
   proxy-by-url
   http-proxy
 
-# Extent extra-eggs.
 [extra-eggs]
-eggs +=
+recipe = zc.recipe.egg
+eggs =
   websockify
   erp5.util
 

software/apache-frontend/instance-apache-frontend.cfg

@@ -248,10 +248,7 @@ extra-context =
 [apache-frontend]
 recipe = slapos.cookbook:wrapper
 command-line = ${apache:location}/bin/httpd -f $${dynamic-apache-frontend-template:rendered} -DFOREGROUND
-wrapper-path = $${directory:service}/frontend_apache
-wait-for-files =
-	       $${ca-frontend:cert-file}
-	       $${ca-frontend:key-file}
+wrapper-path = $${directory:bin}/frontend_apache
 
 [not-found-html]
 recipe = slapos.cookbook:symbolic.link
@@ -315,7 +312,7 @@ crl = $${directory:ca-dir}/crl/
 recipe = slapos.cookbook:certificate_authority.request
 key-file = $${cadirectory:certs}/apache_frontend.key
 cert-file = $${cadirectory:certs}/apache_frontend.crt
-executable = $${directory:service}/frontend_apache
+executable = $${apache-frontend:wrapper-path}
 wrapper = $${directory:service}/frontend_apache
 key-content = $${instance-parameter:configuration.apache-key}
 cert-content = $${instance-parameter:configuration.apache-certificate}

software/caddy-frontend/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = 087bd9404cd120bd7602a9fbfcddc064
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 46e0ccb54c2406b8a44cdada73c061a6
+md5sum = 29a61267959cc9ba7cdcd96fef41641a
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in

software/caddy-frontend/common.cfg

@@ -16,6 +16,7 @@ extends =
   ../../stack/monitor/buildout.cfg
 
 parts +=
+  caucase-eggs
   template
   template-caddy-frontend
   template-caddy-replicate
@@ -72,13 +73,7 @@ depends = ${caddyprofiledeps-develop:recipe}
 recipe = zc.recipe.egg
 eggs =
   caddyprofiledeps
-
-# Extent extra-eggs.
-[extra-eggs]
-eggs +=
   websockify
-  erp5.util
-  ${caucase-eggs:eggs}
   collective.recipe.shelloutput
 
 [template-common]

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -352,15 +352,6 @@ recipe = slapos.cookbook:mkdirectory
 
 {% do part_list.append('slave-log-cache-direct-directories') %}
 
-[caddy-log-access]
-< = jinja2-template-base
-template = {{frontend_configuration.get('template-log-access')}}
-rendered = {{frontend_configuration.get('log-access-configuration')}}
-extra-context =
-    section slave_log_directory slave-log-directory-dict
-    section slave_password slave-password
-    section parameter_dict caddy-log-access-parameters
-
 {% do part_list.append('caddy-log-access') %}
 
 ###############################################
@@ -474,11 +465,11 @@ template = inline:
 
 rendered = ${:file}
 
-[caddy-log-access-header]
+[caddy-log-access-empty]
 # Caddy refuse to start if an `import`ed file is empty, so we prepend a header
 # so that the file is never empty.
 < = jinja2-template-base
-template = inline: # This file contain directives to serve directories with log files
+template = inline: # This file contain directives to serve directories with log files for shared instances, but no shared instances are defined yet.
 rendered = {{frontend_configuration.get('log-access-configuration')}}
 
 [buildout]
@@ -490,10 +481,12 @@ extends =
 parts +=
     kedifa-updater
     kedifa-updater-run
-    caddy-log-access-header
 {% for part in part_list %}
 {{ '    %s' % part }}
 {% endfor %}
+{% if 'caddy-log-access' not in part_list %}
+    caddy-log-access-empty
+{% endif %}
     publish-caddy-information
     tunnel-6to4-base-http_port
     tunnel-6to4-base-https_port

software/dream/buildout.hash.cfg

+[instance]
+filename = instance.cfg
+md5sum = 94e1cd21bbdf7cabb5834223859407e0
\ No newline at end of file

software/dream/instance.cfg

@@ -6,22 +6,26 @@ parts =
   dream_interpreter
   grunt_watch
   publish-connection-parameter
+  dream-platform-url-available
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
+extends = ${monitor-template:rendered}
 
 # parameters
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
 [dream_platform_parameter]
-port = 8080
+port = 18080
 host = $${instance-parameter:ipv6-random}
+url = http://[$${:host}]:$${:port}
 
 # interpreter
 [dream_interpreter]
@@ -35,6 +39,13 @@ recipe = slapos.cookbook:wrapper
 command-line = ${buildout:bin-directory}/dream_platform --debug --host $${dream_platform_parameter:host} --port $${dream_platform_parameter:port} --log $${directory:log}/dream_platform.log
 wrapper-path = $${directory:service}/dream_platform
 
+[dream-platform-url-available]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url=  $${dream_platform_parameter:url}
+
+
 [grunt_watch]
 recipe = slapos.cookbook:wrapper
 command-line = bash -c 'cd ${dream-repository.git:location}; PATH=${nodejs:location}/bin/:$PATH ${dream-repository.git:location}/node_modules/grunt-cli/bin/grunt watch -f > $${directory:log}/grunt.log'
@@ -63,5 +74,5 @@ log = $${:var}/log
 
 [publish-connection-parameter]
 recipe = slapos.cookbook:publishurl
-url = http://[$${dream_platform_parameter:host}]:$${dream_platform_parameter:port}
+url = $${dream_platform_parameter:url}
 

software/dream/software.cfg

@@ -4,17 +4,18 @@ extends =
   ../../stack/slapos.cfg
   ../../stack/nodejs.cfg
   ../../component/manpy/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ./buildout.hash.cfg
 parts =
   slapos-cookbook
   manpy
   dream_testrunner
-  dream_interpreter
   npm_install
   instance
 
 [instance]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/instance.cfg
 
 [dream_testrunner]
@@ -24,17 +25,10 @@ script = dream_testrunner
 initialization =
   ${manpy:initialization}
 
-[dream_interpreter]
-recipe = zc.recipe.egg
-eggs = ${manpy:eggs}
-interpreter = dream_interpreter
-initialization =
-  ${manpy:initialization}
-
 [npm_install]
 recipe = plone.recipe.command
 stop-on-error = true
-command = cd ${dream-repository.git:location} && PATH=${git:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
+command = cd ${dream-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
 update_command = ${:command}
 
 [versions]
@@ -42,7 +36,6 @@ rpy2 = 2.4.0
 pydot = 1.0.28
 xlrd = 0.9.3
 xlwt = 0.7.5
-pyparsing = 2.0.3
 numpy = 1.16.4
 scipy = 0.13.3
 simpy = 3.0.5

software/dream/test/README.md

+Tests for DREAM software release

software/dream/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.dream'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' DREAM",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.cookbook',
+        'slapos.libnetworkcache',
+        'requests'
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

slapos/recipe/postgres/backup.py → software/dream/test/test.py

 ##############################################################################
+# coding: utf-8
 #
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -25,91 +26,24 @@
 #
 ##############################################################################
 
-import textwrap
+import os
 
-from slapos.recipe.librecipe import GenericBaseRecipe
+import requests
 
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
 
-class ExportRecipe(GenericBaseRecipe):
-    """\
-    This recipe creates an exporter script for using with the resilient stack.
+setUpModule, DREAMTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
-    Required options:
-        backup-directory
-            folder that will contain the dump file.
-        bin
-            path to the 'pg_dump' binary.
-        dbname
-            name of the database to dump.
-        pgdata-directory
-            path to postgres configuration and data.
-        wrapper
-            full path of the exporter script to create.
-    """
-
-    def install(self):
-        wrapper = self.options['wrapper']
-        self.createBackupScript(wrapper)
-        return [wrapper]
-
-
-    def createBackupScript(self, wrapper):
-        """\
-        Create a script to backup the database in 'custom' format.
-        """
-        content = textwrap.dedent("""\
-                #!/bin/sh
-                umask 077
-                %(bin)s/pg_dump \\
-                        --host=%(pgdata-directory)s \\
-                        --username postgres \\
-                        --format=custom \\
-                        --file=%(backup-directory)s/database.dump \\
-                        %(dbname)s
-                """ % self.options)
-        self.createExecutable(wrapper, content=content)
-
-
-
-class ImportRecipe(GenericBaseRecipe):
-    """\
-    This recipe creates an importer script for using with the resilient stack.
-
-    Required options:
-        backup-directory
-            folder that contains the dump file.
-        bin
-            path to the 'pg_restore' binary.
-        dbname
-            name of the database to restore.
-        pgdata-directory
-            path to postgres configuration and data.
-        wrapper
-            full path of the importer script to create.
-    """
-
-    def install(self):
-        wrapper = self.options['wrapper']
-        self.createRestoreScript(wrapper)
-        return [wrapper]
-
-
-    def createRestoreScript(self, wrapper):
-        """\
-        Create a script to restore the database from 'custom' format.
-        """
-        content = textwrap.dedent("""\
-                #!/bin/sh
-                %(bin)s/pg_restore \\
-                        --host=%(pgdata-directory)s \\
-                        --username postgres \\
-                        --dbname=%(dbname)s \\
-                        --clean \\
-                        --no-owner \\
-                        --no-acl \\
-                        %(backup-directory)s/database.dump
-                """ % self.options)
-        self.createExecutable(wrapper, content=content)
 
+class TestHTTPAccess(DREAMTestCase):
+  def setUp(self):
+    self.url = self.computer_partition.getConnectionParameterDict()["url"]
 
+  def test(self):
+    self.assertEqual(
+        requests.codes.ok,
+        requests.get(self.url, verify=False).status_code
+    )

software/erp5testnode/software.cfg

@@ -14,9 +14,9 @@ extends =
   ../../stack/monitor/buildout.cfg
 
 parts =
+  slapos-command
   slapos-cookbook
   template
-  lxml-python
   eggs
   zip
   git
@@ -25,27 +25,8 @@ parts =
 [eggs]
 recipe = zc.recipe.egg
 eggs =
+  erp5.util[testnode]
   ${lxml-python:egg}
-  zc.buildout
-  slapos.libnetworkcache
-  slapos.core
-  slapos.recipe.template
-  supervisor
-  jsonschema
-  hexagonit.recipe.download
-  lock_file
-  pytz
-  erp5.util
-  PyXML
-  z3c.etestbrowser
-
-[testnode]
-
-scripts =
-  testnode = erp5.util.testnode:main
-  slapgrid-cp = slapos.grid.slapgrid:runComputerPartition
-  slapgrid-sr = slapos.grid.slapgrid:runSoftwareRelease
-  slapproxy = slapos.proxy:main
 
 
 [template]

software/gateone/software.cfg

@@ -91,7 +91,6 @@ extra-context =
     key template_logrotate_base template-logrotate-base:rendered
     key template_gateone template-gateone:target
     raw gateone_bin ${buildout:bin-directory}/gateone
-    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
 
 [template-gateone]
 < = download-base

software/grafana/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = 214a756a6a9f73455411760b2daa25b7
+md5sum = 0edf9e67bc637d73a7415124c8082a8c
 
 [influxdb-config-file]
 filename = influxdb-config-file.cfg.in

software/grafana/instance.cfg.in

@@ -126,6 +126,18 @@ command-line =
   {{ influx_bin }} -username ${influxdb:auth-username} -password ${influxdb:auth-password} -socket ${influxdb:unix-socket} -execute "CREATE USER ${influxdb:auth-username} WITH PASSWORD '${influxdb:auth-password}' WITH ALL PRIVILEGES"
 wrapper-path = ${directory:promise}/${:_buildout_section_name_}
 
+[influxdb-database-ready-promise]
+recipe = slapos.cookbook:wrapper
+command-line =
+  bash -c "{{ influx_bin }} \
+     -username ${influxdb:auth-username} \
+     -password ${influxdb:auth-password} \
+     -host ${influxdb:host} \
+     -port ${influxdb:http-port} \
+     -unsafeSsl \
+     -ssl \
+     -execute 'show databases' | grep '${influxdb:database}'"
+wrapper-path = ${directory:promise}/${:_buildout_section_name_}
 
 
 [grafana]
@@ -261,6 +273,7 @@ recipe =
 instance-promises =
   ${influxdb-listen-promise:path}
   ${influxdb-password-promise:wrapper-path}
+  ${influxdb-database-ready-promise:wrapper-path}
   ${grafana-listen-promise:path}
   ${loki-listen-promise:path}
   ${promtail-listen-promise:path}

software/jstestnode/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = e6203cba0084289edec2176b185d3427
+md5sum = 42e7d1825f5582fbe2c55977e5abc574
 
 [template-nginx-service]
 filename = template-nginx-service.sh.in

software/jstestnode/instance.cfg.in

@@ -2,6 +2,7 @@
 parts =
   nginx-service
   runTestSuite-instance
+  promises
   publish
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -22,6 +23,7 @@ log = $${:var}/log
 varnginx = $${:var}/nginx
 scripts = $${:etc}/run
 services = $${:etc}/service
+promise = $${:etc}/promise
 www = $${:srv}/www
 home = $${:etc}/home
 ssl = $${:etc}/ssl
@@ -51,15 +53,20 @@ recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
   ${xserver:location}/bin/Xvfb
-    $${:display}
+    :$${:display}
     -screen 0 1024x768x24
     -fbdir $${directory:framebuffer}
 environment =
   XORG_LOCK_DIR=$${:lock-dir}
 
-display = :0
+display = 0
 lock-dir = $${directory:run}
 
+[xvfb-promise]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:promise}/$${:_buildout_section_name_}
+command-line = bash -c "[ -S $${xvfb-instance:lock-dir}/.X11-unix/X$${xvfb-instance:display} ]"
+
 
 #################################
 # Nginx service
@@ -85,14 +92,27 @@ ssl_key = $${directory:ssl}/nginx.key
 ssl_csr = $${directory:ssl}/nginx.csr
 ssl_crt = $${directory:ssl}/nginx.crt
 
+[nginx-listen-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${directory:promise}/$${:_buildout_section_name_}
+hostname = $${nginx-configuration:ip}
+port = $${nginx-configuration:port}
+
+
 #################################
 # SlapOS service
 #################################
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
 configuration._ = {}
+
+[promises]
+recipe =
+depends =
+  $${nginx-listen-promise:recipe}
+  $${xvfb-promise:recipe}
\ No newline at end of file

software/jupyter/software.cfg

@@ -63,6 +63,7 @@ context =
 Pygments = 2.2.0
 astor = 0.5
 backports-abc = 0.5
+backports.functools-lru-cache = 1.6.1
 backports.shutil-get-terminal-size = 1.0.0
 cycler = 0.10.0
 ipykernel = 4.5.2
@@ -73,7 +74,7 @@ jupyter-client = 5.0.0
 jupyter-core = 4.3.0
 jupyterlab = 0.26.3
 jupyterlab-launcher = 0.3.1
-matplotlib = 2.0.1
+matplotlib = 2.1.2
 mistune = 0.7.3
 nbformat = 4.3.0
 notebook = 4.4.1
@@ -121,7 +122,7 @@ pexpect = 4.2.1
 pickleshare = 0.7.4
 
 # Required by:
-# matplotlib==2.0.0
+# matplotlib==2.1.2
 # pandas==0.19.2
 python-dateutil = 2.6.0
 
@@ -138,7 +139,7 @@ scipy = 0.19.0
 singledispatch = 3.4.0.3
 
 # Required by:
-# matplotlib==2.0.0
+# matplotlib==2.1.2
 subprocess32 = 3.2.7
 
 # Required by:
@@ -152,7 +153,6 @@ jupyter-console = 5.1.0
 # jupyter==1.0.0
 qtconsole = 4.3.0
 
-Cython = 0.28.2
 et-xmlfile = 1.0.1
 h5py = 2.7.1
 mpmath = 1.0.0

software/kvm/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = cad7d490b84a484b089e34b27a5675ff
+md5sum = 763f66b915839a9b073fb93f43a44493
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2

software/kvm/instance.cfg.in

@@ -87,7 +87,7 @@ extra-context =
   raw novnc_location ${noVNC:location}
   raw netcat_bin ${netcat:location}/bin/netcat
   raw python_executable ${buildout:executable}
-  raw python_eggs_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
+  raw python_eggs_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
   raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
   raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
   raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}

software/kvm/software.cfg

@@ -8,6 +8,7 @@ extends =
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/netcat/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../stack/slapos.cfg
@@ -40,9 +41,12 @@ parts = ${:common-parts}
 
 #XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu
 
-[extra-eggs]
-eggs +=
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+eggs =
   ${python-cffi:egg}
+  ${lxml-python:egg}
   websockify
   slapos.cookbook
   erp5.util

software/neoppod/stress-testing/software.cfg

@@ -49,6 +49,8 @@ context =
 eggs +=
   zc.buildout
   slapos.libnetworkcache
+[slapos-cookbook]
+scripts =
 
 # Just in case one wants to inspect logs on the testnode itself.
 [neolog]

software/nginx-push-stream/software.cfg

@@ -6,16 +6,13 @@ extends =
 
 parts =
   slapos-cookbook
+  slapos-toolbox
   nginx-push-stream-module
   nginx-push-stream
   template
   template-nginx-service
   template-nginx
 
-[slapos-cookbook]
-eggs +=
-  slapos.toolbox
-
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in

software/slapos-master/test/test/__init__.py

@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
     """Return the output paramters from the root partition"""
     return json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
+
+  def getComputerPartition(self, partition_reference):
+    for computer_partition in self.slap.computer.getComputerPartitionList():
+      if partition_reference == computer_partition.getInstanceParameter(
+          'instance_title'):
+        return computer_partition
+
+  def getComputerPartitionPath(self, partition_reference):
+    partition_id = self.getComputerPartition(partition_reference).getId()
+    return os.path.join(self.slap._instance_root, partition_id)

software/slapos-sr-testing/software.cfg

@@ -138,6 +138,11 @@ setup = ${slapos-repository:location}/software/gitlab/test/
 egg = slapos.test.cloudooo
 setup = ${slapos-repository:location}/software/cloudooo/test/
 
+[slapos.test.dream-setup]
+<= setup-develop-egg
+egg = slapos.test.dream
+setup = ${slapos-repository:location}/software/dream/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -180,6 +185,7 @@ eggs =
   ${slapos.test.nextcloud-setup:egg}
   ${slapos.test.turnserver-setup:egg}
   ${slapos.test.cloudooo-setup:egg}
+  ${slapos.test.dream-setup:egg}
   ${backports.lzma:egg}
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
@@ -239,6 +245,7 @@ extra =
   ${slapos.test.grafana-setup:setup}
   ${slapos.test.gitlab-setup:setup}
   ${slapos.test.cloudooo-setup:setup}
+  ${slapos.test.dream-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = ec70348dd71b319590a5c5837f3d2e45
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = dc3a7734b08977f438b35d441a12f628
+md5sum = f342daca2e87f8fa44fd7b6a2a1c22f8
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -811,7 +811,7 @@ mode = 0744
 context =
     section supervisord supervisord
     section slaprunner slaprunner
-    raw python_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
+    raw python_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
 
 [supervisord-wrapper]
 recipe = slapos.cookbook:wrapper

software/slaprunner/software-py3.cfg

@@ -9,7 +9,7 @@ python = python3
 common-parts -=
   rdiff-backup
 
-[extra-eggs]
+[python-with-eggs]
 eggs -=
 # futures is a backport of Py3's concurrent.futures module
   futures

software/slaprunner/software.cfg

@@ -37,6 +37,7 @@ shared-part-list =
 # to avoid versioning issues
 
 common-parts =
+  slapos-command
   slapos-cookbook
   template
   instance-runner-import
@@ -50,6 +51,7 @@ common-parts =
 parts =
   ${:common-parts}
 
+
 [template-base]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}
@@ -141,8 +143,11 @@ filename = slapos-slapuser-script.in
 < = template-base
 output = ${buildout:directory}/buildout-shared-part-list
 
-[extra-eggs]
-eggs +=
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+eggs =
+  ${slapos-toolbox:eggs}
   erp5.util
   lock-file
   slapos.recipe.build
@@ -151,6 +156,14 @@ eggs +=
 # for gunicorn[gthread]
   futures
   supervisor
+# BBB: eggs used as recipe should be kept otherwise sections depending
+# on it can't be uninstalled
+  collective.recipe.shelloutput
+scripts =
+  gunicorn
+  supervisord
+  supervisorctl
+
 
 [versions]
 Flask-Auth = 0.85

software/theia/README.md

@@ -5,15 +5,6 @@ Theia is a cloud (and desktop) IDE https://www.theia-ide.org
 This version comes pre-configured with a few plugins, but does not come with python plugin, to let
 you choose between theia and vscode one.
 
-## Installing vscode extensions
-
-To install vscode's python plugin ( https://code.visualstudio.com/docs/languages/python ):
-
-<key>F1</key> `Deploy plugin by id` and enter `vscode:extension/ms-python.python` and wait for a while.
-Check theia's log files in case of issues.
-
-see https://github.com/theia-ide/theia/wiki/Testing-VS-Code-extensions for more details.
-
 ## jedi
 
 [jedi](https://github.com/davidhalter/jedi) which is used by both thiea and vscode python plugins has

software/theia/buildout.hash.cfg

@@ -15,12 +15,12 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 7c9444fbe8dc8faea67ede2b77e188ed
+md5sum = 21735765808aac82fb91d53341a3c0d6
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = b63a993a13e7c3b16b66c25fac5ac8b9
+md5sum = c2523a5c832f617c374ee621d50d9e52
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = 6c940b7015f45de6e679fc44807e5220
+md5sum = 6db2a484cac19787fecd87fffefa4aa9

software/theia/instance.cfg.in

@@ -17,13 +17,31 @@ recipe = slapos.cookbook:generate.password
 username = node
 bytes = 12
 
+[frontend-instance-certificate]
+recipe = plone.recipe.command
+command =
+  if [ ! -e $${:key-file} ]
+  then
+    ${openssl-output:openssl} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
+      -newkey rsa:1024 -keyout $${:key-file} \
+      -out $${:cert-file}
+  fi
+update-command = $${:command}
+key-file = $${directory:etc}/$${:_buildout_section_name_}.key
+cert-file = $${directory:etc}/$${:_buildout_section_name_}.crt
+common-name = $${frontend-instance-config:ip}
+location =
+  $${:key-file}
+  $${:cert-file}
+
 [frontend-instance-config]
 recipe = slapos.recipe.template:jinja2
 rendered = $${directory:etc}/$${:_buildout_section_name_}
 template = inline:
-  https://$${:hostname}:$${:port} {
+  :$${:port} {
     bind $${:ip}
-    tls self_signed # TODO
+    tls $${frontend-instance-certificate:cert-file} $${frontend-instance-certificate:key-file}
     log stdout
     errors stderr
     gzip
@@ -57,7 +75,7 @@ ip = $${frontend-instance-config:ip}
 hostname = $${frontend-instance-config:hostname}
 port = $${frontend-instance-config:port}
 pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid
-url = https://$${frontend-instance-password:username}:$${frontend-instance-password:passwd}@$${:hostname}:$${:port}/
+url = https://$${:hostname}:$${:port}/
 
 [frontend-reload]
 recipe = slapos.cookbook:wrapper
@@ -108,10 +126,24 @@ name = $${:_buildout_section_name_}.py
 config-hostname = $${frontend-instance:ip}
 config-port = $${frontend-instance:port}
 
+[apache-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Theia Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = $${frontend-instance:url}
+config-https-only = true
+config-type = websocket
+config-websocket-path-list = /services
+return = domain secure_access
+
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish
-url = $${frontend-instance:url}
-
+url = $${apache-frontend:connection-secure_access}
+username = $${frontend-instance-password:username}
+password = $${frontend-instance-password:passwd}
 
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration

software/theia/python-language-server-requirements.txt

 appdirs==1.4.3
+astroid==2.3.3
 attrs==18.2.0
 black==18.9b0
 Click==7.0
 future==0.17.1
-isort==4.3.4
+isort==4.3.21
 jedi==0.13.2
+lazy-object-proxy==1.4.3
 mccabe==0.6.1
-mypy==0.660
-mypy-extensions==0.4.1
+mypy==0.770
+mypy-extensions==0.4.3
 parso==0.3.2
 pluggy==0.8.1
 pydocstyle==3.0.0
 pyflakes==2.1.0
+pygls==0.8.1
+pylint==2.4.4
 python-jsonrpc-server==0.1.2
 -e git+https://github.com/palantir/python-language-server@50d03d5931d564e9908292ccfa21dd629ee817ba#egg=python_language_server
 rope==0.11.0
 six==1.12.0
 snowballstemmer==1.2.1
 toml==0.10.0
-typed-ast==1.2.0
-yapf==0.28.0
+typed-ast==1.4.1
+typing-extensions==3.7.4.2
+wrapt==1.11.2
+yapf==0.29.0
+zc.buildout.languageserver==0.2.0

software/theia/test/test.py

@@ -44,19 +44,20 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
 
 class TestTheia(SlapOSInstanceTestCase):
   def setUp(self):
-    self.theia_url = self.computer_partition.getConnectionParameterDict(
-    )['url']
+    self.connection_parameters = self.computer_partition.getConnectionParameterDict()
 
   def test_http_get(self):
-    resp = requests.get(self.theia_url, verify=False)
-    self.assertEqual(requests.codes.ok, resp.status_code)
+    resp = requests.get(self.connection_parameters['url'], verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)
 
-    # without login/password, this is unauthorized
-    parsed_url = urlparse(self.theia_url)
+    # with login/password, this is allowed
+    parsed_url = urlparse(self.connection_parameters['url'])
     resp = requests.get(
         parsed_url._replace(
-            netloc='[{}]:{}'.format(
+            netloc='{}:{}@[{}]:{}'.format(
+                self.connection_parameters['username'],
+                self.connection_parameters['password'],
                 parsed_url.hostname,
                 parsed_url.port)).geturl(),
         verify=False)
-    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+    self.assertEqual(requests.codes.ok, resp.status_code)

stack/erp5/buildout.cfg

@@ -732,7 +732,6 @@ zope.app.dependable = 3.5.1
 # five.formlib==1.0.4
 zope.app.form = 4.0.2
 
-Cython = 0.26.1
 et-xmlfile = 1.0.1
 more-itertools = 5.0.0
 h5py = 2.7.1
@@ -781,7 +780,3 @@ beautifulsoup4 = 4.8.2
 # WSGIProxy2==0.4.6
 WebOb = 1.8.5
 soupsieve = 1.9.5
-
-# Required by:
-# soupsieve==1.9.5
-backports.functools-lru-cache = 1.6.1

stack/monitor/buildout.cfg

@@ -7,13 +7,13 @@ extends =
   ../../component/dash/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/lxml-python/buildout.cfg
-  ../../component/pycurl/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
   ../../stack/logrotate/buildout.cfg
+  ../../stack/slapos.cfg
 
 parts =
   slapos-cookbook
-  extra-eggs
+  monitor-eggs
   monitor2-template
 
 [monitor-download-base]
@@ -30,25 +30,17 @@ destination = ${buildout:parts-directory}/monitor-scripts
 on-update = true
 
 [monitor-eggs]
-# XXX: merge with extra-eggs?
+recipe = zc.recipe.egg
+interpreter = monitor-pythonwitheggs
 eggs =
-  ${lxml-python:egg}
-  ${pycurl:egg}
-  ${python-cryptography:egg}
   plone.recipe.command
   collective.recipe.template
   cns.recipe.symlink
-  slapos.toolbox
-  slapos.core
-
-[extra-eggs]
-recipe = zc.recipe.egg
-interpreter = pythonwitheggs
-eggs =
-  ${monitor-eggs:eggs}
   psutil
   PyRSS2Gen
   Jinja2
+depends =
+  ${slapos-toolbox:recipe}
 
 
 # Monitor templates files
@@ -82,12 +74,12 @@ rendered = ${buildout:directory}/template-monitor.cfg
 context =
     key apache_location apache:location
     key template_logrotate_base template-logrotate-base:rendered
-    raw monitor_bin ${buildout:directory}/bin/monitor.bootstrap
-    raw monitor_collect ${buildout:directory}/bin/monitor.collect
-    raw monitor_statistic ${buildout:directory}/bin/monitor.statistic
-    raw monitor_runpromise ${buildout:directory}/bin/monitor.runpromise
-    raw monitor_genstatus ${buildout:directory}/bin/monitor.genstatus
-    raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite
+    raw monitor_bin ${buildout:bin-directory}/monitor.bootstrap
+    raw monitor_collect ${buildout:bin-directory}/monitor.collect
+    raw monitor_statistic ${buildout:bin-directory}/monitor.statistic
+    raw monitor_runpromise ${buildout:bin-directory}/monitor.runpromise
+    raw monitor_genstatus ${buildout:bin-directory}/monitor.genstatus
+    raw monitor_configwrite ${buildout:bin-directory}/monitor.configwrite
     raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
     raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
     raw curl_executable_location ${curl:location}/bin/curl
@@ -97,9 +89,9 @@ context =
     raw monitor_httpd_template ${monitor-httpd-conf:location}/${monitor-httpd-conf:filename}
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
-    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
+    raw python_with_eggs ${buildout:bin-directory}/${monitor-eggs:interpreter}
     raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
-    raw check_disk_space ${buildout:directory}/bin/check-free-disk
+    raw check_disk_space ${buildout:bin-directory}/check-free-disk
     raw bin_directory ${buildout:directory}/bin
 
 [versions]

stack/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = e1c7d1c21a942064e2f3f40463b10177
+md5sum = 84bc2cf29e34b48c51116d93e2be7636
 
 [monitor-httpd-conf]
 _update_hash_filename_ = templates/monitor-httpd.conf.in

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -165,8 +165,8 @@ port = ${monitor-instance-parameter:monitor-httpd-port}
 pid-file = ${directory:run}/monitor-httpd.pid
 access-log = ${directory:log}/monitor-httpd-access.log
 error-log = ${directory:log}/monitor-httpd-error.log
-cert-file = ${ca-directory:certs}/httpd.crt
-key-file = ${ca-directory:certs}/httpd.key
+cert-file = ${ca-directory:certs}/monitor-httpd.crt
+key-file = ${ca-directory:certs}/monitor-httpd.key
 htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path}
 url = https://[${monitor-instance-parameter:monitor-httpd-ipv6}]:${:port}
 httpd-cors-config-file = ${monitor-httpd-cors:rendered}
@@ -194,8 +194,8 @@ recipe = slapos.cookbook:wrapper
 command-line = {{ apache_location }}/bin/httpd -f ${monitor-httpd-conf:rendered} -DFOREGROUND
 wrapper-path = ${directory:bin}/monitor-httpd
 wait-for-files =
-  ${ca-directory:certs}/httpd.key
-  ${ca-directory:certs}/httpd.crt
+  ${monitor-httpd-conf-parameter:key-file}
+  ${monitor-httpd-conf-parameter:cert-file}
   ${monitor-httpd-graceful-wrapper:rendered}
 
 [monitor-httpd-graceful-wrapper]

stack/resilient/README.rst

@@ -33,8 +33,6 @@ The backup data is automatically used to build an historical, incremental archiv
 export
 ------
 
-example:
-    https://lab.nexedi.com/nexedi/slapos/blob/HEAD/stack/lapp/postgres/instance-postgres-export.cfg.in
 
 This is the *active* instance - the one providing live data to the application.
 
@@ -45,7 +43,6 @@ A backup is run via the bin/exporter script: it will
 The pull-backup, upon receiving the notification, will make a copy of the data and transmit it to the 'import' instances.
 
 You should provide the bin/{mysoftware}-exporter script, see for instance
-  https://lab.nexedi.com/nexedi/slapos/blob/HEAD/slapos/recipe/postgres/__init__.py#L207
   https://lab.nexedi.com/nexedi/slapos/blob/1.0.142/slapos/recipe/mydumper.py#L71
 
 By default, as defined in
@@ -57,9 +54,6 @@ the bin/exporter script is run every 60 minutes.
 import
 ------
 
-example:
-    https://lab.nexedi.com/nexedi/slapos/blob/HEAD/stack/lapp/postgres/instance-postgres-import.cfg.in
-
 This is the *fallback* instance - the one that can be activated and thus become active.
 Any number of import instances can be used. Deciding which one should take over can be done manually
 or through a monitoring + election script.
@@ -67,7 +61,6 @@ or through a monitoring + election script.
 
 You should provide the bin/{mysoftware}-importer script, see for instance
 
-  https://lab.nexedi.com/nexedi/slapos/blob/HEAD/slapos/recipe/postgres/__init__.py#L233
   https://lab.nexedi.com/nexedi/slapos/blob/1.0.142/slapos/recipe/mydumper.py#L71
 
 

stack/resilient/buildout.hash.cfg

@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a
 
 [resilient-web-takeover-cgi-script-download]
 filename = resilient-web-takeover-cgi-script.py.in
-md5sum = 675ac9e1cf49ccc8f8eddb541a62d899
+md5sum = a7d6ca463e4b6438e1d826f531533a39
 
 [template-wrapper]
 filename = templates/wrapper.in

stack/resilient/resilient-web-takeover-cgi-script.py.in

@@ -41,17 +41,17 @@ def getLatestBackupDate():
   equeue_database_copy = os.path.join(temporary_directory, 'equeue.db')
   shutil.copyfile(equeue_database, equeue_database_copy)
   db = gdbm.open(equeue_database_copy)
-  # Usually, there is only one callback (so only one key
-  # in the db), but if there are several:
-  # Take the "oldest" one (oldest value).
+  # Usually, there is only one callback (so only one key in the db), but if
+  # there are several we take the "newest" one.  Indeed, sometimes the importer
+  # script change name those introducing a new key inside the db.
   db_keys = db.keys()
   if not db_keys:
     result = False
   else:
-    last_backup = db[db_keys[-1]]
+    last_backup = db[db_keys[0]]
     for callback in db_keys:
       timestamp = float(db[callback])
-      if timestamp < last_backup:
+      if timestamp > last_backup:
         last_backup = timestamp
     result = datetime.datetime.fromtimestamp(last_backup)
   db.close()

stack/slapos.cfg

@@ -30,6 +30,8 @@ extends =
   ../component/python-cachecontrol/buildout.cfg
   ../component/python-cryptography/buildout.cfg
   ../component/python-PyYAML/buildout.cfg
+  ../component/pycurl/buildout.cfg
+
 
 # Separate from site eggs
 allowed-eggs-from-site-packages =
@@ -97,6 +99,22 @@ eggs =
   ${python-cryptography:egg}
   pyOpenSSL
   slapos.cookbook
+
+# slapos.toolbox containing utilities
+[slapos-toolbox]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${pycurl:egg}
+  ${python-cryptography:egg}
+  slapos.toolbox
+
+# Install a slapos command with networkcache enabled in ${buildout:bin-directory}
+[slapos-command]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.core
   slapos.libnetworkcache
 
 [versions]
@@ -144,7 +162,7 @@ slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.4
 slapos.recipe.build = 0.44
-slapos.recipe.cmmi = 0.12
+slapos.recipe.cmmi = 0.13
 slapos.toolbox = 0.109
 stevedore = 1.21.0
 subprocess32 = 3.5.3
@@ -219,7 +237,7 @@ pyrsistent = 0.14.5
 ipaddress = 1.0.18
 
 # Required by:
-# slapos.cookbook==1.0.121
+# slapos.cookbook==1.0.143
 jsonschema = 3.0.2
 
 # Required by: