Update Release Candidate

Conflicts:
	component/groonga/buildout.cfg
	component/mariadb/buildout.cfg
	software/caucase/software.cfg
	software/jupyter/instance.cfg.in
	software/jupyter/software.cfg
	stack/caucase/buildout.hash.cfg

component/apache/buildout.cfg

@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
 [apache]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
-version = 2.4.26
+version = 2.4.27
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = d4d47749a44461cb2e6c9d78a22b522b
+md5sum = 97b6bbfa83c866dbe20ef317e3afd108
 pre-configure =
   cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
   cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util
@@ -118,9 +118,9 @@ environment =
 [apache-2.2]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
-version = 2.2.32
+version = 2.2.34
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = d590ca1a8ce577b6bdfbd738d2de4741
+md5sum = c0bf1bdb779a3b5e6e294caf9ca20441
 patch-options = -p1
 configure-options = --disable-static
                     --enable-authn-alias

component/file/buildout.cfg

@@ -8,8 +8,8 @@ extends =
 
 [file]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.astron.com/pub/file/file-5.23.tar.gz
-md5sum = 61db35209ce71a6d576392ce6e1d2f80
+url = http://ftp.icm.edu.pl/packages/file/file-5.32.tar.gz
+md5sum = 4f2503752ff041895090ed6435610435
 configure-options =
   --disable-static
 environment =

component/gnutls/buildout.cfg

@@ -14,6 +14,7 @@ url = http://www.gnupg.org/ftp/gcrypt/${:_buildout_section_name_}/${:_buildout_s
 configure-options = ${:configure-options-extra}
   --enable-option-checking=fatal
   --disable-dependency-tracking
+  --disable-doc
 environment = ${:environment-extra}
   PATH=${gettext:location}/bin:%(PATH)s
 environment-extra =
@@ -23,25 +24,22 @@ environment-extra =
 version = 1.27
 md5sum = 5217ef3e76a7275a2a3b569a12ddc989
 configure-options-extra =
-  --disable-doc
   --disable-tests
 
 [libgcrypt]
 <= gpg-common
-version = 1.7.7
-md5sum = d1769481b1b506a632fd66c5e5f62e41
+version = 1.8.1
+md5sum = b21817f9d850064d2177285f1073ec55
 configure-options-extra =
   --with-gpg-error-prefix=${libgpg-error:location}
-  --disable-doc
 environment-extra =
   LDFLAGS=-Wl,-rpath=${libgpg-error:location}/lib
 
 [gnutls]
 <= gpg-common
-url = http://www.gnupg.org/ftp/gcrypt/gnutls/v3.5/gnutls-3.5.13.tar.xz
-md5sum = 4fd41ad86572933c2379b4cc321a0959
+url = http://www.gnupg.org/ftp/gcrypt/gnutls/v3.5/gnutls-3.5.15.tar.xz
+md5sum = bcdcbc65c50a7499617ad9f4d0058de9
 configure-options-extra =
-  --disable-doc
   --disable-static
   --disable-libdane
   --disable-guile

component/groonga/buildout.cfg

@@ -14,8 +14,8 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-7.0.4.tar.gz
-md5sum = 90c8ddf58fecdf7dba1a053adc9b9974
+url = http://packages.groonga.org/source/groonga/groonga-7.0.5.tar.gz
+md5sum = d42dd98c5272ed2b9ab50ded54eb818b
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/jupyter/buildout.cfg

+[buildout]
+extends =
+  ../numpy/openblas.cfg
+  ../matplotlib/buildout.cfg
+  ../ipython/buildout.cfg
+  ../python-pyzmq/buildout.cfg
+  ../python-2.7/buildout.cfg
+  ../scipy/buildout.cfg
+  ../scikit-learn/buildout.cfg
+  ../pandas/buildout.cfg
+  ../seaborn/buildout.cfg
+
+parts =
+  jupyter
+  jupyter-notebook-scripts
+
+[jupyter-env]
+<= numpy-env
+
+[jupyter]
+recipe = zc.recipe.egg:scripts
+eggs =
+# jupter base egg
+  jupyter
+# explicit eggs to generate scripts from them
+  jupyter_client
+  jupyter_console
+  jupyter_core
+  nbconvert
+  nbformat
+  notebook
+# additional eggs
+  requests
+  astor
+  ${matplotlib:egg}
+  ${scipy:egg}
+  ${pandas:egg}
+  ${scikit-learn:egg}
+  ${seaborn:egg}
+  ${statsmodels:egg}
+  patsy
+# additional eggs for jupyterlab
+  jupyterlab
+  jupyterlab-launcher
+
+interpreter = pythonjupyter
+
+environment = jupyter-env
+setup-eggs =
+  ${numpy:egg}
+  ${python-pyzmq:egg}
+  ${ipython:egg}
+  
+scripts = 
+  jupyter-kernelspec
+  pythonjupyter
+  jupyter
+  jupyter-trust
+  jupyter-nbconvert
+  jupyter-console
+  jupyter-migrate
+  jupyter-troubleshoot
+  jupyter-run
+  
+[jupyter-notebook-initialized-scripts]
+recipe = zc.recipe.egg:scripts
+eggs = ${jupyter:eggs}
+interpreter = pythonjupyter
+environment = jupyter-env
+
+scripts = 
+  jupyter-nbextension
+  jupyter-notebook
+  jupyter-serverextension
+  jupyter-lab
+  jupyter-labextension
+  jupyter-labhub
+
+initialization =
+  # https://github.com/ipython/ipython/issues/5420
+  import os
+  os.environ['PYTHONPATH'] = ':'.join(sys.path)

component/libtiff/buildout.cfg

@@ -10,15 +10,15 @@ parts =
 
 [libtiff]
 recipe = slapos.recipe.cmmi
-url = http://download.osgeo.org/libtiff/tiff-4.0.7.tar.gz
-md5sum = 77ae928d2c6b7fb46a21c3a29325157b
+url = http://download.osgeo.org/libtiff/tiff-4.0.8.tar.gz
+md5sum = 2a7d1c1318416ddf36d5f6fa4600069b
 configure-options =
   --disable-static
   --without-x
   --disable-lzma
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/tiff_4.0.7-6.debian.patch#b9249bb352e693b7975f9d343018994d
+  ${:_profile_base_location_}/tiff_4.0.8-3.debian.patch#9f52aed13d6ae864b3fb46372fe97780
 environment =
   CPPFLAGS=-I${libjpeg:location}/include -I${jbigkit:location}/include -I${zlib:location}/include
   LDFLAGS=-L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/mariadb/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff

+diff --git a/mrn_mysql_compat.h b/mrn_mysql_compat.h
+index d2b227ff..337de968 100644
+--- a/mrn_mysql_compat.h
++++ b/mrn_mysql_compat.h
+@@ -362,6 +362,7 @@
+ 
+ #if defined(MRN_MARIADB_P) &&                                           \
+   ((MYSQL_VERSION_ID >= 100207) ||                                      \
++   ((MYSQL_VERSION_ID >= 100126) && (MYSQL_VERSION_ID < 100200)) ||     \
+    ((MYSQL_VERSION_ID >= 50557) && (MYSQL_VERSION_ID < 100000)))
+ #  define mrn_create_partition_name(out,                                \
+                                     out_length,                         \

component/mariadb/buildout.cfg

@@ -17,14 +17,17 @@ extends =
   ../readline/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
+# The following lines are only for mariarocks.cfg
+# to be extended last without touching 'parts'.
+  ../gcc/buildout.cfg
 
 parts =
   mariadb
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-url = https://downloads.mariadb.org/f/mariadb-10.1.25/source/mariadb-10.1.25.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-md5sum = 6349e524bc449767c4650196d4141a02
+url = https://downloads.mariadb.org/f/mariadb-10.1.26/source/mariadb-10.1.26.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
+md5sum = bb88afb72434c6d567c742896dd50d41
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
@@ -55,15 +58,17 @@ configure-options =
   -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
   -DWITHOUT_MROONGA_STORAGE_ENGINE=1
   -DWITHOUT_DAEMON_EXAMPLE=1
-  -DCMAKE_C_FLAGS="-I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-  -DCMAKE_CXX_FLAGS="-I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-  -DCMAKE_INSTALL_RPATH=${:CMAKE_INSTALL_RPATH}
-CMAKE_INSTALL_RPATH=${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
+  -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
+  -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
+  -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
+CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include
+CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
+extra_library_path =
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
   CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include
-  CMAKE_LIBRARY_PATH=${bzip2:location}/lib:${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
-  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
+  CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
+  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
   PATH=${patch:location}/bin:%(PATH)s
 post-install =
   mkdir -p ${:location}/include/wsrep &&
@@ -73,12 +78,17 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-7.04.tar.gz
-md5sum = 1d889cdd0e4f3ac0a84d18d16ab920d6
+url = http://packages.groonga.org/source/mroonga/mroonga-7.05.tar.gz
+md5sum = d289667a60a6aa78d8bc276ab61b2996
 pre-configure = set -e
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source
-  ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
+  test -d ${mariadb:location}/include/mysql/server/private && (
+    mkdir -p fake_mariadb_source/sql
+    cp -a ${mariadb:location}/include/mysql/server/private/* fake_mariadb_source/sql
+    cp -a ${mariadb:location}/include/mysql/server/*.h fake_mariadb_source/sql
+    cp -a ${mariadb:location}/include/mysql/server/mysql fake_mariadb_source/sql
+  ) || ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
   cp -ar ${mariadb:location}/include fake_mariadb_source/include
   ! test -f fake_mariadb_source/include/mysql/mariadb_stmt.h ||
   sed -i -E -e '/^enum enum_(cursor|indicator)_type$/,/};/d' fake_mariadb_source/include/mysql/mariadb_stmt.h
@@ -89,7 +99,8 @@ configure-options =
   --disable-document
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/mroonga_boolean.patch#29f85fc98e85d39b31e53b1e1683d288
+  ${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0
+  ${:_profile_base_location_}/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff#a8b8df4dd870a1a269dfd9dc6941467b
 pre-build =
   sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
 environment =

component/mariadb/mariarocks.cfg

+# Do not extend any file that touch buildout:parts.
+
 [mariadb]
-url = https://downloads.mariadb.com/MariaDB/mariadb-10.2.6/source/mariadb-10.2.6.tar.gz
-md5sum = 3d454cdadbd3208e1c9c23e5338a62b0
+url = https://downloads.mariadb.org/f/mariadb-10.2.8/source/mariadb-10.2.8.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
+md5sum = f93cbd5bfde3c0d082994764ff7db580
 stable-patches =
 configure-options +=
   -DPLUGIN_DAEMON_EXAMPLE=NO
@@ -8,4 +10,4 @@ configure-options +=
   -DPLUGIN_MROONGA=NO
   -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
   -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
-  -DCMAKE_INSTALL_RPATH=${gcc:location}/lib:${gcc:location}/lib64:${:CMAKE_INSTALL_RPATH}
+extra_library_path = :${gcc:location}/lib:${gcc:location}/lib64

component/mariadb/mroonga_boolean.patch

-diff --git a/ha_mroonga.cpp b/ha_mroonga.cpp
-index 06691fb..b59e685 100644
---- a/ha_mroonga.cpp
-+++ b/ha_mroonga.cpp
-@@ -8283,7 +8283,7 @@ grn_rc ha_mroonga::generic_ft_init_ext_prepare_expression_in_boolean_mode(
+--- mroonga-7.05/lib/mrn_query_parser.cpp~	2017-07-27 07:47:36.000000000 +0200
++++ mroonga-7.05/lib/mrn_query_parser.cpp	2017-08-14 11:19:25.339015061 +0200
+@@ -52,7 +52,7 @@
  
-   const char *keyword = NULL;
-   uint keyword_length = 0;
+     const char *raw_query = NULL;
+     size_t raw_query_length = 0;
 -    grn_operator default_operator = GRN_OP_OR;
 +    grn_operator default_operator = GRN_OP_ADJUST;
      grn_expr_flags expression_flags = 0;
-   generic_ft_init_ext_parse_pragma(info,
-                                    key,
+     parse_pragma(query,
+                  query_length,

component/nginx/buildout.cfg

@@ -4,6 +4,7 @@ extends =
   ../git/buildout.cfg
   ../libexpat/buildout.cfg
   ../openssl/buildout.cfg
+  ../patch/buildout.cfg
   ../pcre/buildout.cfg
   ../zlib/buildout.cfg
 
@@ -13,6 +14,10 @@ parts = nginx-output
 recipe = slapos.recipe.cmmi
 url = http://nginx.org/download/nginx-1.10.1.tar.gz
 md5sum = 088292d9caf6059ef328aa7dda332e44
+patch-options = -p0
+patch-binary = ${patch:location}/bin/patch
+patches =
+  http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
 
 [nginx]
 <= nginx-common

component/pcre/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [pcre]
 recipe = slapos.recipe.cmmi
-url = http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.40.tar.bz2
-md5sum = 41a842bf7dcecd6634219336e2167d1d 
+url = https://ftp.pcre.org/pub/pcre/pcre-8.41.tar.bz2
+md5sum = c160d22723b1670447341b08c58981c1
 configure-options =
   --disable-static
   --enable-unicode-properties

component/percona-toolkit/buildout.cfg

@@ -8,9 +8,9 @@ parts =
 
 [percona-toolkit]
 recipe = slapos.recipe.cmmi
-version = 2.2.15
-url = http://www.percona.com/redir/downloads/percona-toolkit/${:version}/tarball/percona-toolkit-${:version}.tar.gz
-md5sum = 022f40dadaea9025820530ea1f986192
+version = 3.0.3
+url = https://www.percona.com/downloads/percona-toolkit/${:version}/source/tarball/percona-toolkit-${:version}.tar.gz
+md5sum = 8af181994fdf9aa984475637861098e9
 depends =
   ${perl-DBI:location}
   ${perl-DBD-mariadb:location}

component/postgresql/buildout.cfg

@@ -35,5 +35,5 @@ environment =
 
 [postgresql92]
 <= postgresql-common
-url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.18.tar.bz2
-md5sum = fd175eb5f29557c6ef2eeaf340330f9a
+url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.22.tar.bz2
+md5sum = c5d3fb5229baf9e94ee2287980c55321

component/python-2.7/buildout.cfg

@@ -13,7 +13,6 @@ extends =
   ../zlib/buildout.cfg
   ../file/buildout.cfg
   ../xz-utils/buildout.cfg
-  ../gcc/buildout.cfg
 
 parts =
     python2.7

component/python-mysqlclient/buildout.cfg

@@ -16,6 +16,7 @@ recipe = zc.recipe.egg:custom
 egg = mysqlclient
 patches =
   https://github.com/PyMySQL/mysqlclient-python/commit/1693848c9f6ca863868d94d63499830f7f4f3a1f.diff#a493a91f9263243eb331fcab9901b8b0
+  ${:_profile_base_location_}/mysqlclient-1.3.10-mariadb-10.2.8.patch#807b694fcd7ade4da4bcca321b1a29d2
 patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
 environment = python-mysqlclient-env

component/python-mysqlclient/mysqlclient-1.3.10-mariadb-10.2.8.patch

+diff -ur mysqlclient-1.3.10.orig/_mysql.c mysqlclient-1.3.10/_mysql.c
+--- mysqlclient-1.3.10.orig/_mysql.c	2017-01-04 13:47:08.000000000 +0100
++++ mysqlclient-1.3.10/_mysql.c	2017-08-21 10:53:21.014929937 +0200
+@@ -1060,7 +1060,7 @@
+ 	if (self && PyModule_Check((PyObject*)self))
+ 		self = NULL;
+ 	if (self && self->open) {
+-#if MYSQL_VERSION_ID >= 50707 && !defined(MARIADB_BASE_VERSION)
++#if MYSQL_VERSION_ID >= 50707 && (!defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID))
+ 		len = mysql_real_escape_string_quote(&(self->connection), out, in, size, '\'');
+ #else
+ 		len = mysql_real_escape_string(&(self->connection), out, in, size);
+@@ -1118,7 +1118,7 @@
+ 	out = PyBytes_AS_STRING(str);
+ 	check_server_init(NULL);
+ 	if (self && self->open) {
+-#if MYSQL_VERSION_ID >= 50707 && !defined(MARIADB_BASE_VERSION)
++#if MYSQL_VERSION_ID >= 50707 && (!defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID))
+ 		len = mysql_real_escape_string_quote(&(self->connection), out+1, in, size, '\'');
+ #else
+ 		len = mysql_real_escape_string(&(self->connection), out+1, in, size);
+diff -ur mysqlclient-1.3.10.orig/setup_posix.py mysqlclient-1.3.10/setup_posix.py
+--- mysqlclient-1.3.10.orig/setup_posix.py	2016-07-26 10:12:24.000000000 +0200
++++ mysqlclient-1.3.10/setup_posix.py	2017-08-21 10:54:59.643071374 +0200
+@@ -63,7 +63,7 @@
+         if extra_compile_args[i] == '-arch':
+             extra_link_args += ['-arch', extra_compile_args[i + 1]]
+ 
+-    include_dirs = [dequote(i[2:])
++    include_dirs = [dequote(os.path.isdir(i[2:]+'/server') and i[2:]+'/server' or i[2:])
+                     for i in mysql_config('include') if i.startswith('-I')]
+ 
+     if static:

slapos/recipe/check_port_listening/__init__.py → slapos/recipe/check_port_listening.py

@@ -26,24 +26,35 @@
 #
 ##############################################################################
 from slapos.recipe.librecipe import GenericBaseRecipe
+from zc.buildout.easy_install import _safe_arg, script_header
 import sys
 
+template = script_header + r"""
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+import socket
+import sys
+
+addr = "%(hostname)s", %(port)s
+
+try:
+  socket.create_connection(addr).close()
+except (socket.error, socket.timeout):
+  sys.stderr.write("%%s on %%s isn't listening\n" %% addr)
+  sys.exit(127)
+"""
+
 class Recipe(GenericBaseRecipe):
   """
   Check listening port promise
   """
 
   def install(self):
-    config = dict(
-      hostname=self.options['hostname'],
-      port=self.options['port'],
-      python_path=sys.executable,
-    )
-
-    vnc_promise = self.createExecutable(
-      self.options['path'],
-      self.substituteTemplate(
-        self.getTemplateFilename('socket_connection_attempt.py.in'),
-        config))
+    promise = self.createExecutable(self.options['path'], template % {
+      'python': _safe_arg(sys.executable),
+      'dash_S': '', # BBB buildout 1.x
+      'hostname': self.options['hostname'],
+      'port': self.options['port'],
+      })
 
-    return [vnc_promise]
+    return [promise]

slapos/recipe/check_port_listening/template/socket_connection_attempt.py.in

-#!%(python_path)s
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-import socket
-import sys
-
-hostname = "%(hostname)s"
-port = %(port)s
-
-try:
-  s = socket.create_connection((hostname, port))
-  s.close()
-except (socket.error, socket.timeout):
-  sys.stderr.write("%(port)s on %(hostname)s isn't listening\n")
-  sys.exit(127)
-

software/agent/software.cfg

@@ -51,29 +51,29 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
 slapos.recipe.template = 3.0
-slapos.toolbox = 0.69
+slapos.toolbox = 0.71
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 rpdb = 0.1.5

software/apache-frontend/software.cfg

@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 3.0
-slapos.toolbox = 0.69
+slapos.toolbox = 0.71
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3
@@ -19,41 +19,41 @@ pyasn1 = 0.2.3
 websockify = 0.8.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 dnspython = 1.15.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 erp5.util = 0.4.49
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 passlib = 1.6.5
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 pycurl = 7.43.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 rpdb = 0.1.5

software/caucase/instance-caucase-input-schema.json

 {
   "type": "object",
   "$schema": "http://json-schema.org/draft-04/schema",
+  "extends": "./schema-definitions.json#",
   "title": "Input Parameters",
   "properties": {
     "server-port": {
+      "allOf": [{
+        "$ref": "#/definitions/tcpv4port"
+      }, {
         "title": "http port to use",
         "description": "Caucase http port to use.",
-      "type": "integer",
-      "default": 8009,
-      "minimum": 1,
-      "maximum": 65535
+        "default": 8009
+      }]
     },
     "server-https-port": {
+      "allOf": [{
+        "$ref": "#/definitions/tcpv4port"
+      }, {
         "title": "https port to use",
         "description": "Caucase port to use for https connexion.",
-      "type": "integer",
-      "default": 8010,
-      "minimum": 1,
-      "maximum": 65535
+        "default": 8010
+      }]
     },
     "external-url": {
       "title": "External http url",
@@ -61,6 +64,13 @@
       "description": "The time in seconds before a generated certificate will be deleted on CA server. Set 0 to never delete. Default: 30*24*60*60 seconds (30 days)",
       "default": 5184000,
       "type": "integer"
+    },
+    "auto-sign-csr-amount": {
+      "title": "Number of CSR to sign automatically",
+      "description": "The number of CSR to sign automatically at startup. Has no effect if there is more than the specified value of csr submitted to caucase. This value should be as lowest as possible",
+      "default": 1,
+      "minimum": 1,
+      "type": "integer"
     }
   }
 }
\ No newline at end of file

software/caucase/software.cfg

@@ -5,6 +5,6 @@ extends =
 
 parts = 
   slapos-cookbook
-  cacause-extra-eggs
+  caucase-extra-eggs
   instance-caucase
 

software/dream/software.cfg

@@ -48,7 +48,7 @@ scipy = 0.13.3
 simpy = 3.0.5
 zope.dottedname = 4.1.0
 tablib = 0.10.0
-mysqlclient = 1.3.10+SlapOSPatched001
+mysqlclient = 1.3.10+SlapOSPatched002
 
 # indirect dependancies
 cp.recipe.cmd = 0.5

software/erp5/software-erp5-component.cfg

@@ -4,6 +4,7 @@
 [buildout]
 extends =
   software.cfg
+  ../../component/mariadb/mariarocks.cfg
 
 [slapos.cookbook-repository]
 branch = erp5-component

software/erp5testnode/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[template]
+filename = instance.cfg
+md5sum = 307663d73ef3ef94b02567ecd322252e
+
+[template-default]
+filename = instance-default.cfg
+md5sum = 76f63d443c5fdcea6fac68791c6bb65b

software/erp5testnode/instance-default.cfg

@@ -17,6 +17,7 @@ parts =
   monitor-base
   monitor-publish
   testnode-frontend
+  resiliency-exclude-file
   
 [monitor-publish]
 recipe = slapos.cookbook:publish
@@ -162,6 +163,12 @@ software = $${rootdirectory:srv}/software
 shellinabox = $${rootdirectory:srv}/shellinabox
 ca-dir = $${rootdirectory:srv}/ca
 
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: **
+output = $${directory:srv}/exporter.exclude
+
 [testnode-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional

software/erp5testnode/software.cfg

 [buildout]
 extends =
+  buildout.hash.cfg
   ../../stack/slapos.cfg
   ../../component/git/buildout.cfg
   ../../component/lxml-python/buildout.cfg
@@ -49,17 +50,43 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
 mode = 0644
-md5sum = 307663d73ef3ef94b02567ecd322252e
 
 [template-default]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-default.cfg
 output = ${buildout:directory}/template-default.cfg
 mode = 0644
-md5sum = 05519f3887a309d3ec069e0aa9f52ebc
 
 [versions]
 PyXML = 0.8.5
 erp5.util = 0.4.49
 slapos.recipe.template = 3.0
 ipython = 5.3.0
+apache-libcloud = 2.1.0
+gitdb2 = 2.0.2
+pyasn1 = 0.3.2
+smmap2 = 2.0.3
+
+# Required by:
+# slapos.toolbox==0.70
+GitPython = 2.1.5
+
+# Required by:
+# slapos.toolbox==0.70
+atomize = 0.2.0
+
+# Required by:
+# slapos.toolbox==0.70
+dnspython = 1.15.0
+
+# Required by:
+# slapos.toolbox==0.70
+feedparser = 5.2.1
+
+# Required by:
+# slapos.toolbox==0.70
+lockfile = 0.12.2
+
+# Required by:
+# slapos.toolbox==0.70
+passlib = 1.7.1

software/ipython_notebook/template/ERP5kernel.py.jinja → software/jupyter/ERP5kernel.py

-#!{{ python_executable }}
-
 from ipykernel.kernelbase import Kernel
 from ipykernel.kernelapp import IPKernelApp
 from IPython.core.display import HTML
 import requests
 import json
+import sys
 
-# erp5_url from buildout
-erp5_url = "{{ erp5_url }}"
-if not erp5_url:
-    erp5_url = None
-else:
-    erp5_url = "%s/erp5/Base_executeJupyter" % erp5_url
+erp5_url = None
+if len(sys.argv) > 1:
+    erp5_url = "%s/erp5/Base_executeJupyter" % (sys.argv[1],)
 
 class MagicInfo:
   """
@@ -61,7 +57,7 @@ class ERP5Kernel(Kernel):
   language = 'ERP5'
   language_version = '0.1'
   language_info = {'mimetype': 'text/plain', 'name':'python'}
-  banner = "ERP5 integration with ipython notebook"
+  banner = "ERP5 integration with jupyter notebook"
 
   def __init__(self, user=None, password=None, url=None, status_code=None,
               *args, **kwargs):

software/jupyter/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-jupyter-notebook]
+filename = instance.cfg.in
+md5sum = 89ef078efd20c5e03d1cf7228568b865
+
+[jupyter-notebook-config]
+filename = jupyter_notebook_config.py.jinja
+md5sum = 720e90a829c63371696bc3009917a743
+
+[jupyter-set-password]
+filename = jupyter_set_password.cgi.jinja
+md5sum = b8d31441780b524a7e52d1710dd78385
+
+[erp5-kernel]
+filename = ERP5kernel.py
+md5sum = 7d5309fe79afbcb455c0d8181b42e56c
+
+[kernel-json]
+filename = kernel.json.jinja
+md5sum = 33547be93a67530165e079dc3ecfdac3
+
+[custom-js]
+filename = custom.js
+md5sum = 0bf9e2eb1718b14307265fe05a167018

software/ipython_notebook/template/custom.js.jinja → software/jupyter/custom.js

@@ -81,8 +81,8 @@
  * @static
  */
 
-$([IPython.events]).on('notebook_loaded.Notebook', function(){
-  var kernelname = IPython.notebook.kernel_selector.current_selection;
+$([jupyter.events]).on('notebook_loaded.Notebook', function(){
+  var kernelname = jupyter.notebook.kernel_selector.current_selection;
   var display_text="<div class='output_subarea output_text output_result'>\
   <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
   1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\

software/ipython_notebook/instance.cfg.in → software/jupyter/instance.cfg.in

 [buildout]
 parts =
   instance
-  ipython_notebook
+  jupyter_notebook
   read-knowledge0
   publish-connection-parameter
+  jupyter-notebook-config
   erp5-kernel
   kernel-json
   custom-js
@@ -33,7 +34,7 @@ port = 8888
 host = ${slapconfiguration:ipv6-random}
 cert_file = ${generate-certificate:cert_file}
 key_file = ${generate-certificate:key_file}
-logfile = ${directory:log}/ipython_notebook.log
+logfile = ${directory:log}/jupyter_notebook.log
 notebook_dir = ${directory:notebook_dir}
 
 [dynamic-jinja2-template-base]
@@ -52,13 +53,13 @@ command =
       -out ${instance-parameter:cert_file}
   fi
 update-command = ${:command}
-cert_file = ${directory:etc}/ipython_notebook_cert.crt
-key_file = ${directory:etc}/ipython_notebook_cert.key
+cert_file = ${directory:etc}/jupyter_cert.crt
+key_file = ${directory:etc}/jupyter_cert.key
 
 [instance]
 recipe = slapos.cookbook:wrapper
 command-line =
-  {{ bin_directory }}/ipython notebook
+  {{ bin_directory }}/jupyter-lab
   --no-browser
   --ip=${instance-parameter:host}
   --port=${instance-parameter:port}
@@ -66,24 +67,19 @@ command-line =
   --certfile=${instance-parameter:cert_file}
   --keyfile=${instance-parameter:key_file}
   --notebook-dir=${instance-parameter:notebook_dir}
-  --logfile=${instance-parameter:logfile}
-  --config=${ipython-notebook-config:rendered}
-# Add 'log-level' alias to 'DEBUG' so as to log requests being done while running
-# of service ipython_notebook.
   --log-level="DEBUG"
-wrapper-path = ${directory:service}/ipython_notebook
+wrapper-path = ${directory:service}/jupyter-lab
 parameters-extra = true
-# Explicitly define IPython Directory to be used while starting ipython notebook
-# Also explicitly force IPython to use en_US.utf8 encoding
 environment =
-  IPYTHONDIR=${directory:ipython_dir}
-  JUPYTER_CONFIG_DIR=${directory:ipython_dir}
+  JUPYTER_PATH=${directory:jupyter_dir}
+  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
+  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
   LANG=C.UTF-8
 
-[ipython-notebook-config]
+[jupyter-notebook-config]
 <= dynamic-jinja2-template-base
-template = {{ ipython_notebook_config_location }}/{{ ipython_notebook_config_filename }}
-rendered = ${directory:etc}/ipython_notebook_config.py
+template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
+rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
 mode = 0744
 context =
   raw config_cfg ${buildout:directory}/knowledge0.cfg
@@ -98,12 +94,16 @@ service = ${:etc}/service
 promise = ${:etc}/promise/
 log = ${:var}/log
 notebook_dir = ${:var}/notebooks
-# Add folders to explicitly define ipython directory
-ipython_dir = ${:home}/ipython
-ipython_kernel_dir = ${:ipython_dir}/kernels
-erp5_kernel_dir = ${:ipython_kernel_dir}/ERP5
-
-[ipython_notebook]
+# Add folders to explicitly define jupyter directory
+jupyter_dir = ${:home}/jupyter
+jupyter_config_dir = ${:jupyter_dir}/etc
+jupyter_kernel_dir = ${:jupyter_dir}/kernels
+jupyter_runtime_dir = ${:jupyter_dir}/runtime
+jupyter_custom_dir = ${:jupyter_config_dir}/custom
+jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
+erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
+
+[jupyter_notebook]
 # This part is called like this because knowledge0.write uses the part name for
 # the section name in the config file.
 recipe = slapos.cookbook:zero-knowledge.write
@@ -126,16 +126,13 @@ instance-configuration =
 
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish.serialised
-url = https://[${instance-parameter:host}]:${instance-parameter:port}
+jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
+jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
 
 [erp5-kernel]
-<= dynamic-jinja2-template-base
-template = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
-rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py
-# Use ipython as executable python as we'll be needing requests library in kernel
-context =
-  raw python_executable {{ bin_directory }}/ipython
-  key erp5_url slapconfiguration:configuration.erp5-url
+recipe = slapos.cookbook:symbolic.link
+link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+target-directory = ${directory:erp5_kernel_dir}
 
 [kernel-json]
 <= dynamic-jinja2-template-base
@@ -144,13 +141,13 @@ rendered = ${directory:erp5_kernel_dir}/kernel.json
 # Use python2.7 executable bin file for kernel config
 context =
   raw python_executable {{ python_executable }}
-  key kernel_dir erp5-kernel:rendered
+  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
+  key erp5_url slapconfiguration:configuration.erp5-url 
   raw display_name ERP5
   raw language_name python
 
 [custom-js]
-<= dynamic-jinja2-template-base
-template = {{ custom_js_location }}/{{ custom_js_filename }}
-rendered = ${directory:ipython_dir}/profile_default/static/custom/custom.js
-mode = 0744
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:jupyter_custom_dir}
+link-binary = {{ custom_js_location }}/custom.js
 

software/ipython_notebook/template/ipython_notebook_config.py.jinja → software/jupyter/jupyter_notebook_config.py.jinja

@@ -25,14 +25,14 @@ c = get_config()
 parser = ConfigParser.ConfigParser()
 parser.read(knowledge_0)
 
-if not parser.has_section("ipython_notebook"):
-  parser.add_section("ipython_notebook")
+if not parser.has_section("jupyter_notebook"):
+  parser.add_section("jupyter_notebook")
 
-if not parser.has_option("ipython_notebook", "password") or \
-  parser.get("ipython_notebook", "password") == "":
-  parser.set("ipython_notebook", "password", random_password())
+if not parser.has_option("jupyter_notebook", "password") or \
+  parser.get("jupyter_notebook", "password") == "":
+  parser.set("jupyter_notebook", "password", random_password())
 
-c.NotebookApp.password = passwd(parser.get("ipython_notebook", "password"))
+c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
 
 with open(knowledge_0, 'w') as file:
   parser.write(file)

software/ipython_notebook/template/ipython_set_password.cgi.jinja → software/jupyter/jupyter_set_password.cgi.jinja

@@ -24,16 +24,16 @@ if not os.path.exists(config_file):
 
 parser = ConfigParser.ConfigParser()
 parser.read(config_file)
-if not parser.has_section("ipython_notebook"):
-  parser.add_section("ipython_notebook")
+if not parser.has_section("jupyter_notebook"):
+  parser.add_section("jupyter_notebook")
 
-if not parser.has_option("ipython_notebook", "password"):
-  parser.set("ipython_notebook", "password", "")
+if not parser.has_option("jupyter_notebook", "password"):
+  parser.set("jupyter_notebook", "password", "")
 
 if "password" in form:
-  parser.set("ipython_notebook", "password", passwd(form["password"].value))
+  parser.set("jupyter_notebook", "password", passwd(form["password"].value))
   # subprocess.call('{{ httpd_graceful }}')
-  # TODO: we should restart ipython
+  # TODO: we should restart jupyter
 
 with open(config_file, 'w') as file:
   parser.write(file)
@@ -43,7 +43,7 @@ print "<html><head>"
 print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
 print "<link rel=\"stylesheet\" href=\"static/style.css\">"
 print "</head><body>"
-print "<h1>IPython Notebook Password :</h1>"
+print "<h1>Jupyter Notebook Password :</h1>"
 print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
 print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
 

software/ipython_notebook/template/kernel.json.jinja → software/jupyter/kernel.json.jinja

@@ -2,6 +2,7 @@
   "argv": [
     "{{python_executable}}",
     "{{kernel_dir}}",
+    "{{erp5_url}}",
     "-f",
     "{connection_file}"
   ],

software/ipython_notebook/software.cfg → software/jupyter/software.cfg

@@ -3,59 +3,40 @@ versions = versions
 extends = 
   buildout.hash.cfg
   ../../stack/slapos.cfg
-  ../../component/ipython/buildout.cfg
-  ../../component/scipy/buildout.cfg
-  ../../component/scikit-learn/buildout.cfg
-  ../../component/pandas/buildout.cfg
   ../../component/openssl/buildout.cfg
-  ../../component/seaborn/buildout.cfg
+  ../../component/jupyter/buildout.cfg
   ../../stack/monitor/buildout.cfg
 parts =
   slapos-cookbook
-  ipython-notebook
-  instance-jupyter
-
-[ipython-notebook]
-; In the ipython notebook software, we use more eggs than in the minimal
-; ipython notebook component
-eggs +=
-  requests
-  astor
-  ${scipy:egg}
-  ${pandas:egg}
-  ${scikit-learn:egg}
-
-  ${seaborn:egg}
-  ${statsmodels:egg}
-  patsy
+  jupyter
+  jupyter-notebook-initialized-scripts
+  instance-jupyter-notebook
+
+[jupyter]
+python_executable = ${buildout:bin-directory}/${:interpreter}
 
 [download-file-base]
 recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/template/${:filename}
+url = ${:_profile_base_location_}/${:filename}
 download-only = true
 mode = 0644
 
-[ipython-notebook-config]
+[jupyter-notebook-config]
 <= download-file-base
-filename = ipython_notebook_config.py.jinja
 
-[ipython-notebook-set-password]
+[jupyter-set-password]
 <= download-file-base
-filename = ipython_set_password.cgi.jinja
 
 [erp5-kernel]
 <= download-file-base
-filename = ERP5kernel.py.jinja
 
 [kernel-json]
 <= download-file-base
-filename = kernel.json.jinja
 
 [custom-js]
 <= download-file-base
-filename = custom.js.jinja
 
-[instance-jupyter]
+[instance-jupyter-notebook]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:filename}
 rendered = ${buildout:directory}/template.cfg
@@ -65,11 +46,11 @@ context =
   key develop_eggs_directory buildout:develop-eggs-directory
   key eggs_directory buildout:eggs-directory
   key openssl_output openssl-output:openssl
-  key python_executable python2.7:executable
-  key ipython_notebook_config_location ipython-notebook-config:location
-  key ipython_notebook_config_filename ipython-notebook-config:filename
-  key ipython_notebook_set_password_location ipython-notebook-set-password:location
-  key ipython_notebook_set_password_filename ipython-notebook-set-password:filename
+  key python_executable jupyter:python_executable
+  key jupyter_config_location jupyter-notebook-config:location
+  key jupyter_config_filename jupyter-notebook-config:filename
+  key jupyter_set_password_location jupyter-set-password:location
+  key jupyter_set_password_filename jupyter-set-password:filename
   key erp5_kernel_location erp5-kernel:location
   key erp5_kernel_filename erp5-kernel:filename
   key kernel_json_location kernel-json:location
@@ -90,6 +71,8 @@ ipython-genutils = 0.1.0
 ipywidgets = 6.0.0
 jupyter-client = 5.0.0
 jupyter-core = 4.3.0
+jupyterlab = 0.26.3
+jupyterlab-launcher = 0.3.1
 matplotlib = 2.0.0
 mistune = 0.7.3
 nbformat = 4.3.0
@@ -162,3 +145,10 @@ subprocess32 = 3.2.7
 # Required by:
 # prompt-toolkit==1.0.13
 wcwidth = 0.1.7
+
+jupyter = 1.0.0
+jupyter-console = 5.1.0
+
+# Required by:
+# jupyter==1.0.0
+qtconsole = 4.3.0

software/kvm/software.cfg

@@ -5,7 +5,7 @@ extends = common.cfg
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.69
+slapos.toolbox = 0.71
 erp5.util = 0.4.49
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
@@ -16,23 +16,23 @@ smmap = 0.9.0
 # websockify = 0.8.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 dnspython = 1.14.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 lockfile = 0.12.2
 
 # Required by:
@@ -40,9 +40,9 @@ lockfile = 0.12.2
 numpy = 1.11.2rc1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 paramiko = 2.0.2
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 passlib = 1.6.5
\ No newline at end of file

software/nayuos/README.md

 # NayuOS
 
-This is a SlapOS recipe to build NayuOS. It needs to be put in the <code>/srv/slapgrid/\<part\>/srv/runner/project/slapos/software/</code> directory. The created directory is called <code>\<nayuos_build_dirname\></code> in this documentation.
+This is a SlapOS recipe to build NayuOS.
 
 ## License
 
@@ -8,42 +8,63 @@ GPL v2 or later
 
 ## Requirements
 
-*   sudo on the host (for now)
-*   some environment variables need to be authorized to be propagated when cros_sdk calls sudo:
+*   sudo on the host
+*   environment variables need to be authorized to be propagated when cros_sdk calls sudo: `: Defaults        env_keep += "CROS_CACHEDIR DEPOT_TOOLS"`
+*   slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
+in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen): `: slapuser9    ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill`
 
-: Defaults        env_keep += "CROS_CACHEDIR DEPOT_TOOLS"
+It's useful to have the right to kill cros_sdk processes, when needed. ;)
 
-*   slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
-in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen):
+## Technical notes
 
-: slapuser9    ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill
+After any change to the build process it is necessary to delete (using sudo)
+the building environment at `~/srv/runner/instance/slappart0/parts/chromiumos/<TAG>`.
+
+BEWARE that the web runner is serving images for the [official website](https://nayuos.nexedi.com).
+
+NayuOS and ChromiumOS is "just" a version of Gentoo. Thus it uses `ebuild` packages
+and anything installable in Gentoo can be installed to NayuOS too. Of course only
+during OS build phase and one has to count with limited space.
+
+### Upgrading (building new image)
+
+Please read **Requirements** section carefully. After selecting your desired 
+`release` from the list <https://chromium.googlesource.com/chromiumos/manifest/+refs>
+it is **necessary** to add `sudo` rules for that release as shown there.
 
-It's useful to have the right to kill cros_sdk processes, when needed. ;)
 
 ## Input
+
 In the vifib parameters (softinst\<nb\>.host.vifib.net \> Services \> Parameters):
 
-*   board / ex: peppy, swanky, ... (choosing daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM))
-*   branch / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
-*   keep_cache / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
+*   **board** / ex: peppy, swanky, ... Complete list of devices and board names on [chromiumOS developer guide](http://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices)
+*   **branch** / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
+*   **keep_cache** / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
 
+//Choosing board daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM)
 
 ## Output
-The image will be produced in:
-<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/images/</code>
-and the logs are in:
-<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_sources_dl.log</code> and <code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_build.log</code>
 
-The script that download the sources and build is located in
-<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/etc/run</code>
+Software release produces a build script `<instance_partition>/etc/run/cros_full_build`.
+
+Build produces
+
+*  Image: `<instance_partition>/parts/chromiumos/images/`
+*  Compilation logs: `<instance_partition>/var/log/cros_sources_dl.log`
+*  Build logs: `<instance_partition>/var/log/cros_build.log`
+
 
 ## External documents
 
-*   [ <code>repo</code> command reference ](https://source.android.com/source/using-repo.html)
-*   [ NayuOS official website ](https://www.nayuos.com)
+*   [ `repo` command reference ](https://source.android.com/source/using-repo.html)
+*   [ NayuOS official website ](https://nayuos.nexedi.com)
+*   [ Board names list ](http://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices)
 *   [ crouton for chroot ](https://github.com/dnschneid/crouton) ([warning about verified boot](https://github.com/dnschneid/crouton/blob/2a1fc9da380650f47e2bcf37d00962bfb68c4830/installer/main.sh#L517-L536))
 
 ## Notes for possible improvements
+
 *   [ Running virtual machines on your chromebook ](https://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices/running-virtual-machines-on-your-chromebook)
 *   to have a more common User Agent (the one of ChromiumOS/NayuOS is quite rare and identifies the user, see [studies of the EFF](https://panopticlick.eff.org/static/browser-uniqueness.pdf)), it seems possible to change the User-Agent flag for guest mode in the getOffTheRecord function, and adding a line (key "kUserAgent" , value "some common user agent" string). Then rebuild Chromium and [add it to NayuOS](https://www.chromium.org/chromium-os/developer-guide#TOC-Making-changes-to-the-Chromium-web-).
 *   remove need of root priviledge for entering the chroot, maybe by using fakeroot in 'scripts/wrapper_sudo.in'?
+*   change more options on Chromium OS "Privacy" part by default: there are [a few options](https://support.google.com/chromebook/answer/114836) which still use Google services
+*   provide ChromiumOS package manager [ chromebrew ](https://skycocker.github.io/chromebrew/) by default

software/nayuos/instance.cfg

+# Jinja2 template of a buildout file to create runnable scripts
+#
+# Received variables
+# -  software_dir: location where this repository was cloned to
+# -  instance_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
+
 [buildout]
 parts =
   parameters
+  promise-sudo-on-host
   template-full-build-script
   template-sudo-wrapper
   template-cros-sdk-wrapper
 
-# eggs given by software.cfg
+# standard declaration of eggs directories
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 
@@ -16,15 +23,18 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 recipe = slapos.recipe.build:gitclone
 repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
 branch = master
+git-executable = {{ git_path }}/bin/git
 
 [customize-path]
 # add depot tools directory (for cros_sdk binary among others) and git directory to the path
 command =
-  export PATH="${directory:wrapper_dir}":"{{ git_path }}":"{{ curl_path }}":"${depot-tools:location}":"$PATH";
+  export PATH="${directory:wrapper_dir}":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"{{thin_provisioning_tools}}/sbin":"$PATH";
 
 [nayuos-ebuilds]
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/nayuos-ebuilds.git
+branch = master
+git-executable = {{ git_path }}/bin/git
 
 ############################################################################################
 
@@ -44,20 +54,25 @@ cert = ${slap_connection:cert_file}
 recipe = slapos.cookbook:mkdirectory
 log = ${buildout:directory}/var/log
 run = ${buildout:directory}/etc/run
+promise = ${buildout:directory}/etc/promise
 wrapper_dir = ${buildout:directory}/wrapper_bin
 cros_location = ${buildout:directory}/parts/chromiumos
-ebuilds_dir = ${nayuos-ebuilds:location}
-scripts_dir = {{ scripts_dir }}
-logo_dir = {{ logo_dir }}
 
 [bin]
+# dummy section to hold references to frequently used binaries
 wrapper_cros_sdk=${directory:wrapper_dir}/wrapper_cros_sdk
 bash=/bin/bash
 sudo=/usr/bin/sudo
 
+[promise-sudo-on-host]
+# assert sudo is installed, as it is required to enter the chroot 'cros_sdk'
+recipe  = slapos.cookbook:wrapper
+wrapper-path = ${directory:promise}/${:_buildout_section_name_}
+command-line = sudo -V
+
 [template-sudo-wrapper]
 recipe = slapos.recipe.template:jinja2
-template = {{ scripts_dir }}/wrapper_sudo.in
+template = {{ software_dir }}/scripts/wrapper_sudo.in
 rendered = ${directory:wrapper_dir}/sudo
 md5sum = ded5a92be4e37ec32eb9d3087d3e19bd
 mode = 0700
@@ -68,23 +83,23 @@ context =
 
 [template-cros-sdk-wrapper]
 recipe = slapos.recipe.template:jinja2
-template = {{ scripts_dir }}/wrapper_cros_sdk.in
+template = {{ software_dir }}/scripts/wrapper_cros_sdk.in
 rendered = ${bin:wrapper_cros_sdk}
 md5sum = 7159fe3d5b85a283733cf686c4ee0a74
 mode = 0700
 context =
 # XXX bash path is the one from the host
   key bash_path bin:bash
-  raw git_path {{ git_path }}
-  raw curl_path {{ curl_path }}
+  raw git_path  {{ git_path }}/bin
+  raw curl_path {{ curl_path }}/bin
 
 [template-full-build-script]
 # create the rendered script in the buildout-directory/etc/run
 # (use jinja for templating)
 recipe = slapos.recipe.template:jinja2
-template = {{ scripts_dir }}/cros_full_build.in
+template = {{ software_dir }}/scripts/cros_full_build.in
 rendered = ${directory:run}/cros_full_build
-md5sum = 75599e6b8418a5f3756c7c7b26600399
+md5sum = b2ef515a871b18216baf187485be74f8
 mode = 0700
 context =
   key bash_path                         bin:bash
@@ -94,7 +109,11 @@ context =
   key branch                            parameters:configuration.branch
   key boards_list                       parameters:configuration.boards
   key keep_cache                        parameters:configuration.keep_cache
-  key ebuilds_dir directory:ebuilds_dir
-  key scripts_dir directory:scripts_dir
-  key logo_dir directory:logo_dir
-  raw nayu_dev_packages net-libs/nodejs net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
+  key ebuilds_dir                       nayuos-ebuilds:location
+  raw scripts_dir                       {{ software_dir }}/scripts
+  raw logo_dir                          {{ software_dir }}/logo
+# packages to be copied from our overlay into ChromiuOS' overlay
+# those packages will be marked as dependency of root filesystem
+  raw nayu_dev_rootfs_packages          app-misc/nayuos-chromium-policy
+# those packages will be marked as dependency of the chromium build itself
+  raw nayu_dev_packages                 net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup

software/nayuos/scripts/test_nayuos_image

@@ -14,8 +14,10 @@ ORIGINAL_GRANDENET_SCRIPT=~/trunk/src/third_party/chromiumos-overlay/net-misc/re
 GRANDENET_SCRIPT="usr/local/bin/grandenet"
 INIT_SSH_SERVER="etc/init/openssh-server.conf"
 BASHRC="etc/skel/.bashrc"
-EXPECTED_ALIAS="alias git='git --exec-path=/usr/local/libexec/git-core/'"
+GIT_CORE="usr/local/libexec/git-core/"
+GIT_EXPECTED_EXPORT="export GIT_EXEC_PATH=/usr/local/libexec/git-core"
 VIRTUALENV_BIN="usr/local/bin/virtualenv"
+CHROMIUM_POLICY="etc/chromium/policies/recommended/nayuos_policy.json"
 
 HAS_FAILED=0
 
@@ -32,14 +34,17 @@ function print_result() {
 
 # MOUNT IMAGE AND GET INFO
 
-install -d ${MOUNTPOINT}
+install -d "${MOUNTPOINT}"
 ./mount_gpt_image.sh --safe -f $( ./get_latest_image.sh --board=${BOARD} ) -r ${MOUNTPOINT}
 echo $(ls "${MOUNTPOINT}/usr/local")
 if [[ $(ls "${MOUNTPOINT}/usr/local") ]] ; then
-  my_diff=$(diff ${ORIGINAL_GRANDENET_SCRIPT} "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
+  my_diff=$(diff "${ORIGINAL_GRANDENET_SCRIPT}" "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
   opensshd_config=$(ls "${MOUNTPOINT}/${INIT_SSH_SERVER}")
-  gitalias=$(grep "${EXPECTED_ALIAS}" "${MOUNTPOINT}/${BASHRC}")
+  gitcore=$(ls "${MOUNTPOINT}/${GIT_CORE}")
+  gitexport=$(grep "${GIT_EXPECTED_EXPORT}" "${MOUNTPOINT}/${BASHRC}")
   virtualenvbin=$(ls "${MOUNTPOINT}/${VIRTUALENV_BIN}")
+  chromium_policy=$(ls "${MOUNTPOINT}/${CHROMIUM_POLICY}")
+
 else
   is_empty=1
 fi
@@ -50,7 +55,7 @@ rmdir ${MOUNTPOINT}
 # PRINT RESULTS
 
 echo "* test if /usr/local exists"
-if [[ ${is_empty} == 1 ]] ; then
+if [[ "${is_empty}" == "1" ]] ; then
   print_result ${FAILURE} "/usr/local is empty."
 else
   print_result ${SUCCESS} "/usr/local is not empty."
@@ -71,10 +76,18 @@ else
     print_result ${SUCCESS} "opensshd config removed."
   fi
 
-  echo "* test git quick fix for option --exec-path"
+  echo "* test if git core directory is not empty"
+
+  if [[ "${gitcore}" == "" ]] ; then
+    print_result ${FAILURE} "No git file in ${MOUNTPOINT}/${GIT_CORE}."
+  else
+    print_result ${SUCCESS} "git core directory exists and is not empty."
+  fi
+
+  echo "* test bashrc changes for git paths quick fix"
 
-  if [[ ${gitalias} == ""  ]] ; then
-    print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. Should be: ${EXPECTED_ALIAS}"
+  if [[ "${gitexport}" == ""  ]] ; then
+    print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. There should be: ${GIT_EXPECTED_EXPORT}"
   else
     print_result ${SUCCESS} "git alias is correct."
   fi
@@ -86,6 +99,14 @@ else
   else
     print_result ${FAILURE} "Expected virtualenv binary not in ${MOUNTPOINT}/${VIRTUALENV_BIN}."
   fi
+
+  echo "* test if Chromium policies are installed"
+
+  if [[ ${chromium_policy} ]] ; then
+    print_result ${SUCCESS} "Chromium policies are installed."
+  else
+    print_result ${FAILURE} "Expected Chromium policies file not in ${MOUNTPOINT}/${CHROMIUM_POLICY}."
+  fi
 fi
 
 exit ${HAS_FAILED}

software/nayuos/software.cfg

@@ -2,48 +2,55 @@
 extends =
 # basic needs
   ../../stack/slapos.cfg
+  ../../component/automake/buildout.cfg
+  ../../component/libexpat/buildout.cfg
+  ../../component/libaio/buildout.cfg
+  ../../component/boost-lib/buildout.cfg
 
 parts +=
-# use stack/slapos.cfg
   slapos-cookbook
-  verify-packages-exist-on-host
   template-instance
-  install-eggs-for-the-instance
+  git
+  curl
+  thin-provisioning-tools
+
+versions = versions
 
 [versions]
 slapos.recipe.template = 3.0
 
-[verify-packages-exist-on-host]
-recipe = plone.recipe.command
-stop-on-error = true
-# run the same command when installing and on updates
-update-command = ${:command}
-command = sudo -V
 
 [template-instance]
-# create the instance.cfg file in the buildout directory
-# (use jinja for templating)
+# jinja2 render instance.cfg file in the buildout directory
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance.cfg
 rendered = ${buildout:directory}/instance.cfg
-md5sum = be6bd665c0b356a84090d200ea14b33c
 mode = 0644
-scripts_dir = ${:_profile_base_location_}/scripts
-logo_dir = ${:_profile_base_location_}/logo
-curl_path = ${curl:location}/bin
-git_path = ${git:location}/bin
+md5sum = 22be269701616bb6f5453eb769e0cdaa
 context =
-# for access to the eggs from the instance
+  key software_dir            :_profile_base_location_
+  key instance_dir            buildout:directory
+  key curl_path               curl:location
+  key git_path                git:location
   key eggs_directory          buildout:eggs-directory
   key develop_eggs_directory  buildout:develop-eggs-directory
-  key scripts_dir :scripts_dir
-  key logo_dir :logo_dir
-  key curl_path :curl_path
-  key git_path :git_path
+  key thin_provisioning_tools thin-provisioning-tools:location
 
-[install-eggs-for-the-instance]
-# after installation of the eggs, the recipes
-# will be available (added to sys.path)
-recipe = zc.recipe.egg
-eggs = slapos.recipe.build
-       plone.recipe.command
+[thin-provisioning-tools]
+recipe = slapos.recipe.cmmi
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+url = https://github.com/jthornber/thin-provisioning-tools/archive/v0.7.1.tar.gz
+md5sum = 44cc03dd39a55b2f0364b330850d63fd
+pre-configure = ${autoconf:location}/bin/autoconf
+make-options = PREFIX=${:location}
+# environment has to define CXXFLAGS and not CPPFLAGS because of the ./configure
+# which has kind-of special behaviour
+environment =
+  PATH=${m4:location}/bin:${automake:location}/bin:${autoconf:location}/bin:%(PATH)s
+  CXXFLAGS=-I${libexpat:location}/include -I${libaio:location}/include -I${boost-lib:location}/include
+  LDFLAGS=-L${libexpat:location}/lib -L${libaio:location}/lib -L${boost-lib:location}/lib
+dependencies =
+  automake
+  libexpat
+  libaio
+  boost-lib

software/neoppod/software-common.cfg

@@ -113,44 +113,44 @@ apache-libcloud = 1.5.0
 ecdsa = 0.13
 gitdb2 = 2.0.0
 msgpack-python = 0.4.8
-mysqlclient = 1.3.10+SlapOSPatched001
+mysqlclient = 1.3.10+SlapOSPatched002
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.template = 3.0
-slapos.toolbox = 0.69
+slapos.toolbox = 0.71
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0
 zodbtools = 0.0.0.dev3
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 GitPython = 2.1.3
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 PyRSS2Gen = 1.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 dnspython = 1.14.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 feedparser = 5.2.1
 
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 paramiko = 2.1.2
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 passlib = 1.7.1

software/nginx-push-stream/software.cfg

@@ -46,4 +46,4 @@ mode = 0644
 
 [versions]
 slapos.recipe.template = 3.0
-slapos.toolbox = 0.69
+slapos.toolbox = 0.71

software/re6stnet/software.cfg

@@ -112,15 +112,15 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.template = 3.0
-slapos.toolbox = 0.69
+slapos.toolbox = 0.71
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 atomize = 0.2.0
 
 # Required by:
@@ -128,11 +128,11 @@ atomize = 0.2.0
 backports.ssl-match-hostname = 3.4.0.2
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 lockfile = 0.12.2
 
 # Required by:
@@ -140,10 +140,10 @@ lockfile = 0.12.2
 miniupnpc = 1.9
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 rpdb = 0.1.5
 

software/slapos-master/buildout.hash.cfg

@@ -15,11 +15,11 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 50834c59ded302034cafcce7315a55c3
+md5sum = aa10af616493b97d2b630c46d869a9d7
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 5562e6738470b0fc96211f46bf8d41bc
+md5sum = c5ce18fa4d4be9b9a2d789f3bbd37840
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -204,7 +204,7 @@ crl =
 {%     do part_list.append('logrotate-' ~ base_name) -%}
 {%     do apache_service_log_list.__setitem__(family_name, base_name) -%}
 [logrotate-{{ base_name }}]
- < = logrotate-entry-base
+< = logrotate-entry-base
 name = {{ base_name }}
 log = ${apache-conf-parameter-dict:log-dir}/{{ base_name }}-error.log ${apache-conf-parameter-dict:log-dir}/{{ base_name }}-access.log
 post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1

software/slapos-master/instance-erp5.cfg.in

@@ -9,6 +9,7 @@
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
 {% set monitor_base_url_dict = {} -%}
+{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
 [request-common]
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
@@ -36,12 +37,13 @@ config-{{ k }} = {{ dumps(v) }}
 {% for k, v in key_config.iteritems() -%}
 config-{{ k }} = {{ '${' ~ v ~ '}' }}
 {% endfor -%}
+config-name = {{ name }}
 {% endmacro -%}
 
 {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
-{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {% if has_posftix -%}
 {{   request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
 {%- else %}
@@ -199,6 +201,7 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
+config-name = {{ partition_name }}
 {% endfor -%}
 
 {# if not explicitly configured, connect jupyter to first zope family, which  -#}
@@ -310,7 +313,10 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-shared-certificate-authority-path = ${directory:ca-dir} 
 config-monitor-passwd = ${monitor-htpasswd:passwd}
-
+config-name = ${:name}
+config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
+config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
+ '--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix  --error-detail --js-embed --quiet')) }}
 
 [request-frontend-base]
 {% if has_frontend -%}
@@ -366,7 +372,7 @@ parts +=
   monitor-base
 
 [monitor-conf-parameters]
-monitor-title = ERP5 monitor
+monitor-title = ERP5
 password = ${monitor-htpasswd:passwd}
 
 [monitor-base-url-dict]

software/slaprunner/buildout.hash.cfg

@@ -31,7 +31,7 @@ md5sum = 5cfa49bcf20612844e1c50a85740d0b3
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
-md5sum = ef87691a9675beb646ae1c99ef69e551
+md5sum = 94c0eddb2af3290942f64e04f95a707c
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in

software/slaprunner/software.cfg

@@ -17,41 +17,41 @@ gunicorn = 19.4.5
 prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 3.0
-slapos.toolbox = 0.69
+slapos.toolbox = 0.71
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 PyRSS2Gen = 1.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 dnspython = 1.14.0
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 erp5.util = 0.4.49
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 paramiko = 2.0.2
 
 # Required by:
-# slapos.toolbox==0.69
+# slapos.toolbox==0.71
 passlib = 1.6.5

software/slaprunner/template/runner-export.sh.jinja2

@@ -20,7 +20,7 @@ tmp_directory='{{ directory["tmp"] }}'
 
 rsync () {
   set -x
-  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@"
+  '{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@"
   set +x
 }
 

software/wendelin/software.cfg

@@ -11,7 +11,8 @@ parts +=
   msgpack-python
   ipython
   wendelin.core
-  ipython-notebook
+  jupyter
+  jupyter-notebook-initialized-scripts
   wendelin_test_suite_runner
   wendelin_testrunner
 
@@ -59,15 +60,15 @@ initialization =
 repository_id_list += wendelin
 
 [local-bt5-repository]
-# we need to override it
-list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${wendelin:location}/bt5/
+list += ${wendelin:location}/bt5
 
 # Jupyter is by default enabled in Wendelin
 [erp5-defaults]
 jupyter-enable-default = true
 
 [wendelin]
-<= erp5
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
 repository = https://lab.nexedi.com/nexedi/wendelin.git
 branch = master
 

stack/boinc/buildout.cfg

@@ -69,7 +69,7 @@ pycrypto = 2.6
 apache-libcloud = 0.12.4
 async = 0.6.1
 gitdb = 0.5.4
-mysqlclient = 1.3.10+SlapOSPatched001
+mysqlclient = 1.3.10+SlapOSPatched002
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.4.2
 slapos.toolbox = 0.40.4

stack/caucase/buildout.cfg

@@ -88,17 +88,18 @@ context =
 
 [versions]
 Flask-User = 0.6.11
-SQLAlchemy = 1.1.9
-caucase = 0.1.3
+apache-libcloud = 2.1.0
+bcrypt = 3.1.3
+caucase = 0.1.4
 futures = 3.1.1
+gitdb2 = 2.0.2
 gunicorn = 19.7.1
 slapos.recipe.template = 3.0
-
-APacheDEX = 1.6.2
-bcrypt = 3.1.3
+slapos.toolbox = 0.71
+smmap2 = 2.0.3
 
 # Required by:
-# caucase==0.1.3
+# caucase==0.1.4
 Flask-AlchemyDumps = 0.0.10
 
 # Required by:
@@ -122,6 +123,18 @@ Flask-Script = 2.0.5
 # Flask-User==0.6.11
 Flask-WTF = 0.14.2
 
+# Required by:
+# slapos.toolbox==0.71
+GitPython = 2.1.5
+
+# Required by:
+# slapos.toolbox==0.71
+PyRSS2Gen = 1.1
+
+# Required by:
+# Flask-AlchemyDumps==0.0.10
+SQLAlchemy = 1.1.11
+
 # Required by:
 # Flask-AlchemyDumps==0.0.10
 Unipath = 1.1
@@ -130,14 +143,45 @@ Unipath = 1.1
 # Flask-WTF==0.14.2
 WTForms = 2.1
 
+# Required by:
+# slapos.toolbox==0.71
+atomize = 0.2.0
+
 # Required by:
 # Flask-Mail==0.9.1
 blinker = 1.4
 
 # Required by:
-# caucase==0.1.3
+# slapos.toolbox==0.71
+dnspython = 1.15.0
+
+# Required by:
+# slapos.toolbox==0.71
+erp5.util = 0.4.49
+
+# Required by:
+# slapos.toolbox==0.71
+feedparser = 5.2.1
+
+# Required by:
+# slapos.toolbox==0.71
+lockfile = 0.12.2
+
+# Required by:
+# slapos.toolbox==0.71
+passlib = 1.7.1
+
+# Required by:
+# caucase==0.1.4
 pem = 16.1.0
 
+# caucase==0.1.4
+pyasn1 = 0.2.3
+
+# Required by:
+# caucase==0.1.4
+pyasn1-modules = 0.0.9
+
 # Required by:
 # Flask-User==0.6.11
 pycryptodome = 3.4.6

stack/caucase/buildout.hash.cfg

@@ -28,7 +28,7 @@ md5sum = a317d2f948cd3d16c860d05cc07ecf42
 
 [template-caucase]
 filename = instance-caucase.cfg.jinja2.in
-md5sum = 1988d9c98c2331f49335a1b9a4c26026
+md5sum = 7db9e8bf23cf4689e7986c381b94d2cb
 
 [instance-caucase]
 filename = instance.cfg.in

stack/caucase/instance-caucase.cfg.jinja2.in

@@ -24,6 +24,12 @@ crl-life-period = {{ slapparameter_dict.get('crl-life-period', 0.02) }}
 ca-life-period = {{ slapparameter_dict.get('ca-life-period', 10) }}
 # time before clean certificate on CA: 60*24*60*60
 crt-keep-time = {{ slapparameter_dict.get('crt-keep-time', 5184000) }}
+# number of csr to sign automaticaly, minimum value is 1
+{% if int(slapparameter_dict.get('auto-sign-csr-amount', 1)) < 1 -%}
+auto-sign-csr-amount = 1
+{% else -%}
+auto-sign-csr-amount = {{ slapparameter_dict.get('auto-sign-csr-amount', 1) }}
+{% endif -%}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -135,6 +141,8 @@ input = inline:
   ca-life-period ${ca-parameters:ca-life-period}
   # time before clean certificate on CA: 60*24*60*60
   crt-keep-time ${ca-parameters:crt-keep-time}
+  # number of csr to sign automaticaly
+  auto-sign-csr-amount ${ca-parameters:auto-sign-csr-amount}
 
 output = ${directory:etc}/ca.conf
 mode = 700
@@ -211,6 +219,7 @@ curl_path = {{ curl_executable_location }}
 recipe = slapos.cookbook:publish.serialised
 http-url = ${caucase-server:http-url}
 https-url = ${caucase-server:url}
+init-user = admin
 {% do part_list.append('publish-connection-parameter') -%}
 {% endif -%}
 

stack/erp5/buildout.cfg

@@ -52,6 +52,7 @@ extends =
   ../../component/coreutils/buildout.cfg
   ../../component/grep/buildout.cfg
   ../../component/dash/buildout.cfg
+  ../../component/bash/buildout.cfg
   ../../component/wget/buildout.cfg
   ../../component/aspell/buildout.cfg
   ../../component/cloudooo/buildout.cfg
@@ -61,8 +62,8 @@ extends =
   ../../component/userhosts/buildout.cfg
   ../../component/postfix/buildout.cfg
   ../monitor/buildout.cfg
-  ../../software/ipython_notebook/software.cfg
   ../../software/caucase/software.cfg
+  ../../software/jupyter/software.cfg
   ../../software/neoppod/software-common.cfg
 # keep neoppod extends last
 
@@ -100,6 +101,7 @@ parts +=
   percona-toolkit
   zabbix-agent
   dash
+  bash
   wget
   userhosts
   postfix
@@ -139,15 +141,16 @@ parts +=
   template
 
 # jupyter
-  ipython-notebook
-  instance-jupyter
+  jupyter
+  jupyter-notebook-initialized-scripts
+  instance-jupyter-notebook
 
 # override python2.7 to add SlapOS libstdc++ in RPATH.
 [python2.7]
 extra-ldflags = -Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64
 
-# override instance-jupyter not to render into default template.cfg
-[instance-jupyter]
+# override instance-jupyter-notebook not to render into default template.cfg
+[instance-jupyter-notebook]
 rendered = ${buildout:directory}/template-jupyter.cfg
 
 [instance-caucase]
@@ -167,6 +170,10 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 <= download-base
 mode = 755
 
+[mariadb-slowquery-check-script]
+<= download-base
+mode = 755
+
 [mariadb-slow-query-report-script]
 <= download-base
 mode = 755
@@ -231,6 +238,8 @@ context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
     key apache_location apache:location
+    key apdex_result_check_script apdex-result-check-script:target
+    key mariadb_slowquery_check_script mariadb-slowquery-check-script:target 
     key aspell_location aspell:location
     key bin_directory buildout:bin-directory
     key buildout_bin_directory buildout:bin-directory
@@ -241,6 +250,7 @@ context =
     key curl_location curl:location
     key cyrus_sasl_location cyrus-sasl:location
     key dash_location dash:location
+    key bash_location bash:location
     key dbus_glib_location dbus-glib:location
     key dbus_location dbus:location
     key dcron_location dcron:location
@@ -297,7 +307,7 @@ context =
     key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
-    key template_jupyter_cfg instance-jupyter:rendered
+    key template_jupyter_cfg instance-jupyter-notebook:rendered
     key template_kumofs template-kumofs:target
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
@@ -350,6 +360,9 @@ link-binary =
 [template-balancer]
 <= download-base
 
+[apdex-result-check-script]
+<= download-base
+
 [template-haproxy-cfg]
 <= download-base
 
@@ -371,7 +384,7 @@ recipe = plone.recipe.command
 stop-on-error = true
 genbt5list = ${erp5:location}/product/ERP5/bin/genbt5list
 command =
-  ${buildout:executable} ${:genbt5list} ${local-bt5-repository:list}
+  echo '${local-bt5-repository:list}' |xargs ${buildout:executable} ${:genbt5list}
 update-command = ${:command}
 
 [erp5_repository_list]
@@ -668,7 +681,7 @@ Products.DCWorkflowGraph = 0.4.1
 # Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
 Products.ExternalEditor = 1.1.1
 Products.GenericSetup = 1.8.6
-Products.LongRequestLogger = 2.0.0
+Products.LongRequestLogger = 2.1.0
 # Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition.
 Products.MimetypesRegistry = 2.0.10
 Products.PluginRegistry = 1.4

stack/erp5/instance-cloudoo.cfg.in

@@ -111,7 +111,7 @@ framebuffer = ${:srv}/framebuffer
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ tcpv4_port + 2 }}
-monitor-title = Cloudooo monitor
+monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-passwd'] }}
 
 [binary-link]

stack/erp5/instance-kumofs.cfg.in

@@ -112,5 +112,5 @@ port = ${kumofs-instance:manager-port}
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ tcpv4_port + 4 }}
-monitor-title = Kumofs monitor
+monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-passwd'] }}