Merge remote-tracking branch 'upstream/master' into latest-kvm

CHANGES.rst

 Changes
 =======
+
+1.0.123 (2019-10-03)
+--------------------
+
+- wrapper: accept hash-files already existing inside the partition directory
+
+
+1.0.122 (2019-09-24)
+--------------------
+
+- wrapper: add hash-existing-files option
+
+
+1.0.121 (2019-09-12)
+--------------------
+
+- generic.mysql.wrap_update_mysql: prepare for MariaDB 10.4
+- publish-early: process -init entries in specified order
+- Partial support of Python 3
+- Remove unused generic.mysql recipe
+
+
+1.0.119 (2019-08-14)
+--------------------
+
+ * publish_early: rework API
+
+
+1.0.118 (2019-08-13)
+--------------------
+
+* NEO: new recipe to fix/optimize propagation of the 'masters' parameter
+* publish_early: new '-update' option, keep published values out of buildout installed file
+* publish: new -publish option to list explicitly options to publish
+* re6stnet: Fix typo
+* librecipe: Try to reuse existing file to avoid excessive IO on update and other minor optimisations
+* certificate_authority: unique_subject = no
+* wrapper: handle "=" in environment variables' content
+ 
+
 1.0.92 (2019-02-21)
 -------------------
 

component/apache-php/buildout.cfg

@@ -3,6 +3,7 @@ parts = apache-php
 
 extends =
   ../apache/buildout.cfg
+  ../argon2/buildout.cfg
   ../bzip2/buildout.cfg
   ../cclient/buildout.cfg
   ../curl/buildout.cfg
@@ -16,14 +17,54 @@ extends =
   ../openldap/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../zlib/buildout.cfg
+  ../libzip/buildout.cfg
+  ../autoconf/buildout.cfg
+  ../automake/buildout.cfg
+  ../imagemagick/buildout.cfg
+  ../icu/buildout.cfg
+  ../openssl/buildout.cfg
+
+[php-redis]
+recipe = slapos.recipe.cmmi
+url = https://github.com/phpredis/phpredis/archive/5.0.0.tar.gz
+md5sum = 4f11e0567a10c29394aae52a4fa8bb40
+configure-command = 
+  phpize && ./configure
+environment =
+  PATH=${autoconf:location}/bin:${automake:location}/bin:${m4:location}/bin:${apache-php:location}/bin:%(PATH)s
+
+[php-imagick]
+recipe = slapos.recipe.cmmi
+url = https://github.com/Imagick/imagick/archive/3.4.4.tar.gz
+md5sum = ef6cbadd834eb306bd91874a8f5dea03
+configure-command = 
+  phpize && ./configure
+configure-options =
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+environment =
+  PKG_CONFIG_PATH=${imagemagick:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:${imagemagick:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${m4:location}/bin:${apache-php:location}/bin:%(PATH)s
+  IM_IMAGEMAGICK_PREFIX=${imagemagick:location}
+
+[php-apcu]
+recipe = slapos.recipe.cmmi
+url = https://github.com/krakjoe/apcu/archive/v5.1.17.tar.gz
+md5sum = f64b6cd5108aea63df2d5cc301c58b2b
+configure-command = 
+  phpize && ./configure
+configure-options =
+  --with-php-config=${apache-php:location}/bin/php-config
+environment =
+  PATH=${autoconf:location}/bin:${automake:location}/bin:${m4:location}/bin:${apache-php:location}/bin:%(PATH)s
 
 
 [apache-php]
 # Note: Shall react on each build of apache and reinstall itself
 recipe = slapos.recipe.cmmi
-url = http://fr2.php.net/distributions/php-5.4.12.tar.bz2
-md5sum = 5c7b614242ae12e9cacca21c8ab84818
+url = https://www.php.net/distributions/php-7.3.6.tar.bz2
+md5sum = bde9a912fb311182cd460dad1abbc247
 configure-options =
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
   --with-apxs2=${apache:location}/bin/apxs
   --with-libxml-dir=${libxml2:location}
   --with-mysql=${mariadb:location}
@@ -44,21 +85,31 @@ configure-options =
   --with-ldap=${openldap:location}
   --with-imap-ssl
   --with-openssl=${openssl:location}
+  --with-libzip=${libzip:location}
+  --with-icu-dir=${icu:location}
+  --with-password-argon2=${argon2:location}
+  --enable-apcu-bc
+  --enable-intl
   --enable-libxml
+  --enable-json
   --enable-mbstring
+  --enable-pcntl
   --enable-session
   --enable-exif
-  --enable-zip
   --enable-ftp
+  --enable-zip
+  --disable-zend-test
+  --disable-static
 
 # Changing TMPDIR is required for PEAR installation.
 # It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
 # XXX we could mkdir tmp there
 
 environment =
-  PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${libzip:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
-  LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid
+  CPPFLAGS=-I${libzip:location}/include
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu
   TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
   HOME=${apache:location}
 

component/apache/apache-backend.conf.in

@@ -103,7 +103,7 @@ LoadModule headers_module modules/mod_headers.so
 LoadModule deflate_module modules/mod_deflate.so
 LoadModule filter_module modules/mod_filter.so
 
-AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml application/x-font-ttf application/font-woff application/font-woff2 application/x-font-opentype
+AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript application/javascript image/svg+xml application/x-font-ttf application/font-woff application/font-woff2 application/x-font-opentype application/wasm
 
 PidFile "{{ parameter_dict['pid-file'] }}"
 ServerAdmin admin@
@@ -131,13 +131,11 @@ SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:EC
 SSLSessionCache shmcb:{{ parameter_dict['ssl-session-cache'] }}(512000)
 SSLProxyEngine On
 
-# As backend is trusting REMOTE_USER header unset it always
-RequestHeader unset REMOTE_USER
-RequestHeader unset SSL_CLIENT_SERIAL
+# As backend is trusting Remote-User header unset it always
+RequestHeader unset Remote-User
 {% if parameter_dict['ca-cert'] -%}
 SSLVerifyClient optional
-RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
-RequestHeader set SSL_CLIENT_SERIAL "%{SSL_CLIENT_M_SERIAL}s"
+RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s
 SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
 {%   if parameter_dict['crl'] -%}
 SSLCARevocationCheck chain

component/apache/buildout.hash.cfg

@@ -14,5 +14,5 @@
 # not need these here).
 [template-apache-backend-conf]
 filename = apache-backend.conf.in
-md5sum = c969256e5a9e85a295baf3a695a7c21d
+md5sum = bb8c175a93336f0e1838fd47225426f9
 

component/argon2/buildout.cfg

+[buildout]
+
+parts =
+  argon2
+
+
+[argon2]
+recipe = slapos.recipe.cmmi
+url = https://github.com/P-H-C/phc-winner-argon2/archive/20190702.tar.gz
+md5sum = 0f234cc21ac6cebfe373da6afa70bb1d
+configure-command = :
+make-options =
+  PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
\ No newline at end of file

component/aspell/Compile-Fixes-for-GCC-7.patch

-From 8089fa02122fed0a6394eba14bbedcb1d18e2384 Mon Sep 17 00:00:00 2001
-From: Kevin Atkinson <kevina@gnu.org>
-Date: Thu, 29 Dec 2016 00:50:31 -0500
-Subject: Compile Fixes for GCC 7.
-
-Closes #519.
----
- modules/filter/tex.cpp | 2 +-
- prog/check_funs.cpp    | 2 +-
- 2 files changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/modules/filter/tex.cpp b/modules/filter/tex.cpp
-index a979539..19ab63c 100644
---- a/modules/filter/tex.cpp
-+++ b/modules/filter/tex.cpp
-@@ -174,7 +174,7 @@ namespace {
- 
-     if (c == '{') {
- 
--      if (top.in_what == Parm || top.in_what == Opt || top.do_check == '\0')
-+      if (top.in_what == Parm || top.in_what == Opt || *top.do_check == '\0')
- 	push_command(Parm);
- 
-       top.in_what = Parm;
-diff --git a/prog/check_funs.cpp b/prog/check_funs.cpp
-index db54f3d..89ee09d 100644
---- a/prog/check_funs.cpp
-+++ b/prog/check_funs.cpp
-@@ -647,7 +647,7 @@ static void print_truncate(FILE * out, const char * word, int width) {
-     }
-   }
-   if (i == width-1) {
--    if (word == '\0')
-+    if (*word == '\0')
-       put(out,' ');
-     else if (word[len] == '\0')
-       put(out, word, len);
--- 
-2.15.0.rc0
-

component/aspell/buildout.cfg

@@ -16,15 +16,14 @@ configure-command = ./configure --vars ASPELL=${aspell:location}/bin/aspell PREZ
 [aspell]
 recipe = slapos.recipe.cmmi
 shared = false
-url = http://ftp.gnu.org/gnu/aspell/aspell-0.60.6.1.tar.gz
-md5sum = e66a9c9af6a60dc46134fdacf6ce97d7
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/Compile-Fixes-for-GCC-7.patch#0159a8841e1241153d2105d157fc8225
+url = https://ftp.gnu.org/gnu/aspell/aspell-0.60.7.tar.gz
+md5sum = 8ef2252609c511cd2bb26f3a3932ef28
 environment =
   PATH=${patch:location}/bin:${perl:location}/bin:%(PATH)s
   CPPFLAGS=-I${ncurses:location}/include
   LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
+# aspell parallel build sometimes fail
+make-options = -j1
 
 [aspell-en-dictionary]
 <= aspell-dictionary-common

component/bzip2/buildout.cfg

@@ -5,10 +5,10 @@ parts =
 [bzip2]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://fossies.org/linux/misc/bzip2-1.0.6.tar.gz
-md5sum = 00b516f4704d4a7cb50a1d97e6e8e15b
+url = https://fossies.org/linux/misc/bzip2-1.0.8.tar.gz
+md5sum = 67e051268d0c475ea773822f7500d0e5 
 configure-command = true
 make-options =
   PREFIX=@@LOCATION@@
   CFLAGS="-fpic -fPIC -Wall -Winline -O2 -g -D_FILE_OFFSET_BITS=64"
-post-make-hook =  ${:_profile_base_location_}/bzip2-hooks.py#066c8355b7d726f30176ea5b6a35e1a2:post_make_hook
+post-make-hook =  ${:_profile_base_location_}/bzip2-hooks.py#785148a77f1a8456d914aa02ddf9d410:post_make_hook

component/bzip2/bzip2-hooks.py

@@ -4,7 +4,7 @@ def post_make_hook(options, buildout):
   make_options_list = [q for q in options.get('make-options', '').split('\n') if q]
   if os.system('make %s -f Makefile-libbz2_so' % ' '.join(make_options_list)) != 0:
     raise ValueError('Generation of dynamic library failed')
-  original = 'libbz2.so.1.0.6'
+  original = 'libbz2.so.1.0.8'
   link_list = ['libbz2.so.1.0', 'libbz2.so.1', 'libbz2.so']
   destination = os.path.join(options['location'], 'lib')
   for filename in [original] + link_list:

component/caddy/buildout.cfg

@@ -17,8 +17,8 @@ depends =
 
 [caddy]
 # revision and repository can be used to control which caddy version is used
-revision = v1.0.0
-repository = github.com/mholt/caddy/caddy
+revision = v1.0.3
+repository = github.com/caddyserver/caddy/caddy
 
 recipe  = plone.recipe.command
 update-command = ${:command}

component/cclient/buildout.cfg

@@ -6,17 +6,8 @@ extends =
   ../openssl/buildout.cfg
 
 parts =
-  cclient-patch
   cclient
 
-[cclient-patch]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-download-only = true
-url = ${:_profile_base_location_}/imap-2007f.patch
-md5sum = 42c77fdd5d7a976fc302b93aadb3da98
-filename = imap-2007f.patch
-
 [cclient]
 recipe = slapos.recipe.cmmi
 url = ftp://ftp.cac.washington.edu/imap/imap-2007f.tar.gz
@@ -31,10 +22,12 @@ make-options =
   IP=6
   SSLLIB=${openssl:location}/lib
   EXTRACFLAGS=-fPIC
+  EXTRALDFLAGS="-Wl,-rpath -Wl,${openssl:location}/lib"
   CCLIENT=${buildout:parts-directory}
   -j1
 
-patches = 
-  ${cclient-patch:location}/${cclient-patch:filename}
+patches =
+  ${:_profile_base_location_}/imap-2007f.patch#42c77fdd5d7a976fc302b93aadb3da98
+  ${:_profile_base_location_}/imap-2007f-openssl-1.1.patch#c726354e888f2f3b3954e334903cef80
 
 patch-options = -p1

component/cclient/imap-2007f-openssl-1.1.patch

+From c3f68d987c00284d91ad6599a013b7111662545b Mon Sep 17 00:00:00 2001
+From: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
+Date: Fri, 2 Sep 2016 21:33:33 +0000
+Subject: [PATCH] uw-imap: compile against openssl 1.1.0
+
+I *think* I replaced access to cert->name with certificate's subject name. I
+assume that the re-aranged C-code is doing the same thing. A double check
+wouldn't hurt :)
+
+Signed-off-by: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
+---
+ src/osdep/unix/ssl_unix.c | 28 +++++++++++++++++-----------
+ 1 file changed, 17 insertions(+), 11 deletions(-)
+
+diff --git a/src/osdep/unix/ssl_unix.c b/src/osdep/unix/ssl_unix.c
+index 3bfdff3..836e9fa 100644
+--- a/src/osdep/unix/ssl_unix.c
++++ b/src/osdep/unix/ssl_unix.c
+@@ -59,7 +59,7 @@ typedef struct ssl_stream {
+ static SSLSTREAM *ssl_start(TCPSTREAM *tstream,char *host,unsigned long flags);
+ static char *ssl_start_work (SSLSTREAM *stream,char *host,unsigned long flags);
+ static int ssl_open_verify (int ok,X509_STORE_CTX *ctx);
+-static char *ssl_validate_cert (X509 *cert,char *host);
++static char *ssl_validate_cert (X509 *cert,char *host, char *cert_subj);
+ static long ssl_compare_hostnames (unsigned char *s,unsigned char *pat);
+ static char *ssl_getline_work (SSLSTREAM *stream,unsigned long *size,
+ 			       long *contd);
+@@ -210,6 +210,7 @@ static char *ssl_start_work (SSLSTREAM *stream,char *host,unsigned long flags)
+   BIO *bio;
+   X509 *cert;
+   unsigned long sl,tl;
++  char cert_subj[250];
+   char *s,*t,*err,tmp[MAILTMPLEN];
+   sslcertificatequery_t scq =
+     (sslcertificatequery_t) mail_parameters (NIL,GET_SSLCERTIFICATEQUERY,NIL);
+@@ -266,14 +267,19 @@ static char *ssl_start_work (SSLSTREAM *stream,char *host,unsigned long flags)
+   if (SSL_write (stream->con,"",0) < 0)
+     return ssl_last_error ? ssl_last_error : "SSL negotiation failed";
+ 				/* need to validate host names? */
+-  if (!(flags & NET_NOVALIDATECERT) &&
+-      (err = ssl_validate_cert (cert = SSL_get_peer_certificate (stream->con),
+-				host))) {
+-				/* application callback */
+-    if (scq) return (*scq) (err,host,cert ? cert->name : "???") ? NIL : "";
++  if (!(flags & NET_NOVALIDATECERT)) {
++
++	cert_subj[0] = '\0';
++	cert = SSL_get_peer_certificate(stream->con);
++	if (cert)
++		X509_NAME_oneline(X509_get_subject_name(cert), cert_subj, sizeof(cert_subj));
++	err = ssl_validate_cert (cert, host, cert_subj);
++	if (err)
++		/* application callback */
++		if (scq) return (*scq) (err,host,cert ? cert_subj : "???") ? NIL : "";
+ 				/* error message to return via mm_log() */
+-    sprintf (tmp,"*%.128s: %.255s",err,cert ? cert->name : "???");
+-    return ssl_last_error = cpystr (tmp);
++	sprintf (tmp,"*%.128s: %.255s",err,cert ? cert_subj : "???");
++	return ssl_last_error = cpystr (tmp);
+   }
+   return NIL;
+ }
+@@ -313,7 +319,7 @@ static int ssl_open_verify (int ok,X509_STORE_CTX *ctx)
+  * Returns: NIL if validated, else string of error message
+  */
+ 
+-static char *ssl_validate_cert (X509 *cert,char *host)
++static char *ssl_validate_cert (X509 *cert,char *host, char *cert_subj)
+ {
+   int i,n;
+   char *s,*t,*ret;
+@@ -322,9 +328,9 @@ static char *ssl_validate_cert (X509 *cert,char *host)
+ 				/* make sure have a certificate */
+   if (!cert) ret = "No certificate from server";
+ 				/* and that it has a name */
+-  else if (!cert->name) ret = "No name in certificate";
++  else if (cert_subj[0] == '\0') ret = "No name in certificate";
+ 				/* locate CN */
+-  else if (s = strstr (cert->name,"/CN=")) {
++  else if (s = strstr (cert_subj,"/CN=")) {
+     if (t = strchr (s += 4,'/')) *t = '\0';
+ 				/* host name matches pattern? */
+     ret = ssl_compare_hostnames (host,s) ? NIL :
+-- 
+2.9.3
+

component/coreutils/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 [coreutils]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/gnu/coreutils/coreutils-8.30.tar.xz
-md5sum = ab06d68949758971fe744db66b572816
+url = https://ftp.gnu.org/gnu/coreutils/coreutils-8.31.tar.xz
+md5sum = 0009a224d8e288e8ec406ef0161f9293
 environment =
   PATH=${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
   LDFLAGS=-Wl,--as-needed -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib

component/coturn/buildout.cfg

+[buildout]
+extends =
+  ../sqlite3/buildout.cfg
+  ../openssl/buildout.cfg
+  ../libevent/buildout.cfg
+parts = coturn
+
+[coturn]
+recipe = slapos.recipe.cmmi
+url = https://github.com/coturn/coturn/archive/4.5.1.1.tar.gz
+md5sum = ef20628b026d666be24df056f20a86ea
+environment =
+  PATH=${sqlite3:location}/bin:%(PATH)s
+  CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
+  CFLAGS=-I${libevent2:location}/include
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib

component/f2c/buildout.cfg

+[buildout]
+
+parts =
+  f2c
+
+[libf2c]
+recipe = slapos.recipe.cmmi
+url = https://www.netlib.org/f2c/libf2c.zip#${:md5sum}
+configure-command = true
+md5sum = e80fb30fa3717660db71b1bed65b88b7
+make-binary = make -f makefile.u
+make-targets = install LIBDIR=${:location}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[f2c]
+recipe = slapos.recipe.cmmi
+url = http://www.netlib.org/f2c/src.tgz#${:md5sum}
+md5sum = 7e97530d10ec5783e9d54c1a61eaeed7
+configure-command = true
+make-binary = make -f makefile.u
+make-targets =
+post-install = cp f2c ${:location}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+environment = 
+  CPPFLAGS=-I${libf2c:location}/include
+  LDFLAGS=-L${libf2c:location}/lib
\ No newline at end of file

component/ffmpeg/buildout.cfg

@@ -5,12 +5,13 @@ extends =
   ../patch/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../zlib/buildout.cfg
+  ../xorg/buildout.cfg
 
 parts =
   ffmpeg
 
 [ffmpeg]
-<= ffmpeg-0.8
+<= ffmpeg-4.1
 
 [libogg]
 recipe = slapos.recipe.cmmi
@@ -25,9 +26,11 @@ url = http://downloads.xiph.org/releases/vorbis/libvorbis-1.3.2.tar.bz2
 md5sum = 798a4211221073c1409f26eac4567e8b
 configure-options =
   --disable-static
+pkg_config_depends = ${libogg:location}/lib/pkgconfig
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${libogg:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${:pkg_config_depends}
+  LDFLAGS=-Wl,-rpath=${libogg:location}/lib
 
 [libtheora]
 recipe = slapos.recipe.cmmi
@@ -39,9 +42,16 @@ patches =
 patch-options = -p1
 configure-options =
   --disable-static
+pkg_config_depends = ${libpng:location}/lib/pkgconfig:${libvorbis:location}/lib/pkgconfig:${libvorbis:pkg_config_depends}
 environment =
   PATH=${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${libogg:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${libvorbis:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${:pkg_config_depends}
+  LDFLAGS=-Wl,-rpath=${libvorbis:location}/lib
+
+[nasm]
+recipe = slapos.recipe.cmmi
+url = https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.bz2
+md5sum = 3f489aa48ad2aa1f967dc5e293bbd06f
 
 [yasm]
 recipe = slapos.recipe.cmmi
@@ -49,22 +59,22 @@ url = http://www.tortall.net/projects/yasm/releases/yasm-1.1.0.tar.gz
 
 [libvpx]
 recipe = slapos.recipe.cmmi
-url = http://webm.googlecode.com/files/libvpx-v0.9.6.tar.bz2
-md5sum = 383f3f07a76099682abb43f79b692b72
+url = https://storage.googleapis.com/downloads.webmproject.org/releases/webm/libvpx-1.5.0.tar.bz2
+md5sum = 49e59dd184caa255886683facea56fca
 configure-options =
- --enable-shared
+  --enable-shared
 environment =
   PATH=${yasm:location}/bin:%(PATH)s
 
 [libx264]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.videolan.org/pub/videolan/x264/snapshots/x264-snapshot-20111005-2245-stable.tar.bz2
-md5sum = f86260b2ea6d6cb5186937c9363891d7
+url = ftp://ftp.videolan.org/pub/videolan/x264/snapshots/x264-snapshot-20190214-2245.tar.bz2
+md5sum = 37109ae11d37e4a8963954b53041d43a
 configure-options =
   --enable-shared
   --enable-pic
 environment =
-  PATH=${yasm:location}/bin:%(PATH)s
+  PATH=${yasm:location}/bin:${nasm:location}/bin:%(PATH)s
 
 [lame]
 recipe = slapos.recipe.cmmi
@@ -81,10 +91,11 @@ md5sum = 8e8b8b253eb046340ff7b6bf7a6ccd3e
 configure-options =
   --disable-static
 
-[ffmpeg-0.8]
+[ffmpeg-4.1]
 recipe = slapos.recipe.cmmi
-url = http://ffmpeg.org/releases/ffmpeg-0.8.5.tar.bz2
-md5sum = 726877b19ece7ea64def8b7e6727e182
+url = https://ffmpeg.org/releases/ffmpeg-4.1.4.tar.bz2
+md5sum = 611d171e4aee749b85e04d17e2aee71d
+pkg_config_depends = ${libxcb:location}/lib/pkgconfig:${libxcb:pkg_config_depends}:${libtheora:location}/lib/pkgconfig:${libtheora:pkg_config_depends}:${libvpx:location}/lib/pkgconfig:${libx264:location}/lib/pkgconfig:${opencore-amr:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
 configure-options =
  --enable-gpl
  --enable-version3
@@ -99,9 +110,13 @@ configure-options =
  --enable-libopencore-amrwb
  --enable-shared
  --enable-zlib
+ --enable-libxcb
+ --enable-libxcb-shm
+ --enable-libxcb-xfixes
+ --enable-libxcb-shape
  --disable-static
- --extra-ldflags="-Wl,-rpath=${buildout:parts-directory}/${:_buildout_section_name_}/lib"
 environment =
-  CPPFLAGS=-I${bzip2:location}/include -I${libogg:location}/include -I${libvorbis:location}/include -I${libtheora:location}/include -I${libvpx:location}/include -I${libx264:location}/include -I${lame:location}/include -I${opencore-amr:location}/include -I${zlib:location}/include
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${libogg:location}/lib -Wl,-rpath=${libogg:location}/lib -L${libvorbis:location}/lib -Wl,-rpath=${libvorbis:location}/lib -L${libtheora:location}/lib -Wl,-rpath=${libtheora:location}/lib -L${libvpx:location}/lib -Wl,-rpath=${libvpx:location}/lib -L${libx264:location}/lib -Wl,-rpath=${libx264:location}/lib -L${lame:location}/lib -Wl,-rpath=${lame:location}/lib -L${opencore-amr:location}/lib -Wl,-rpath=${opencore-amr:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-  PATH=${yasm:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${:pkg_config_depends}
+  CPPFLAGS=-I${bzip2:location}/include -I${libogg:location}/include -I${libtheora:location}/include -I${opencore-amr:location}/include -I${lame:location}/include
+  LDFLAGS=-Wl,-rpath=${buildout:parts-directory}/${:_buildout_section_name_}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${libxcb:location}/lib -L${libogg:location}/lib -L${libvorbis:location}/lib -Wl,-rpath=${libvorbis:location}/lib -L${libtheora:location}/lib -Wl,-rpath=${libtheora:location}/lib -L${libvpx:location}/lib -Wl,-rpath=${libvpx:location}/lib -L${libx264:location}/lib -Wl,-rpath=${libx264:location}/lib -L${lame:location}/lib -Wl,-rpath=${lame:location}/lib -L${opencore-amr:location}/lib -Wl,-rpath=${opencore-amr:location}/lib -Wl,-rpath=${zlib:location}/lib
+  PATH=${pkgconfig:location}/bin:${yasm:location}/bin:%(PATH)s
\ No newline at end of file

component/file/0001-fix-libmagic-correctly-detect-msooxml-files.patch

+From a975c66c81e45433a668b7daeb4c903a78cb9d33 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
+Date: Fri, 27 Sep 2019 01:56:24 +0200
+Subject: [PATCH] fix libmagic correctly detect msooxml files
+
+Coming from slapos patch on component:
+
+commit 7a24c4e4ff96a7d00072db891761cea8db7b9122
+Author: Boris Kocherov <bk@raskon.ru>
+Date:   Sun Feb 4 10:52:17 2018 +0300
+
+    component/file: fix libmagic correctly detect msooxml files
+
+that was reformatted as a patch so that component/file supports shared
+slapos.recipe.cmmi. Maybe file now include a proper fix.
+---
+ magic/Magdir/msooxml | 27 ++++++++++++++++++---------
+ 1 file changed, 18 insertions(+), 9 deletions(-)
+
+diff --git a/magic/Magdir/msooxml b/magic/Magdir/msooxml
+index 059e729c..f8431dc8 100644
+--- a/magic/Magdir/msooxml
++++ b/magic/Magdir/msooxml
+@@ -1,4 +1,3 @@
+-
+ #------------------------------------------------------------------------------
+ # $File: msooxml,v 1.5 2014/08/05 07:38:45 christos Exp $
+ # msooxml:  file(1) magic for Microsoft Office XML
+@@ -13,24 +12,34 @@
+ #   which can distinguish between the three types
+ 
+ # start by checking for ZIP local file header signature
+-0		string		PK\003\004
++0               string          PK\003\004
+ !:strength +10
+ # make sure the first file is correct
+->0x1E		regex		\\[Content_Types\\]\\.xml|_rels/\\.rels
++>0x1E           regex           \\[Content_Types\\]\\.xml|_rels/\\.rels
+ # skip to the second local file header
+ # since some documents include a 520-byte extra field following the file
+ # header, we need to scan for the next header
+->>(18.l+49)	search/2000	PK\003\004
++>>(18.l+49)     search/2000     PK\003\004
+ # now skip to the *third* local file header; again, we need to scan due to a
+ # 520-byte extra field following the file header
+->>>&26		search/1000	PK\003\004
++>>>&26          search/1000     PK\003\004
++# and check the subdirectory name to determine which type of OOXML
++# file we have.  Correct the mimetype with the registered ones:
++# http://technet.microsoft.com/en-us/library/cc179224.aspx
++>>>>&26         string          word/           Microsoft Word 2007+
++!:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document
++>>>>&26         string          ppt/            Microsoft PowerPoint 2007+
++!:mime application/vnd.openxmlformats-officedocument.presentationml.presentation
++>>>>&26         string          xl/             Microsoft Excel 2007+
++!:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
++
++>>1104          search/300      PK\003\004
+ # and check the subdirectory name to determine which type of OOXML
+ # file we have.  Correct the mimetype with the registered ones:
+ # http://technet.microsoft.com/en-us/library/cc179224.aspx
+->>>>&26		string		word/		Microsoft Word 2007+
++>>>&26          string          word/           Microsoft Word 2007+
+ !:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document
+->>>>&26		string		ppt/		Microsoft PowerPoint 2007+
++>>>&26          string          ppt/            Microsoft PowerPoint 2007+
+ !:mime application/vnd.openxmlformats-officedocument.presentationml.presentation
+->>>>&26		string		xl/		Microsoft Excel 2007+
++>>>&26          string          xl/             Microsoft Excel 2007+
+ !:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+->>>>&26		default		x		Microsoft OOXML
+-- 
+2.11.0
+

component/file/buildout.cfg

@@ -4,19 +4,12 @@
 [buildout]
 parts = file
 extends =
+  ../patch/buildout.cfg
   ../zlib/buildout.cfg
 
-[file-msooxml]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/msooxml
-md5sum = c889ad135cbfb343db36b729a3897432
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-shared = false
-filename = msooxml
-
 [file]
 recipe = slapos.recipe.cmmi
-shared = false
+shared = true
 url = http://ftp.icm.edu.pl/packages/file/file-5.32.tar.gz
 md5sum = 4f2503752ff041895090ed6435610435
 configure-options =
@@ -24,7 +17,8 @@ configure-options =
 environment =
   CPPFLAGS=-I${zlib:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-pre-configure =
-# patch for fix msooxml files detect correctly
-  test -f ./magic/Magdir/msooxml
-  cp ${file-msooxml:location}/msooxml ./magic/Magdir/
+
+patch-binary = ${patch:location}/bin/patch
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/0001-fix-libmagic-correctly-detect-msooxml-files.patch#5e66a340d8ec7212d485e17d9af95f24

component/file/msooxml

-#------------------------------------------------------------------------------
-# $File: msooxml,v 1.5 2014/08/05 07:38:45 christos Exp $
-# msooxml:  file(1) magic for Microsoft Office XML
-# From: Ralf Brown <ralf.brown@gmail.com>
-
-# .docx, .pptx, and .xlsx are XML plus other files inside a ZIP
-#   archive.  The first member file is normally "[Content_Types].xml".
-#   but some libreoffice generated files put this later. Perhaps skip
-#   the "[Content_Types].xml" test?
-# Since MSOOXML doesn't have anything like the uncompressed "mimetype"
-#   file of ePub or OpenDocument, we'll have to scan for a filename
-#   which can distinguish between the three types
-
-# start by checking for ZIP local file header signature
-0               string          PK\003\004
-!:strength +10
-# make sure the first file is correct
->0x1E           regex           \\[Content_Types\\]\\.xml|_rels/\\.rels
-# skip to the second local file header
-# since some documents include a 520-byte extra field following the file
-# header, we need to scan for the next header
->>(18.l+49)     search/2000     PK\003\004
-# now skip to the *third* local file header; again, we need to scan due to a
-# 520-byte extra field following the file header
->>>&26          search/1000     PK\003\004
-# and check the subdirectory name to determine which type of OOXML
-# file we have.  Correct the mimetype with the registered ones:
-# http://technet.microsoft.com/en-us/library/cc179224.aspx
->>>>&26         string          word/           Microsoft Word 2007+
-!:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document
->>>>&26         string          ppt/            Microsoft PowerPoint 2007+
-!:mime application/vnd.openxmlformats-officedocument.presentationml.presentation
->>>>&26         string          xl/             Microsoft Excel 2007+
-!:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-
->>1104          search/300      PK\003\004
-# and check the subdirectory name to determine which type of OOXML
-# file we have.  Correct the mimetype with the registered ones:
-# http://technet.microsoft.com/en-us/library/cc179224.aspx
->>>&26          string          word/           Microsoft Word 2007+
-!:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document
->>>&26          string          ppt/            Microsoft PowerPoint 2007+
-!:mime application/vnd.openxmlformats-officedocument.presentationml.presentation
->>>&26          string          xl/             Microsoft Excel 2007+
-!:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet

component/findutils/buildout.cfg

@@ -7,8 +7,8 @@ parts =
 [findutils]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/gnu/findutils/findutils-4.4.2.tar.gz
-md5sum = 351cc4adb07d54877fa15f75fb77d39f
+url = http://ftp.debian.org/debian/pool/main/f/findutils/findutils_4.6.0+git+20190510.orig.tar.xz
+md5sum = 9ae8d2b323b0b12a484abcbff1d2c486
 
 [findutils-output]
 # Shared binary location to ease migration

component/firefox/buildout.cfg

@@ -47,6 +47,11 @@ script =
   ))
     os.fchmod(f.fileno(), 0o755)
 
+[firefox-wrapper-68]
+<= firefox-wrapper
+wrapper-name = firefox-68
+part = ${firefox-68:location}
+
 [firefox-wrapper-60]
 <= firefox-wrapper
 wrapper-name = firefox-60
@@ -69,7 +74,13 @@ part = ${firefox-51:location}
 # installed in ${buildout:bin-directory}.
 # Installing ${firefox:} is not enough, because wrapper setting $LD_LIBRARY_PATH
 # would not be created.
-<= firefox-52
+<= firefox-68
+
+[firefox-68]
+<= firefox-download
+version = 68.0.2esr
+i686-md5sum = eaa9e0246eb2a31ccf55c100dc2edd5a
+x86_64-md5sum = d22dc17ce0949cdff78009afca6f2043
 
 [firefox-60]
 <= firefox-download
@@ -115,6 +126,7 @@ library =
   ${gdk-pixbuf:location}/lib
   ${gettext:location}/lib
   ${glib:location}/lib
+  ${gtk-2:location}/lib
   ${gtk-3:location}/lib
   ${harfbuzz:location}/lib
   ${libepoxy:location}/lib
@@ -153,7 +165,13 @@ script =
 
 [geckodriver]
 # Current geckodriver installed as ${buildout:bin-directory}/geckodriver
-<= geckodriver-0.17.0
+<= geckodriver-0.24.0
+
+[geckodriver-0.24.0]
+<= geckodriver-base
+version = 0.24.0
+i686-md5sum = b88eee754f6c90b01f760f7a453dda95
+x86_64-md5sum = 7552b85e43973c84763e212af7cca566
 
 [geckodriver-0.22.0]
 <= geckodriver-base

component/fontconfig/buildout.cfg

@@ -14,7 +14,6 @@ parts =
 
 [fontconfig]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
 md5sum = 733f5e2371ca77b69707bd7b30cc2163
 pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig

component/ghostscript/buildout.cfg

@@ -11,7 +11,6 @@ parts = ghostscript
 
 [ghostscript-common]
 recipe = slapos.recipe.cmmi
-shared = true
 pkg_config_depends = ${libtiff:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}
 configure-options =
   --disable-cups
@@ -33,5 +32,5 @@ environment =
 
 [ghostscript-9]
 <= ghostscript-common
-url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs925/ghostscript-9.25.tar.xz
-md5sum = d5ac3f3d76cf82a549bafdf86d58395b
+url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs927/ghostscript-9.27.tar.xz
+md5sum = dd531503dbbc524f73528359e2ea145c

component/git/buildout.cfg

@@ -18,8 +18,8 @@ parts =
 [git]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.17.1.tar.xz
-md5sum = 5179245515c637357b4a134e8d4e9a6f
+url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.23.0.tar.xz
+md5sum = 93ee0f867f81a39e0ef29eabfb1d2c5b
 configure-options =
   --with-curl=${curl:location}
   --with-openssl=${openssl:location}

component/golang/buildout.cfg

@@ -12,8 +12,9 @@ parts = gowork
 
 [golang-common]
 recipe = slapos.recipe.cmmi
+shared = true
 configure-command = :
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+location = @@LOCATION@@
 make-binary =
 make-targets= cd src && ./all.bash && cp -alf .. ${:location}
 # some testdata files have an issue with slapos.extension.strip.
@@ -23,14 +24,11 @@ environment =
   GOROOT_FINAL=${:location}
   ${:environment-extra}
 
-[golang14-repository]
-<= git-repository
-repository = https://github.com/golang/go.git
-revision = go1.4.3-16-g4d5426a
-
 [golang14]
 <= golang-common
-path = ${golang14-repository:location}
+# https://golang.org/doc/install/source#bootstrapFromSource
+url = https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz
+md5sum = dbf727a4b0e365bf88d97cbfde590016
 environment-extra =
 make-targets= cd src && ./make.bash && cp -alf .. ${:location}
 
@@ -64,8 +62,8 @@ environment-extra =
 
 [golang1.12]
 <= golang-common
-url = https://dl.google.com/go/go1.12.6.src.tar.gz
-md5sum = 48a4141fc718dd742d106431294f08bf
+url = https://dl.google.com/go/go1.12.9.src.tar.gz
+md5sum = 6132109d4050da349eadc9f7b0304ef4
 
 # go1.11 needs go1.4 to bootstrap
 environment-extra =

component/graphviz/buildout.cfg

@@ -13,7 +13,6 @@ extends =
 
 [graphviz]
 recipe = slapos.recipe.cmmi
-shared = true
 url = https://ftp.osuosl.org/pub/blfs/conglomeration/graphviz/graphviz-2.40.1.tar.gz
 md5sum = 4ea6fd64603536406166600bcc296fc8
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}

component/groonga/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = false
-url = https://packages.groonga.org/source/groonga/groonga-9.0.0.tar.gz
-md5sum = 5475818c734dfc6414d209babea90921
+url = https://packages.groonga.org/source/groonga/groonga-9.0.9.tar.gz
+md5sum = 877bc3e2840e2935f606fb3d2d359a2c
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
@@ -46,8 +46,8 @@ environment =
 [groonga-normalizer-mysql]
 recipe = slapos.recipe.cmmi
 shared = false
-url = https://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-1.1.3.tar.gz
-md5sum = ad30404cb9999a842e98f3902057b152
+url = https://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-1.1.4.tar.gz
+md5sum = effa67fb271d49810850a3b275d040f6
 location = ${groonga:location}
 configure-options =
   --disable-static

component/gtk-2/buildout.cfg

@@ -25,7 +25,6 @@ parts =
 
 [cairo]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://cairographics.org/releases/cairo-1.14.2.tar.xz
 md5sum = e1cdfaf1c6c995c4d4c54e07215b0118
 pkg_config_depends = ${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${glib:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libpng:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
@@ -47,9 +46,8 @@ environment =
 
 [harfbuzz]
 recipe = slapos.recipe.cmmi
-shared = true
-url = http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-0.9.35.tar.bz2
-md5sum = 531ee8650626ecddcd90b2a4637e31d4
+url = http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-2.3.1.tar.bz2
+md5sum = 531de9df7c8a5405dd9c6a873fcee8c2
 pkg_config_depends = ${cairo:location}/lib/pkgconfig:${cairo:pkg_config_depends}:${icu4c:location}/lib/pkgconfig
 configure-options =
   --disable-static
@@ -60,14 +58,22 @@ configure-options =
 environment =
   PATH=${glib:location}/bin:${freetype:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${icu4c:location}/lib -Wl,-rpath=${icu4c:location}/lib
 
-[pango]
+[fribidi]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/pango-1.40.3.tar.xz
-md5sum = 17c26720f5a862a12f7e1745e2f1d966
-pkg_config_depends = ${harfbuzz:location}/lib/pkgconfig:${harfbuzz:pkg_config_depends}
+url = https://github.com/fribidi/fribidi/releases/download/v1.0.5/fribidi-1.0.5.tar.bz2
+md5sum = 0f97f65038545340316ec0f4bd53df0b
+configure-options =
+  --enable-shared
+  --disable-static
+
+[pango]
+recipe = slapos.recipe.cmmi
+url = http://ftp.gnome.org/pub/gnome/core/3.32/3.32.2/sources/pango-1.42.4.tar.xz
+md5sum = deb171a31a3ad76342d5195a1b5bbc7c
+pkg_config_depends = ${fribidi:location}/lib/pkgconfig:${harfbuzz:location}/lib/pkgconfig:${harfbuzz:pkg_config_depends}
 configure-options =
   --disable-static
   --disable-gtk-doc-html
@@ -75,7 +81,7 @@ environment =
   PATH=${glib:location}/bin:${freetype:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${bzip2:location}/include -I${zlib:location}/include
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${harfbuzz:location}/lib -Wl,-rpath=${harfbuzz:location}/lib
 
 [gdk-pixbuf]
 recipe = slapos.recipe.cmmi
@@ -111,7 +117,6 @@ environment =
 
 [gtk-2]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/gtk+-2.24.31.tar.xz
 md5sum = 526a1008586094a2cbb4592fd3f9ee10
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig
@@ -128,4 +133,4 @@ environment =
   PKG_CONFIG_PATH=${:pkg_config_depends}
 # not taken from pkg-config result...
   CPPFLAGS=-I${libX11:location}/include/ -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${renderext:location}/include -I${libXext:location}/include
-  LDFLAGS=-L${libX11:location}/lib -L${libXext:location}/lib -L${libXrender:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-L${libX11:location}/lib -L${libXext:location}/lib -L${libXrender:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${harfbuzz:location}/lib -Wl,-rpath=${harfbuzz:location}/lib

component/gtkmm/buildout.cfg

@@ -16,7 +16,6 @@ parts =
 
 [cairomm]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://cairographics.org/releases/cairomm-1.13.1.tar.gz
 md5sum = 21fe892652741b7544f52da6965d27fc
 pkg_config_depends = ${cairo:location}/lib/pkgconfig:${cairo:pkg_config_depends}:${libsigc:location}/lib/pkgconfig
@@ -32,7 +31,6 @@ environment =
 
 [pangomm]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/pangomm-2.40.1.tar.xz
 md5sum = 874eadd9434613dbacf0272c82c3ac23
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${glibmm:location}/lib/pkgconfig:${glibmm:pkg_config_depends}:${cairomm:location}/lib/pkgconfig
@@ -43,11 +41,10 @@ environment =
   PATH=${gcc:location}/bin:${glib:location}/bin:${freetype:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   LD_LIBRARY_PATH=${gcc:location}/lib:${gcc:location}/lib64
-  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
+  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${harfbuzz:location}/lib -Wl,-rpath=${harfbuzz:location}/lib
 
 [atkmm]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/atkmm-2.24.2.tar.xz
 md5sum = d53b60b0f1be597e86070954a49cf0c3
 pkg_config_depends = ${atk:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${glibmm:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
@@ -62,7 +59,6 @@ environment =
 
 [gtkmm]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://ftp.gnome.org/pub/GNOME/sources/gtkmm/2.24/gtkmm-2.24.5.tar.xz
 md5sum = 6c59ae8bbff48fad9132f23af347acf1
 pkg_config_depends = ${pangomm:location}/lib/pkgconfig:${pangomm:pkg_config_depends}:${atkmm:location}/lib/pkgconfig:${atkmm:pkg_config_depends}:${gtk-2:location}/lib/pkgconfig:${gtk-2:pkg_config_depends}
@@ -73,4 +69,4 @@ environment =
   PATH=${gcc:location}/bin:${gdk-pixbuf:location}/bin:${glib:location}/bin:${gtk-2:location}/bin:${pango:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   LD_LIBRARY_PATH=${gcc:location}/lib:${gcc:location}/lib64
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${harfbuzz:location}/lib -Wl,-rpath=${harfbuzz:location}/lib

component/icu/buildout.cfg

@@ -4,9 +4,9 @@ extends =
 parts = icu4c
 
 [icu4c]
-# need for couchdb
 recipe = slapos.recipe.cmmi
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+shared = true
+location = @@LOCATION@@
 url = http://download.icu-project.org/files/icu4c/58.2/icu4c-58_2-src.tgz
 md5sum = fac212b32b7ec7ab007a12dff1f3aea1
 configure-command = source/configure

component/imagemagick/buildout.cfg

@@ -25,7 +25,6 @@ extends =
 
 [imagemagick]
 recipe = slapos.recipe.cmmi
-shared = true
 version = 7.0.2-10
 url = https://www.imagemagick.org/download/releases/ImageMagick-${:version}.tar.xz
 md5sum = e1cb23d9c10a8eff228ef30ee281711a

component/inkscape/buildout.cfg

@@ -27,7 +27,6 @@ extends =
 
 [gsl]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://ftp.gnu.org/gnu/gsl/gsl-2.3.tar.gz
 md5sum = 905fcbbb97bc552d1037e34d200931a0
 configure-options =
@@ -40,7 +39,6 @@ environment =
 
 [inkscape]
 recipe = slapos.recipe.cmmi
-shared = true
 url = https://inkscape.org/gallery/item/10682/inkscape-0.92.1.tar_XlpI7qT.bz2
 md5sum = db2eb2a566cf35ff949fc9ccb172889a
 location = @@LOCATION@@

component/kumo/buildout.cfg

@@ -14,7 +14,6 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/downloads/etolabo/kumofs/kumofs-0.4.13.tar.gz
 md5sum = 46148e9536222d0ad2ef36777c55714d
-pre-configure-hook = ${:_profile_base_location_}/kumo-hooks.py#958a595a02de75624728f8d65e39d800:pre_configure_hook
 patches =
   ${:_profile_base_location_}/kumofs-0.4.13_ipv6support_multiiplistenfix.patch#53af9f1f1375940841c589a6cbe11425
   ${:_profile_base_location_}/kumofs-0.4.13_fix_gcc-4.9_ftbfs.patch#c09e04c620ce11c3fdd4afc3459cd355

component/kumo/kumo-hooks.py

-import os
-import sys
-import traceback
-from shutil import copy
-from subprocess import Popen, PIPE
-
-CONFIGURE_PATH = os.path.join('configure')
-CONFIGURE_BACKUP_PATH = CONFIGURE_PATH + '_disabled'
-# Fake configure, generating a fake Makefile which will create a marker file
-# instead of actually installing anything.
-# This is needed (?) to fetch --prefix from configure parameters, so we know
-# where to tell Makefile to put the dummy file.
-FAKE_CONFIGURE = '''#!%(python)s -S
-import os
-import sys
-print 'Configuration is disabled on this host because %%s'
-print 'Original configure file available at %(backup)s'
-prefix = None
-next = False
-for arg in sys.argv:
-    if next:
-        prefix = arg
-        break
-    if arg.startswith('--prefix'):
-        if arg.startswith('--prefix='):
-            _, prefix = arg.split('=', 1)
-            break
-        next = True
-if prefix is None:
-    raise '--prefix parameter not found'
-# Generate Makefile with proper prefix
-open('Makefile', 'w').write("""all:
-\techo 'make disabled, see configure'
-
-install:
-\ttouch %%%%s""" %%%% (
-  os.path.join(prefix, 'BUILD_DISABLED_BY_BUILDOUT'),
-))
-sys.exit(0)
-''' % {
-    'backup': CONFIGURE_BACKUP_PATH,
-    'python': sys.executable,
-}
-
-def pre_configure_hook(options, buildout):
-    gcc_executable = os.getenv('CC', 'gcc')
-    try:
-        gcc = Popen([gcc_executable, '-v'], stdout=PIPE, stderr=PIPE,
-            close_fds=True)
-    except OSError, (errno, _):
-        if errno == 2:
-            # No gcc installed, nothing to check
-            pass
-        else:
-            print 'Unexpected failure trying to detect gcc version'
-            traceback.print_exc()
-    else:
-        gcc.wait()
-        # Considered innocent until proven guilty.
-        error = None
-        for line in '\n'.join((gcc.stdout.read(), gcc.stderr.read())).splitlines():
-            if line.startswith('gcc version'):
-                if '4.1.1' in line and 'prerelease' in line:
-                    # There is a bug in 4.1.1 prerelease (ie, as of mandriva
-                    # 2007.0) g++ preventing kumo compilation from succeeding.
-                    error = 'broken GCC version: %s' % (line, )
-                break
-        else:
-            print >>sys.stderr, 'GCC version could not be detected, ' \
-                'building anyway'
-        if error is not None:
-            print 'Disabling build, with reason:', error
-            # Copy to preserver permission
-            copy(CONFIGURE_PATH, CONFIGURE_BACKUP_PATH)
-            open(CONFIGURE_PATH, 'w').write(FAKE_CONFIGURE % (error, ))
-

component/libdmtx/buildout.cfg

@@ -22,7 +22,6 @@ configure-options =
 
 [dmtx-utils]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://prdownloads.sourceforge.net/libdmtx/dmtx-utils-0.7.4.tar.bz2
 md5sum = b132ab9fb1d289869469b8bb4959a08a
 configure-options =

component/libevent/buildout.cfg

 [buildout]
 parts = libevent
+extends =
+  ../openssl/buildout.cfg
 
 [libevent]
 recipe = slapos.recipe.cmmi
@@ -7,3 +9,11 @@ url = http://www.monkey.org/~provos/libevent-1.4.13-stable.tar.gz
 md5sum = 0b3ea18c634072d12b3c1ee734263664
 configure-options =
   --disable-static
+
+[libevent2]
+<= libevent
+url = https://github.com/libevent/libevent/releases/download/release-2.1.11-stable/libevent-2.1.11-stable.tar.gz
+md5sum = 7f35cfe69b82d879111ec0d7b7b1c531
+environment =
+  CPPFLAGS=-I${openssl:location}/include
+  LDFLAGS=-L${openssl:location}/lib
\ No newline at end of file

component/librsvg/buildout.cfg

@@ -22,7 +22,6 @@ environment =
 
 [librsvg]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/librsvg-2.40.16.tar.xz
 md5sum = f474fe37177a2bf8050787df2046095c
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${zlib:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${libcroco:location}/lib/pkgconfig

component/libzip/buildout.cfg

+[buildout]
+
+extends =
+  ../bzip2/buildout.cfg
+  ../cmake/buildout.cfg
+  ../zlib/buildout.cfg
+
+parts = libzip
+
+[libzip]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://libzip.org/download/libzip-1.5.2.tar.xz
+md5sum = f9dd38d273bcdec5d3d1498fe6684f42
+location = @@LOCATION@@
+configure-command =
+  mkdir build && cd build && \
+  ${cmake:location}/bin/cmake \
+    -DCMAKE_INSTALL_PREFIX=${:location} \
+    -DCMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include \
+    -DCMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib \
+    ..
+make-binary =
+  cd build && make
+environment =
+  PATH=${cmake:location}/bin:%(PATH)s
+  CMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
+  CMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
+  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib

component/lz4/buildout.cfg

@@ -5,6 +5,6 @@ parts =
 
 [lz4]
 recipe = slapos.recipe.cmmi
-url = https://github.com/lz4/lz4/archive/v1.8.3.tar.gz
-md5sum = d5ce78f7b1b76002bbfffa6f78a5fc4e
+url = https://github.com/lz4/lz4/archive/v1.9.2.tar.gz
+md5sum = 3898c56c82fb3d9455aefd48db48eaad
 configure-command = true

component/make/buildout.cfg

@@ -17,7 +17,7 @@ recipe = slapos.recipe.cmmi
 url = http://ftp.de.debian.org/debian/pool/main/m/make-dfsg/make-dfsg_3.81.orig.tar.gz
 md5sum = 7c93b1ab4680eb21c2c13f4f47741e2d
 patches =
-  ${:_profile_base_location_}/make-dfsg_3.81-8.2.diff#fa77bb989a096fafbe7c78582e9415e3
+  ${:_profile_base_location_}/make-dfsg_3.81-8.2.diff#320ce09344a8618b6c47ddb88c09cea9
 patch-options = -p1
 environment =
   PATH=${patch:location}/bin:%(PATH)s

component/make/make-dfsg_3.81-8.2.diff

@@ -1044,6 +1044,15 @@
  # Autoconf setup
  AC_CONFIG_AUX_DIR(config)
  AC_CONFIG_SRCDIR(vpath.c)
+@@ -354,7 +356,7 @@
+ #define GLOB_INTERFACE_VERSION 1
+ #if !defined _LIBC && defined __GNU_LIBRARY__ && __GNU_LIBRARY__ > 1
+ # include <gnu-versions.h>
+-# if _GNU_GLOB_INTERFACE_VERSION == GLOB_INTERFACE_VERSION
++# if _GNU_GLOB_INTERFACE_VERSION >= GLOB_INTERFACE_VERSION
+    gnu glob
+ # endif
+ #endif
 @@ -372,6 +374,24 @@
  MAKE_HOST="$host"
  AC_SUBST(MAKE_HOST)
@@ -16744,7 +16753,15 @@
  /* end confdefs.h.  */
  
  #include <features.h>
-@@ -13629,13 +9246,13 @@
+@@ -13622,20 +9239,20 @@
+ #define GLOB_INTERFACE_VERSION 1
+ #if !defined _LIBC && defined __GNU_LIBRARY__ && __GNU_LIBRARY__ > 1
+ # include <gnu-versions.h>
+-# if _GNU_GLOB_INTERFACE_VERSION == GLOB_INTERFACE_VERSION
++# if _GNU_GLOB_INTERFACE_VERSION >= GLOB_INTERFACE_VERSION
+    gnu glob
+ # endif
+ #endif
  
  _ACEOF
  if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |

component/mariadb/buildout.cfg

@@ -29,9 +29,13 @@ parts =
 [mariadb]
 recipe = slapos.recipe.cmmi
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.3.14
-md5sum = b1b9628bbc3ff15e8f5cfc7896c73975
+version = 10.3.20
+md5sum = a87a9532568c0ed50d25ce7e7c60d67d
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
+pre-configure =
+  set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
+  grep -q "$@"
+  sed -i "/$1/d" "$2"
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
   -DCMAKE_INSTALL_PREFIX=${:location}
@@ -80,8 +84,8 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = https://packages.groonga.org/source/mroonga/mroonga-9.00.tar.gz
-md5sum = a1deff08a3649d8370436f1c903ed432
+url = https://packages.groonga.org/source/mroonga/mroonga-9.09.tar.gz
+md5sum = 1b4ec3a8c6b4b459bd7a94f2bb8ad585
 pre-configure = set -e
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source

component/nginx/buildout.cfg

@@ -12,19 +12,12 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-shared = true
-url = http://nginx.org/download/nginx-1.10.1.tar.gz
-md5sum = 088292d9caf6059ef328aa7dda332e44
-patch-options = -p0
-patch-binary = ${patch:location}/bin/patch
-patches =
-  http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
-  ${:_profile_base_location_}/fix-gcc7-implicit-fallthrough-errors.patch
+url = https://nginx.org/download/nginx-1.17.1.tar.gz
+md5sum = 51021f3e8204a5fc809f5e695a4508db
 
 [nginx]
 <= nginx-common
 configure-options=
-  --with-ipv6
   --with-http_ssl_module
   --with-http_v2_module
   --with-http_gzip_static_module

component/nginx/fix-gcc7-implicit-fallthrough-errors.patch

-commit 8449f750e62cd229026e9df3bd023ec7e073a7d4
-Author: Maxim Dounin <mdounin@mdounin.ru>
-Date:   Thu Apr 27 16:57:18 2017 +0300
-
-    Added missing "fall through" comments (ticket #1259).
-    
-    Found by gcc7 (-Wimplicit-fallthrough).
-
-diff --git src/core/ngx_murmurhash.c src/core/ngx_murmurhash.c
-index c31e0e03..5ade658d 100644
---- src/core/ngx_murmurhash.c
-+++ src/core/ngx_murmurhash.c
-@@ -35,8 +35,10 @@ ngx_murmur_hash2(u_char *data, size_t len)
-     switch (len) {
-     case 3:
-         h ^= data[2] << 16;
-+        /* fall through */
-     case 2:
-         h ^= data[1] << 8;
-+        /* fall through */
-     case 1:
-         h ^= data[0];
-         h *= 0x5bd1e995;
-diff --git src/http/ngx_http_parse.c src/http/ngx_http_parse.c
-index 36220fdc..e8e51563 100644
---- src/http/ngx_http_parse.c
-+++ src/http/ngx_http_parse.c
-@@ -1396,6 +1396,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
-                 goto done;
-             case '+':
-                 r->plus_in_uri = 1;
-+                /* fall through */
-             default:
-                 state = sw_usual;
-                 *u++ = ch;
-@@ -1437,6 +1438,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
-                 goto done;
-             case '+':
-                 r->plus_in_uri = 1;
-+                /* fall through */
-             default:
-                 state = sw_usual;
-                 *u++ = ch;
-@@ -1484,6 +1486,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
-                 goto done;
-             case '+':
-                 r->plus_in_uri = 1;
-+                /* fall through */
-             default:
-                 state = sw_usual;
-                 *u++ = ch;
-diff --git src/os/unix/ngx_process.c src/os/unix/ngx_process.c
-index dd50b5ca..993c032a 100644
---- src/os/unix/ngx_process.c
-+++ src/os/unix/ngx_process.c
-@@ -413,6 +413,7 @@ ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
-                 break;
-             }
-             ngx_debug_quit = 1;
-+            /* fall through */
-         case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
-             ngx_quit = 1;
-             action = ", shutting down";

component/nodejs/0001-fix-execvp-printf-argument-list-too-long.patch

---- a/deps/npm/node_modules/node-gyp/gyp/pylib/gyp/generator/make.py
-+++ b/deps/npm/node_modules/node-gyp/gyp/pylib/gyp/generator/make.py
-@@ -144,6 +144,25 @@
- quiet_cmd_link = LINK($(TOOLSET)) $@
- cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ -Wl,--start-group $(LD_INPUTS) -Wl,--end-group $(LIBS)
- 
-+define xargs
-+  $(1) $(wordlist 1,100,$(2))
-+  $(if $(word 101,$(2)),$(call xargs,$(1),$(wordlist 101,$(words $(2)),$(2))))
-+endef
-+
-+define write-to-file
-+  @echo >$(1)
-+  $(call xargs,printf "%s\\n" >>$(1),$(2))
-+endef
-+
-+OBJ_FILE_LIST_SFX := ar-file-list
-+
-+define create_archive
-+  $(eval OBJ_FILE_LIST := $(basename $(notdir $(1))).$(OBJ_FILE_LIST_SFX))
-+  rm -f $(1) $(OBJ_FILE_LIST); mkdir -p `dirname $(1)`
-+  $(call write-to-file,$(OBJ_FILE_LIST),$(filter %.o,$(2)))
-+  $(AR.$(TOOLSET)) crs $(1) @$(OBJ_FILE_LIST)
-+endef
-+
- # We support two kinds of shared objects (.so):
- # 1) shared_library, which is just bundling together many dependent libraries
- # into a link line.
-@@ -1583,8 +1600,7 @@
-         self.WriteDoCmd([self.output_binary], link_deps, 'alink_thin',
-                         part_of_all, postbuilds=postbuilds)
-       else:
--        self.WriteDoCmd([self.output_binary], link_deps, 'alink', part_of_all,
--                        postbuilds=postbuilds)
-+        self.WriteMakeRule([self.output_binary], link_deps, actions = ['$(call create_archive,$@,$^)'])
-     elif self.type == 'shared_library':
-       self.WriteLn('%s: LD_INPUTS := %s' % (
-             QuoteSpaces(self.output_binary),
---- a/tools/gyp/pylib/gyp/generator/make.py
-+++ b/tools/gyp/pylib/gyp/generator/make.py
-@@ -144,6 +144,25 @@
- quiet_cmd_link = LINK($(TOOLSET)) $@
- cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ -Wl,--start-group $(LD_INPUTS) -Wl,--end-group $(LIBS)
- 
-+define xargs
-+  $(1) $(wordlist 1,100,$(2))
-+  $(if $(word 101,$(2)),$(call xargs,$(1),$(wordlist 101,$(words $(2)),$(2))))
-+endef
-+
-+define write-to-file
-+  @echo >$(1)
-+  $(call xargs,printf "%s\\n" >>$(1),$(2))
-+endef
-+
-+OBJ_FILE_LIST_SFX := ar-file-list
-+
-+define create_archive
-+  $(eval OBJ_FILE_LIST := $(basename $(notdir $(1))).$(OBJ_FILE_LIST_SFX))
-+  rm -f $(1) $(OBJ_FILE_LIST); mkdir -p `dirname $(1)`
-+  $(call write-to-file,$(OBJ_FILE_LIST),$(filter %.o,$(2)))
-+  $(AR.$(TOOLSET)) crs $(1) @$(OBJ_FILE_LIST)
-+endef
-+
- # We support two kinds of shared objects (.so):
- # 1) shared_library, which is just bundling together many dependent libraries
- # into a link line.
-@@ -1584,8 +1601,7 @@
-         self.WriteDoCmd([self.output_binary], link_deps, 'alink_thin',
-                         part_of_all, postbuilds=postbuilds)
-       else:
--        self.WriteDoCmd([self.output_binary], link_deps, 'alink', part_of_all,
--                        postbuilds=postbuilds)
-+        self.WriteMakeRule([self.output_binary], link_deps, actions = ['$(call create_archive,$@,$^)'])
-     elif self.type == 'shared_library':
-       self.WriteLn('%s: LD_INPUTS := %s' % (
-             QuoteSpaces(self.output_binary),

component/nodejs/buildout.cfg

@@ -13,7 +13,7 @@ parts =
   nodejs
 
 [nodejs]
-<= nodejs-0.12
+<= nodejs-8.9.4
 
 [nodejs-8.9.4]
 <= nodejs-base
@@ -88,57 +88,6 @@ environment =
   CPPFLAGS=-I${zlib:location}/include
   LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
 
-[nodejs-0.12]
-# Server-side Javascript.
-recipe = slapos.recipe.cmmi
-version = v0.12.18
-url = http://nodejs.org/dist/${:version}/node-${:version}.tar.gz
-md5sum = 1292af0584de9ba773eb06120b5ddb86
-configure-options =
-  --openssl-includes=${openssl:location}/include
-  --openssl-libpath=${openssl:location}/lib
-environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
-  PATH=${pkgconfig:location}/bin:${patch:location}/bin:${python2.7:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/0001-fix-execvp-printf-argument-list-too-long.patch#9a7335043e1e030d623f7c624d8456f0
-
-
-[nodejs-0.10]
-# Server-side Javascript.
-recipe = slapos.recipe.cmmi
-version = v0.10.43
-url = http://nodejs.org/dist/${:version}/node-${:version}.tar.gz
-md5sum = 2356f55d70a013d7952aad17c25cbcfe
-configure-options =
-  --openssl-includes=${openssl:location}/include
-  --openssl-libpath=${openssl:location}/lib
-environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
-  PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-
-[nodejs-0.8]
-# Server-side Javascript.
-recipe = slapos.recipe.cmmi
-url = http://nodejs.org/dist/v0.8.23/node-v0.8.23.tar.gz
-md5sum = 22fe54ac365f52d3d80ecf748e7323d5
-configure-options =
-  --openssl-includes=${openssl:location}/include
-  --openssl-libpath=${openssl:location}/lib
-environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
-  PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-
 [nodejs-0.6]
 # Server-side Javascript.
 recipe = slapos.recipe.cmmi

component/nss/buildout.cfg

@@ -6,8 +6,8 @@ extends =
 
 [nss]
 recipe = slapos.recipe.cmmi
-url = https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_29_RTM/src/nss-3.29.tar.gz
-md5sum = 253d1ae4cf5a560373bbadb4bf483945
+url = https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_45_RTM/src/nss-3.45.tar.gz
+md5sum = f1752d7223ee9d910d551e57264bafa8
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-command = make -j1 -C nss/coreconf/nsinstall all
 # XXX How to build nss-config and pkg-config files ?

component/openldap/buildout.cfg

@@ -9,8 +9,8 @@ extends =
 [openldap]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-2.4.42.tgz
-md5sum = 47c8e2f283647a6105b8b0325257e922
+url = http://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-2.4.47.tgz
+md5sum = e508f97bfd778fec7799f286e5c07176
 configure-options =
   --disable-static
   --disable-slapd
@@ -24,6 +24,6 @@ configure-options =
   --with-tls=openssl
 
 environment =
-  CPPFLAGS=-I${openssl-1.0:location}/include -I${cyrus-sasl:location}/include
-  LDFLAGS=-L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib -L${cyrus-sasl:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib
+  CPPFLAGS=-I${openssl:location}/include -I${cyrus-sasl:location}/include
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${cyrus-sasl:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib
   PATH=${groff:location}/bin:%(PATH)s

component/openssh/buildout.cfg

@@ -15,12 +15,12 @@ parts =
 
 [openssh]
 recipe = slapos.recipe.cmmi
-md5sum = b2db2a83caf66a208bb78d6d287cdaa3
-url = http://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.4p1.tar.gz
+md5sum = 68ba883aff6958297432e5877e9a0fe2
+url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.7p1.tar.gz
 patch-binary = ${patch:location}/bin/patch
-#patch-options = -p1
+patch-options = -p1
 patches =
-  ${:_profile_base_location_}/no_create_privsep_path.patch#d5b61a2442fffa457cebe4ad1dc68f4e
+  ${:_profile_base_location_}/no_create_privsep_path.patch#f341dc11d73df6f43c7ae1fa47b8c003
 environment = 
   CPPFLAGS=-I${zlib:location}/include -I${openssl-1.0:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib

component/openssh/no_create_privsep_path.patch

---- Makefile.in	2016-07-28 00:54:27.000000000 +0200
-+++ Makefile.in	2016-08-19 13:02:30.227177750 +0200
-@@ -304,7 +304,6 @@
- 	$(srcdir)/mkinstalldirs $(DESTDIR)$(mandir)/$(mansubdir)5
- 	$(srcdir)/mkinstalldirs $(DESTDIR)$(mandir)/$(mansubdir)8
- 	$(srcdir)/mkinstalldirs $(DESTDIR)$(libexecdir)
--	(umask 022 ; $(srcdir)/mkinstalldirs $(DESTDIR)$(PRIVSEP_PATH))
+From 46cf5eba19cf52ffae48ea95f07a36b4c107ebd8 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
+Date: Sun, 20 May 2018 20:47:16 +0900
+Subject: [PATCH] Do not create PRIVSEP_PATH
+
+For SlapOS, this would need to be in instance, but instance paths are
+not known at software compilation time.  Because we don't do privileges
+separation, just disable creation of this directory.
+---
+ Makefile.in | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/Makefile.in b/Makefile.in
+index 04e1c8e5..9bd5d01b 100644
+--- a/Makefile.in
++++ b/Makefile.in
+@@ -329,7 +329,6 @@ install-files:
+ 	$(MKDIR_P) $(DESTDIR)$(mandir)/$(mansubdir)5
+ 	$(MKDIR_P) $(DESTDIR)$(mandir)/$(mansubdir)8
+ 	$(MKDIR_P) $(DESTDIR)$(libexecdir)
+-	$(MKDIR_P) -m 0755 $(DESTDIR)$(PRIVSEP_PATH)
  	$(INSTALL) -m 0755 $(STRIP_OPT) ssh$(EXEEXT) $(DESTDIR)$(bindir)/ssh$(EXEEXT)
  	$(INSTALL) -m 0755 $(STRIP_OPT) scp$(EXEEXT) $(DESTDIR)$(bindir)/scp$(EXEEXT)
  	$(INSTALL) -m 0755 $(STRIP_OPT) ssh-add$(EXEEXT) $(DESTDIR)$(bindir)/ssh-add$(EXEEXT)
+-- 
+2.11.0
+

component/percona-toolkit/buildout.cfg

@@ -7,6 +7,7 @@ parts =
   percona-toolkit
 
 [percona-toolkit]
+shared = false
 <= perl-CPAN-package
 # XXX it's not on CPAN, so we use url
 version = 3.0.3

component/perl-DBD-MySQL/buildout.cfg

@@ -10,6 +10,7 @@ parts =
 
 [perl-DBD-MySQL-common]
 <= perl-CPAN-package
+shared = false
 module = DBD/DBD-mysql
 version = 4.046_01
 md5sum = 90f87aec4b7ab95a9543f2dd10969a07

component/perl-Image-Magick/buildout.cfg

@@ -12,6 +12,7 @@ parts = perl-Image-Magick
 # version 6.77 is not in `by-module` folder, so we use url= until we update
 url = http://search.cpan.org/CPAN/authors/id/J/JC/JCRISTY/PerlMagick-6.77.tar.gz
 md5sum = fa0f66fa0cabbd1b196254f94dec8e99
+shared = false
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/perl-Image-Magick_MakefilePL.patch#31043e2b79e725d3b251aa09b4549046

component/perl/buildout.cfg

@@ -8,10 +8,11 @@ parts =
 
 [perl]
 recipe = slapos.recipe.cmmi
+shared = true
 version = 5.28.1
 url = http://www.cpan.org/src/5.0/perl-${:version}.tar.xz
 md5sum = fbb590c305f2f88578f448581b8cf9c4
-siteprefix = ${buildout:parts-directory}/site_${:_buildout_section_name_}
+siteprefix =@@LOCATION@@/site_${:_buildout_section_name_}
 patch-options = -p1
 patches =
   ${:_profile_base_location_}/perl-keep-linker-flags-in-ldflags.patch#4e8e0c59d7176eafb0c7402dea17bef1
@@ -21,7 +22,7 @@ patches =
   ${:_profile_base_location_}/0004-disable-probing.patch#e68446ccd155c2282639e495a5be612b
 configure-command =
   sh Configure -des \
-    -Dprefix=${buildout:parts-directory}/${:_buildout_section_name_} \
+    -Dprefix=@@LOCATION@@ \
     -Dsiteprefix=${:siteprefix} \
     -Dcflags=-I${gdbm:location}/include \
     -Dldflags="-L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib" \
@@ -76,6 +77,7 @@ perl-PERL5LIB =
 
 # Implementation
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://www.cpan.org/modules/by-module/${:module}-${:version}.tar.gz
 configure-command =
   ${:extra-env}
@@ -84,7 +86,7 @@ configure-command =
   Makefile.PL \
   PREFIX=${:location} \
   ${:extra-configure-args}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+location = @@LOCATION@@
 # Install scripts in "perl-bin", but create wrappers in ./bin/
 make-options =
   INSTALLSITESCRIPT=${:location}/perl-bin/
@@ -93,7 +95,7 @@ make-binary=
   ${:extra-env} PERL5LIB="${:inc}:${:install-inc}:${:perl-PERL5LIB}" make
 
 # this post-make-hook is same for all users of the macro.
-post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#d012f7ba3300b14b2c6b173351d410be:post_make_hook
+post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#f28c45a0f473ae050ca3ebaed5c39a4e:post_make_hook
 
 perl_location = ${perl:location}
 

component/perl/perl-CPAN-package-create-wrapper.py

@@ -17,7 +17,7 @@ def post_make_hook(options, buildout, environmet):
 export PERL5LIB="{site_perl}:$PERL5LIB"
 exec {perl_location}/bin/perl "$@"
 '''.format(**locals()))
-  os.chmod(perl_wrapper_path, 0755)
+  os.chmod(perl_wrapper_path, 0o755)
 
   # create a wrapper for each scripts installed in perl-bin
   for script_path in glob.glob(os.path.join(prefix, 'perl-bin', '*')):
@@ -28,4 +28,4 @@ exec {perl_location}/bin/perl "$@"
 export PERL5LIB="{site_perl}:$PERL5LIB"
 exec {perl_location}/bin/perl {script_path} "$@"
 '''.format(**locals()))
-      os.chmod(wrapper_path, 0755)
+      os.chmod(wrapper_path, 0o755)

component/phantomjs/buildout.cfg

@@ -2,7 +2,6 @@
 extends =
   ../fontconfig/buildout.cfg
   ../libexpat/buildout.cfg
-  ../dash/buildout.cfg
 parts =
   phantomjs
 
@@ -19,17 +18,19 @@ x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686
 x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
 
 script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
+  if not self.options.get('url'):
+    self.options['url'], self.options['md5sum'] = \
+      self.options[guessPlatform()].split(' ')
+  extract_dir = self.extract(self.download(self.options['url'],
+                             self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
   self.copyTree(workdir, "%(location)s")
   wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
-  wrapper = open(wrapper_location, 'w')
-  wrapper.write("""#!${dash:location}/bin/dash
+  with open(wrapper_location, 'w') as wrapper:
+    wrapper.write("""#!/bin/sh
   cd %(location)s
   export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
   export PATH=${fontconfig:location}/bin:$PATH
-  exec %(location)s/bin/phantomjs $*""")
-  wrapper.flush()
-  wrapper.close()
-  os.chmod(wrapper_location, 0755)
+  exec %(location)s/bin/phantomjs "$@"
+  """)
+  os.chmod(wrapper_location, 0o755)

component/poppler/buildout.cfg

@@ -13,7 +13,6 @@ extends =
 
 [poppler]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://poppler.freedesktop.org/poppler-0.43.0.tar.xz
 md5sum = 1d2b001663119855cdfbc0713dbfb9c6
 configure-options =

component/protobuf/buildout.cfg

@@ -5,13 +5,8 @@ parts = protobuf
 recipe = slapos.recipe.cmmi
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-python-3.4.0.tar.gz
 md5sum = 0820cc2e56d71aef8e99794fcbd184cd
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [protobuf-cpp]
 recipe = slapos.recipe.cmmi
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
 md5sum = 6d59dad503bea5ad420fd09ddad84481
-configure-command =
-  ./autogen.sh
-  ./configure --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-

component/python-cachecontrol/buildout.cfg

+[buildout]
+
+[python-cachecontrol]
+recipe = zc.recipe.egg:custom
+egg = cachecontrol
+setup-eggs =
+  pbr
+  lockfile

component/python3/buildout.cfg

@@ -6,11 +6,13 @@ extends =
   ../gettext/buildout.cfg
   ../libexpat/buildout.cfg
   ../libffi/buildout.cfg
+  ../libpng/buildout.cfg
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
   ../patch/buildout.cfg
   ../readline/buildout.cfg
   ../sqlite3/buildout.cfg
+  ../gcc/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
 
@@ -37,6 +39,7 @@ configure-options =
   --with-system-expat
   --with-system-ffi
   --with-threads
+  --with-openssl=${openssl:location}
 # Profiled build:
   --enable-optimizations
 pre-install = mkdir profile-opt
@@ -58,3 +61,19 @@ md5sum = 57d1f8bfbabf4f2500273fb0706e6f21
 package_version = 3.6.6
 md5sum = c3f30a0aff425dda77d19e02f420d6ba
 executable = @@LOCATION@@/bin/python3.6
+
+# Python 3.7 have to use gcc 8(actually, gcc 4+)
+# See https://bugs.python.org/issue34112
+# I think gcc 5 may meet the requirement, use gcc 8 all in one step
+[python3.7]
+<= python3-common
+package_version = 3.7.2
+md5sum = df6ec36011808205beda239c72f947cb
+patch-options =
+patches =
+executable = @@LOCATION@@/bin/python3.7
+environment =
+  PATH=${gcc-8.2:location}/bin::${xz-utils:location}/bin:%(PATH)s
+  CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
+  LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
+  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
\ No newline at end of file

component/qemu-kvm/buildout.cfg

@@ -19,8 +19,8 @@ extends =
 [kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-2.12.0.tar.xz
-md5sum = ca553eb04c933f58111c304452fc4cc5
+url = https://download.qemu.org/qemu-4.1.0.tar.xz
+md5sum = cdf2b5ca52b9abac9bacb5842fa420f8
 configure-options =
   --target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
   --enable-system
@@ -38,6 +38,7 @@ configure-options =
   --extra-cflags="-I${gnutls:location}/include -I${libuuid:location}/include -I${ncurses:location}/include -I${zlib:location}/include -I${libpng:location}/include -I${libaio:location}/include -I${attr:location}/include -I${libcap:location}/include"
   --extra-ldflags="-Wl,-rpath -Wl,${glib:location}/lib -L${glib:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath -Wl,${libgpg-error:location}/lib -L${libgpg-error:location}/lib -L${gettext:location}/lib -Wl,-rpath -Wl,${gettext:location}/lib -Wl,-rpath -Wl,${libpng:location}/lib -L${libpng:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lpng -lz -lgnutls -L${attr:location}/lib -Wl,-rpath=${attr:location}/lib -L${libcap:location}/lib -Wl,-rpath=${libcap:location}/lib"
   --disable-werror
+  --disable-xfsctl
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
@@ -67,15 +68,21 @@ md5sum = 096c1c18b44c269808bd815d58c53c8f
 version = 8.11.1
 md5sum = df0ce86d0b1d81e232ad08eef58754ed
 
+[debian-amd64-stretch-netinst.iso]
+<= debian-amd64-netinst-base
+version = 9.11.0
+md5sum = f525f0c3f1c4ca184a604a75dabf4f71
+
 [debian-amd64-netinst.iso]
-# Download the installer of Debian 9 (Stretch)
+# Download the installer of Debian 10 (Buster)
+# XXX: This is not the latest version because
+#      Debian does not provide a stable URL for it.
 <= debian-amd64-netinst-base
-version = 9.8.0
-md5sum = e0a43cbb8b991735c1b38e7041019658
+version = 10.0.0
+md5sum = f31779fcca35f5ce9833a9661a9bd5bd
 
 [debian-amd64-testing-netinst.iso]
-# Download the installer of Debian Buster
 <= debian-amd64-netinst-base
-release = buster_di_rc1
-version = buster-DI-rc1
-md5sum = cf8f8e3afef91f3ce3a09e7cc5f530f0
+release = daily/20190923-1
+version = testing
+md5sum = 00eda4218c401c46c15f491add41cd4e

component/randomsleep/buildout.cfg

@@ -7,16 +7,16 @@ extends =
 recipe = slapos.recipe.build
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 bin_dir = ${buildout:bin-directory}
-bash_script_code = 
+bash_script_code =
   if [ "$#" -ne 1 ]; then
     echo "usage: randomsleep maxseconds"
     exit
   fi
-  ${coreutils:location}/bin/sleep $((RANDOM % $1)).$((RANDOM % 100))
-wrapper_script_code = 
+  exec ${coreutils:location}/bin/sleep $((RANDOM * $1 >> 15)).$((RANDOM * 100 >> 15))
+wrapper_script_code =
   #!${bash:location}/bin/bash
-  ${bash:location}/bin/bash ${:location}/randomsleep.bash "$@"
-script = 
+  . ${:location}/randomsleep.bash
+script =
   os.makedirs(self.options['location'])
 
   bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')

component/serf/buildout.cfg

@@ -9,7 +9,6 @@ extends =
 
 [serf]
 recipe = slapos.recipe.cmmi
-shared = true
 url = https://serf.googlecode.com/files/serf-1.2.1.tar.bz2
 md5sum = 4f8e76c9c6567aee1d66aba49f76a58b
 configure-options =

component/slapos/buildout.cfg

@@ -3,23 +3,13 @@
 extends =
   ../../stack/slapos.cfg
   ../bison/buildout.cfg
-  ../bzip2/buildout.cfg
   ../firewalld/buildout.cfg
-  ../gdbm/buildout.cfg
-  ../gettext/buildout.cfg
   ../glib/buildout.cfg
-  ../libxml2/buildout.cfg
-  ../libxslt/buildout.cfg
   ../m4/buildout.cfg
-  ../ncurses/buildout.cfg
-  ../openssl/buildout.cfg
-  ../patch/buildout.cfg
   ../pkgconfig/buildout.cfg
-  ../python-2.7/buildout.cfg
   ../readline/buildout.cfg
   ../sqlite3/buildout.cfg
   ../swig/buildout.cfg
-  ../zlib/buildout.cfg
   ../socat/buildout.cfg
 
 parts =
@@ -87,6 +77,7 @@ eggs =
   ${lxml-python:egg}
   ${python-cffi:egg}
   ${python-PyYAML:egg}
+  ${python-cachecontrol:egg}
   ${python-cliff:egg}
   ${python-cryptography:egg}
   pyOpenSSL

component/subversion/buildout.cfg

@@ -20,7 +20,6 @@ parts =
 
 [subversion]
 recipe = slapos.recipe.cmmi
-shared = true
 url = http://apache.mirrors.tds.net/subversion/subversion-1.8.14.tar.bz2
 md5sum = fe476ba26d6835eba4393780ea907361
 # Patch available thanks to gentoo developpers

component/tmux/buildout.cfg

@@ -8,10 +8,10 @@ parts +=
 
 [tmux]
 recipe = slapos.recipe.cmmi
-url = https://github.com/tmux/tmux/releases/download/2.0/tmux-2.0.tar.gz
-md5sum = 9fb6b443392c3978da5d599f1e814eaa
+url = https://github.com/tmux/tmux/releases/download/2.9a/tmux-2.9a.tar.gz
+md5sum = f0564dbf4cf6b301c4845219d27ed3ad
 environment =
-  CFLAGS=-I${ncurses:location}/include -I${libevent:location}/include/
-  LDFLAGS=-L${ncurses:location}/lib/ -L${libevent:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/ -Wl,-rpath=${libevent:location}/lib/
+  CFLAGS=-I${ncurses:location}/include -I${libevent2:location}/include/
+  LDFLAGS=-L${ncurses:location}/lib/ -L${libevent2:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/ -Wl,-rpath=${libevent2:location}/lib/
 configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}

component/tomcat/buildout.cfg

@@ -20,8 +20,8 @@ md5sum = 3dde098fd0b3a08d3f2867e4a95591ba
 recipe = hexagonit.recipe.download
 ignore-existing = true
 strip-top-level-dir = true
-url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.91/bin/apache-tomcat-7.0.91.tar.gz
-md5sum = 8bfbb358b51f90374067879f8db1e91c
+url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.96/bin/apache-tomcat-7.0.96.tar.gz
+md5sum = 0669aa2996b67c61662a5a4f993767b8
 
 [tomcat9]
 recipe = hexagonit.recipe.download

component/vm-img/debian.cfg

@@ -22,7 +22,7 @@ dist = ${vm-debian:dists}
 [vm-debian]
 recipe = slapos.recipe.build:vm.install-debian
 environment = vm-install-environment
-dists = debian-stretch
+dists = debian-stable
 size = 1Gi
 late-command =
 # rdnssd causes too much trouble with recent QEMU, because the latter acts as
@@ -60,23 +60,33 @@ preseed.apt-setup/enable-source-repositories = false
 preseed.recommends = false
 preseed.tasks =
 
+[debian-stable]
+x86_64.iso = debian-amd64-netinst.iso
+x86_64.kernel = install.amd/vmlinuz
+x86_64.initrd = install.amd/initrd.gz
+
+[debian-testing]
+<= debian-stable
+x86_64.iso = debian-amd64-testing-netinst.iso
+
 [debian-squeeze]
-<= debian-stretch
+<= debian-stable
 x86_64.iso = debian-amd64-squeeze-netinst.iso
 
 [debian-wheezy]
-<= debian-stretch
+<= debian-stable
 x86_64.iso = debian-amd64-wheezy-netinst.iso
 
 [debian-jessie]
-<= debian-stretch
+<= debian-stable
 x86_64.iso = debian-amd64-jessie-netinst.iso
 
 [debian-stretch]
-x86_64.iso = debian-amd64-netinst.iso
-x86_64.kernel = install.amd/vmlinuz
-x86_64.initrd = install.amd/initrd.gz
+<= debian-stable
+x86_64.iso = debian-amd64-stretch-netinst.iso
 
 [debian-buster]
-<= debian-stretch
-x86_64.iso = debian-amd64-testing-netinst.iso
+<= debian-stable
+
+[debian-bullseye]
+<= debian-testing

component/xorg/buildout.cfg

@@ -116,8 +116,9 @@ patch-options = -p1
 configure-options =
   --disable-static
   --disable-build-docs
+pkg_config_depends = ${libXau:location}/lib/pkgconfig:${xcbproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
 environment =
-  PKG_CONFIG_PATH=${libXau:location}/lib/pkgconfig:${xcbproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${:pkg_config_depends}
   PATH=${patch:location}/bin:${pkgconfig:location}/bin:${libxslt:location}/bin:%(PATH)s
   PYTHON=${buildout:executable}
 # Python note: libxcb requires python with ElementTree. In case of appliance
@@ -353,7 +354,6 @@ environment =
 
 [libxkbfile]
 recipe = slapos.recipe.cmmi
-shared = true
 url = https://www.x.org/releases/individual/lib/libxkbfile-1.0.9.tar.gz
 md5sum = 5aab87eba67f37dd910a19be5c1129ee
 environment =
@@ -364,7 +364,6 @@ environment =
 
 [xkeyboard-config]
 recipe = slapos.recipe.cmmi
-shared = true
 url = https://www.x.org/releases/individual/data/xkeyboard-config-2.5.1.tar.gz
 md5sum = 62c6583b4ec5775717e7c8b05732763c
 configure-options =
@@ -378,7 +377,6 @@ environment =
 
 [xkbcomp]
 recipe = slapos.recipe.cmmi
-shared = true
 url = https://www.x.org/releases/individual/app/xkbcomp-1.4.2.tar.gz
 md5sum = 84b6bafb660181a8c2572981a7fff54d
 # Hardcoded location for xkeyboard-config, needed during compile time
@@ -561,7 +559,6 @@ environment =
 
 [xwd]
 recipe = slapos.recipe.cmmi
-shared = true
 url = https://www.x.org/releases/individual/app/xwd-1.0.7.tar.gz
 md5sum = 3ebd74f7a1980305e5e19ec8ff7aa794
 environment =
@@ -573,7 +570,6 @@ environment =
 [xserver]
 # Adds Xvfb functionnality
 recipe = slapos.recipe.cmmi
-shared = true
 url = https://www.x.org/releases/individual/xserver/xorg-server-1.20.1.tar.gz
 md5sum = f5ba68452b1710306aabc32308c6ac59
 patches =

component/zstd/buildout.cfg

@@ -4,8 +4,9 @@ parts =
 
 [zstd]
 recipe = slapos.recipe.cmmi
-url = https://github.com/facebook/zstd/archive/v1.3.8.tar.gz
-md5sum = 0e85b062003729776c6114cd9632cf75
+url = https://github.com/facebook/zstd/releases/download/v${:version}/zstd-${:version}.tar.gz
+version = 1.4.4
+md5sum = 487f7ee1562dee7c1c8adf85e2a63df9
 shared = true
 location = @@LOCATION@@
 configure-command = :

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.92'
+version = '1.0.123'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read() + "\n" + \
     open("CHANGES.rst").read() + "\n"
@@ -110,9 +110,7 @@ setup(name=name,
           'generic.cloudooo = slapos.recipe.generic_cloudooo:Recipe',
           'generic.kumofs = slapos.recipe.generic_kumofs:Recipe',
           'generic.memcached = slapos.recipe.generic_memcached:Recipe',
-          'generic.mysql = slapos.recipe.generic_mysql:Recipe',
           'generic.mysql.wrap_update_mysql = slapos.recipe.generic_mysql:WrapUpdateMySQL',
-          'generic.mysql.wrap_mysqld = slapos.recipe.generic_mysql:WrapMySQLd',
           'generic.varnish = slapos.recipe.generic_varnish:Recipe',
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
@@ -134,6 +132,7 @@ setup(name=name,
           'mydumper = slapos.recipe.mydumper:Recipe',
           'mysql = slapos.recipe.mysql:Recipe',
           'nbdserver = slapos.recipe.nbdserver:Recipe',
+          'neoppod.cluster = slapos.recipe.neoppod:Cluster',
           'neoppod.admin = slapos.recipe.neoppod:Admin',
           'neoppod.master = slapos.recipe.neoppod:Master',
           'neoppod.storage = slapos.recipe.neoppod:Storage',

slapos/recipe/apachephpconfigure/__init__.py

@@ -24,6 +24,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
+
 from slapos.recipe.librecipe import GenericBaseRecipe
 import zc.buildout
 import sys
@@ -101,17 +103,17 @@ class Recipe(GenericBaseRecipe):
 
     # TODO factor
     if delete != []:
-      print "Creating lampconfigure with 'delete' arguments"
+      print("Creating lampconfigure with 'delete' arguments")
       command = argument + delete
     if rename != []:
       for parameters in rename:
-        print "Creating lampconfigure with 'rename' arguments"
+        print("Creating lampconfigure with 'rename' arguments")
         command = argument + rename
     if chmod != []:
-      print "Creating lampconfigure with 'chmod' arguments"
+      print("Creating lampconfigure with 'chmod' arguments")
       command = argument + chmod
     if data != []:
-      print "Creating lampconfigure with 'run' arguments"
+      print("Creating lampconfigure with 'run' arguments")
       command = argument + data
 
 

slapos/recipe/bonjourgrid/__init__.py

@@ -94,7 +94,7 @@ class Recipe(GenericBaseRecipe):
       dict(ip_address=self.options['ipv6'].strip(),
             project=project,
             middleware=type)))
-    os.chmod(config_info_file, 0744)
+    os.chmod(config_info_file, 0o744)
     path_list.append(config_info)
 
   update = install

slapos/recipe/certificate_authority/__init__.py

@@ -26,11 +26,11 @@
 ##############################################################################
 import os
 import hashlib
-import ConfigParser
+from six.moves import configparser
 import tempfile
 
 from slapos.recipe.librecipe import GenericBaseRecipe
-from certificate_authority import popenCommunicate
+from .certificate_authority import popenCommunicate
 
 class Recipe(GenericBaseRecipe):
 
@@ -119,7 +119,7 @@ class Request(Recipe):
       open(certificate, 'w').write(cert_content)
       request_needed = False
     else:
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       parser.add_section('certificate')
       parser.set('certificate', 'name', name)
       parser.set('certificate', 'key_file', key)

slapos/recipe/certificate_authority/certificate_authority.py

+from __future__ import print_function
+
 import os
 import subprocess
 import time
-import ConfigParser
+from six.moves import configparser
 import uuid
 
 
@@ -95,12 +97,12 @@ class CertificateAuthority:
 
   def checkRequestDir(self):
     for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
       if self._checkCertificate(parser.get('certificate', 'name'),
           parser.get('certificate', 'key_file'), parser.get('certificate',
             'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
+        print('Created certificate %r' % parser.get('certificate', 'name'))
 
 def runCertificateAuthority(*args):
   ca = CertificateAuthority(*args)

slapos/recipe/condor/__init__.py

@@ -29,7 +29,6 @@ import os
 import subprocess
 import zc.buildout
 import filecmp
-import urlparse
 import shutil
 import re
 import json
@@ -130,9 +129,9 @@ class Recipe(GenericBaseRecipe):
 
     #create condor binary launcher for slapos
     if not os.path.exists(self.wrapper_bin):
-      os.makedirs(self.wrapper_bin, int('0744', 8))
+      os.makedirs(self.wrapper_bin, int('0o744', 8))
     if not os.path.exists(self.wrapper_sbin):
-      os.makedirs(self.wrapper_sbin, int('0744', 8))
+      os.makedirs(self.wrapper_sbin, int('0o744', 8))
     #generate script for each file in prefix/bin
     for binary in os.listdir(self.prefix+'/bin'):
       wrapper_location = os.path.join(self.wrapper_bin, binary)
@@ -153,7 +152,7 @@ class Recipe(GenericBaseRecipe):
       wrapper.write(content)
       wrapper.close()
       path_list.append(wrapper_location)
-      os.chmod(wrapper_location, 0744)
+      os.chmod(wrapper_location, 0o744)
 
     #generate script for each file in prefix/sbin
     for binary in os.listdir(self.prefix+'/sbin'):
@@ -175,7 +174,7 @@ class Recipe(GenericBaseRecipe):
       wrapper.write(content)
       wrapper.close()
       path_list.append(wrapper_location)
-      os.chmod(wrapper_location, 0744)
+      os.chmod(wrapper_location, 0o744)
 
     #generate script for start condor
     wrapper = self.createPythonScript(
@@ -228,7 +227,7 @@ class AppSubmit(GenericBaseRecipe):
         for file in file_list:
           if file and (file.startswith('http') or file.startswith('ftp')):
             file_list[file] = self.download(file_list[file])
-          os.chmod(file_list[file], 0600)
+          os.chmod(file_list[file], 0o600)
       else:
         app_list[app]['files'] = {}
 
@@ -236,11 +235,11 @@ class AppSubmit(GenericBaseRecipe):
       if executable and (executable.startswith('http') or executable.startswith('ftp')):
         app_list[app]['executable'] = self.download(executable,
                                       app_list[app]['executable-name'])
-        os.chmod(app_list[app]['executable-name'], 0700)
+        os.chmod(app_list[app]['executable-name'], 0o700)
       submit_file = app_list[app].get('description-file', '')
       if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
         app_list[app]['description-file'] = self.download(submit_file, 'submit')
-        os.chmod(app_list[app]['description-file'], 0600)
+        os.chmod(app_list[app]['description-file'], 0o600)
 
     return app_list
 

slapos/recipe/davstorage/__init__.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 import subprocess
-import httplib
+from six.moves import http_client as httplib
 import base64
 import os
 import shutil

slapos/recipe/dcron.py

@@ -29,6 +29,8 @@ import os
 from slapos.recipe.librecipe import GenericBaseRecipe
 from zc.buildout import UserError
 
+from six.moves import map
+
 class Recipe(GenericBaseRecipe):
 
   def install(self):
@@ -124,7 +126,7 @@ def systemd_to_cron(spec):
     x = spec[i]
     if x != '*':
       for x in x.split(','):
-        x = map(int, x.split('/', 1))
+        x = list(map(int, x.split('/', 1)))
         a = x[0] - y
         if 0 <= a < z:
           if len(x) == 1:

slapos/recipe/erp5_promise/__init__.py

@@ -26,7 +26,7 @@
 ##############################################################################
 
 from slapos.recipe.librecipe import GenericBaseRecipe
-import ConfigParser
+from six.moves import configparser
 
 class Recipe(GenericBaseRecipe):
   """
@@ -34,7 +34,7 @@ class Recipe(GenericBaseRecipe):
   """
 
   def install(self):
-    promise_parser = ConfigParser.RawConfigParser()
+    promise_parser = configparser.RawConfigParser()
     for section_name, option_id_list in (
           ('portal_templates', (
             ('repository', 'bt5-repository-url'),

slapos/recipe/erp5testnode/__init__.py

@@ -24,10 +24,10 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import ConfigParser
+from six.moves import configparser
+import io
 import json
 import os
-import StringIO
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -40,13 +40,13 @@ class Recipe(GenericBaseRecipe):
     CONFIG['PATH'] = os.environ['PATH']
 
     if self.options['instance-dict']:
-      config_instance_dict = ConfigParser.ConfigParser()
+      config_instance_dict = configparser.ConfigParser()
       config_instance_dict.add_section('instance_dict')
       instance_dict = json.loads(self.options['instance-dict'])
 
       for k ,v in instance_dict.iteritems():
         config_instance_dict.set('instance_dict', k, v)
-      value = StringIO.StringIO()
+      value = io.StringIO()
       config_instance_dict.write(value)
       CONFIG['instance_dict'] = value.getvalue()
 

slapos/recipe/free_port.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 
-import ConfigParser
+from six.moves import configparser
 import os
 import netaddr
 import socket
@@ -48,7 +48,7 @@ class Recipe(object):
     # If this check isn't done, a new port would be picked for every upgrade
     # of the software release
     try:
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       if os.path.exists(buildout['buildout']['installed']):
         with open(buildout['buildout']['installed']) as config_file:
           parser.readfp(config_file)
@@ -59,7 +59,7 @@ class Recipe(object):
         if port != '0':
           self.options['port'] = port
           return
-    except (IOError, ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+    except (IOError, configparser.NoSectionError, configparser.NoOptionError):
       pass
 
     # Otherwise, let's find one

slapos/recipe/generic_cloudooo/__init__.py

@@ -24,9 +24,11 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from functools import cmp_to_key
 import zc.buildout
 from slapos.recipe.librecipe import GenericBaseRecipe
 
+@cmp_to_key
 def compareMimetypeEntryPair(a, b):
   """
     Like comparing strings, but here the star `*` is stronger than any other
@@ -115,7 +117,7 @@ class Recipe(GenericBaseRecipe):
       if l and not l.isspace()
     ]
     mimetype_entry_list.extend(default_mimetype_entry_list)
-    mimetype_entry_list.sort(compareMimetypeEntryPair)
+    mimetype_entry_list.sort(key=compareMimetypeEntryPair)
     conversion_server_dict['MIMETYPE_ENTRY_LIST'] = \
       "\n".join(["  " + l for l in mimetype_entry_list])
     config_file = self.createFile(self.options['configuration-file'],

slapos/recipe/generic_mysql/__init__.py

@@ -25,206 +25,6 @@
 #
 ##############################################################################
 from slapos.recipe.librecipe import GenericBaseRecipe
-import os
-
-class Recipe(GenericBaseRecipe):
-
-  def _options(self, options):
-    options['password'] = self.generatePassword()
-    if 'test-database' in options:
-      options['test-password'] = self.generatePassword()
-    options.setdefault('parallel-test-database-amount', '0')
-    for x in xrange(int(options['parallel-test-database-amount'])):
-      options['test-password-%s' % x] = self.generatePassword()
-
-  def install(self):
-    path_list = []
-
-    template_filename = self.getTemplateFilename('my.cnf.in')
-
-    mysql_binary = self.options['mysql-binary']
-    socket = self.options['socket']
-
-    if 'ip' in self.options:
-      networking = 'port = %s\nbind-address = %s' % (
-        self.options['port'],
-        self.options['ip'],
-      )
-    else:
-      networking = 'skip-networking'
-
-    log_bin = self.options.get('binlog-path', '')
-    if log_bin:
-      log_bin = 'log_bin = %s' % log_bin
-    expire_logs_days = self.options.get('binlog-expire-days')
-    if expire_logs_days > 0:
-      expire_logs_days = 'expire_logs_days = %s' % expire_logs_days
-    else:
-      expire_logs_days = ''
-
-    mysql_conf_file = self.createFile(
-      self.options['conf-file'],
-      self.substituteTemplate(template_filename, {
-        'networking': networking,
-        'data_directory': self.options['data-directory'],
-        'pid_file': self.options['pid-file'],
-        'socket': self.options['socket'],
-        'error_log': self.options['error-log'],
-        'slow_query_log': self.options['slow-query-log'],
-        'log_bin': log_bin,
-        'expire_logs_days': expire_logs_days,
-      })
-    )
-    path_list.append(mysql_conf_file)
-
-    mysql_script_list = []
-
-    # user defined functions
-    udf_registration = "DROP FUNCTION IF EXISTS last_insert_grn_id;\nDROP FUNCTION IF EXISTS mroonga_snippet;\nDROP FUNCTION IF EXISTS mroonga_command;\n"
-    mroonga = self.options.get('mroonga', 'ha_mroonga.so')
-    if mroonga:
-      udf_registration += "CREATE FUNCTION last_insert_grn_id RETURNS " \
-        "INTEGER SONAME '" + mroonga + "';\n"
-      udf_registration += "CREATE FUNCTION mroonga_snippet RETURNS " \
-        "STRING SONAME '" + mroonga + "';\n"
-      udf_registration += "CREATE FUNCTION mroonga_command RETURNS " \
-        "STRING SONAME '" + mroonga + "';\n"
-    mysql_script_list.append(self.substituteTemplate(
-      self.getTemplateFilename('mysql-init-function.sql.in'),
-      {
-        'udf_registration': udf_registration,
-      }
-    ))
-    # real database
-    mysql_script_list.append(self.substituteTemplate(
-      self.getTemplateFilename('initmysql.sql.in'),
-      {
-        'mysql_database': self.options['database'],
-        'mysql_user': self.options['user'],
-        'mysql_password': self.options['password']
-      }
-    ))
-    # default test database
-    if 'test-database' in self.options:
-      mysql_script_list.append(self.substituteTemplate(
-        self.getTemplateFilename('initmysql.sql.in'),
-        {
-          'mysql_database': self.options['test-database'],
-          'mysql_user': self.options['test-user'],
-          'mysql_password': self.options['test-password']
-        }
-      ))
-    # parallel test databases
-    for x in xrange(int(self.options['parallel-test-database-amount'])):
-      mysql_script_list.append(self.substituteTemplate(
-        self.getTemplateFilename('initmysql.sql.in'),
-        {
-          'mysql_database': self.options['mysql-test-database-base'] + '_%s' % x,
-          'mysql_user': self.options['mysql-test-user-base'] + '_%s' % x,
-          'mysql_password': self.options['test-password-%s' % x]
-        }
-      ))
-    mysql_script_list.append('EXIT')
-    mysql_script = '\n'.join(mysql_script_list)
-
-    mysql_upgrade_binary = self.options['mysql-upgrade-binary']
-    mysql_update = self.createPythonScript(
-      self.options['update-wrapper'],
-      '%s.mysql.updateMysql' % __name__,
-      [dict(
-        mysql_script=mysql_script,
-        mysql_binary=mysql_binary,
-        mysql_upgrade_binary=mysql_upgrade_binary,
-        socket=socket,
-      )]
-    )
-    path_list.append(mysql_update)
-
-    mysqld = self.createPythonScript(
-      self.options['wrapper'],
-      '%s.mysql.runMysql' % __name__,
-      [dict(
-        mysql_base_directory=self.options['mysql-base-directory'],
-        mysql_install_binary=self.options['mysql-install-binary'],
-        mysqld_binary=self.options['mysqld-binary'],
-        data_directory=self.options['data-directory'],
-        mysql_binary=mysql_binary,
-        socket=socket,
-        configuration_file=mysql_conf_file,
-       )]
-    )
-    path_list.append(mysqld)
-    environment = {'PATH': self.options['bin-directory']}
-    # TODO: move to a separate recipe (ack'ed by Cedric)
-    if 'backup-script' in self.options:
-      # backup configuration
-      full_backup = self.options['full-backup-directory']
-      incremental_backup = self.options['incremental-backup-directory']
-      innobackupex_argument_list = [self.options['perl-binary'],
-          self.options['innobackupex-binary'],
-          '--defaults-file=%s' % mysql_conf_file,
-          '--socket=%s' % socket.strip(), '--user=root',
-          '--ibbackup=%s'% self.options['xtrabackup-binary']]
-      innobackupex_incremental = self.createWrapper(
-        self.options['innobackupex-incremental'],
-        innobackupex_argument_list + ['--incremental'], environment)
-      path_list.append(innobackupex_incremental)
-      innobackupex_full = self.createWrapper(
-        self.options['innobackupex-full'],
-        innobackupex_argument_list, environment)
-      path_list.append(innobackupex_full)
-      backup_controller = self.createPythonScript(self.options['backup-script'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup])
-      path_list.append(backup_controller)
-    # TODO: move to a separate recipe (ack'ed by Cedric)
-    # percona toolkit (formerly known as maatkit) installation
-    for pt_script_name in (
-        'pt-align',
-        'pt-archiver',
-        'pt-config-diff',
-        'pt-deadlock-logger',
-        'pt-diskstats',
-        'pt-duplicate-key-checker',
-        'pt-fifo-split',
-        'pt-find',
-        'pt-fingerprint',
-        'pt-fk-error-logger',
-        'pt-heartbeat',
-        'pt-index-usage',
-        'pt-ioprofile',
-        'pt-kill',
-        'pt-mext',
-        'pt-mysql-summary',
-        'pt-online-schema-change',
-        'pt-pmp',
-        'pt-query-digest',
-        'pt-show-grants',
-        'pt-sift',
-        'pt-slave-delay',
-        'pt-slave-find',
-        'pt-slave-restart',
-        'pt-stalk',
-        'pt-summary',
-        'pt-table-checksum',
-        'pt-table-sync',
-        'pt-table-usage',
-        'pt-upgrade',
-        'pt-variable-advisor',
-        'pt-visual-explain',
-        ):
-      option_name = pt_script_name + '-binary'
-      if option_name not in self.options:
-        continue
-      pt_argument_list = [self.options['perl-binary'],
-          self.options[option_name],
-          '--defaults-file=%s' % mysql_conf_file,
-          '--socket=%s' % socket.strip(), '--user=root',
-          ]
-      pt_exe = self.createWrapper(
-        os.path.join(self.options['bin-directory'], pt_script_name),
-        pt_argument_list, environment)
-      path_list.append(pt_exe)
-
-    return path_list
 
 class WrapUpdateMySQL(GenericBaseRecipe):
   def install(self):
@@ -232,26 +32,10 @@ class WrapUpdateMySQL(GenericBaseRecipe):
       self.createPythonScript(
         self.options['output'],
         __name__ + '.mysql.updateMysql',
-        [{
+        kw = {
           'mysql_upgrade_binary': self.options['binary'],
           'mysql_binary': self.options['mysql'],
           'mysql_script_file': self.options['init-script'],
-        }]
-      ),
-    ]
-
-class WrapMySQLd(GenericBaseRecipe):
-  def install(self):
-    return [
-      self.createPythonScript(
-        self.options['output'],
-        __name__ + '.mysql.runMysql',
-        [{
-          'mysqld_binary': self.options['binary'],
-          'configuration_file': self.options['configuration-file'],
-          'data_directory': self.options['data-directory'],
-          'mysql_install_binary': self.options['mysql-install-binary'],
-          'mysql_base_directory': self.options['mysql-base-directory'],
-        }]
+        }
       ),
     ]

slapos/recipe/generic_mysql/innobackupex.py

-import os
-import glob
-def controller(innobackupex_incremental, innobackupex_full,
-               full_backup, incremental_backup):
-  """Creates full or incremental backup
-
-  If no full backup is done, it is created
-
-  If full backup exists incremental backup is done starting with base
-
-  base is the newest (according to date) full or incremental backup
-  """
-  if len(os.listdir(full_backup)) == 0:
-    print 'Doing full backup in %r' % full_backup
-    os.execv(innobackupex_full, [innobackupex_full, full_backup])
-  else:
-    backup_list = filter(os.path.isdir, glob.glob(full_backup + "/*") +
-      glob.glob(incremental_backup + "/*"))
-    backup_list.sort(key=lambda x: os.path.getmtime(x), reverse=True)
-    base = backup_list[0]
-    print 'Doing incremental backup in %r using %r as a base' % (
-        incremental_backup, base)
-    os.execv(innobackupex_incremental, [innobackupex_incremental,
-      '--incremental-basedir=%s'%base, incremental_backup])

slapos/recipe/generic_mysql/mysql.py

@@ -4,118 +4,45 @@ import time
 import sys
 import pytz
 
-
-def runMysql(conf):
-  sleep = 60
-  mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' %
-      conf['configuration_file']]
-  # we trust mysql_install that if mysql directory is available mysql was
-  # correctly initalised
-  if not os.path.isdir(os.path.join(conf['data_directory'], 'mysql')):
-    while True:
-      # XXX: Protect with proper root password
-      # XXX: Follow http://dev.mysql.com/doc/refman/5.0/en/default-privileges.html
-      popen = subprocess.Popen([conf['mysql_install_binary'],
-        '--defaults-file=%s' % conf['configuration_file'],
-        '--skip-name-resolve',
-        '--datadir=%s' % conf['data_directory'],
-        '--basedir=%s' % conf['mysql_base_directory']],
-        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-      result = popen.communicate()[0]
-      if popen.returncode is None or popen.returncode != 0:
-        print "Failed to initialise server.\nThe error was: %s" % result
-        print "Waiting for %ss and retrying" % sleep
-        time.sleep(sleep)
-      else:
-        print "Mysql properly initialised"
-        break
-  else:
-    print "MySQL already initialised"
-  print "Starting %r" % mysqld_wrapper_list[0]
-  sys.stdout.flush()
-  sys.stderr.flush()
-  # try to increase the maximum number of open file descriptors.
-  # it seems that mysqld requires (max_connections + 810) file descriptors.
-  # to make it possible, you need to set the hard limit of nofile in
-  # /etc/security/limits.conf like the following :
-  #   @slapsoft hard nofile 2048
-  try:
-    import resource
-    required_nofile = 2048 # XXX hardcoded value more than 1000 + 810
-    nofile_limit_list = [max(x, required_nofile) for x in resource.getrlimit(resource.RLIMIT_NOFILE)]
-    resource.setrlimit(resource.RLIMIT_NOFILE, nofile_limit_list)
-  except ImportError:
-    # resource library is only available on Unix platform.
-    pass
-  except ValueError:
-    # 'ValueError: not allowed to raise maximum limit'
-    pass
-  os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list)
-
-
-def updateMysql(conf):
+def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
   sleep = 30
-  is_succeed = False
-  try:
-    script_filename = conf.pop('mysql_script_file')
-  except KeyError:
-    pass
-  else:
-    assert 'mysql_script' not in conf
-    with open(script_filename) as script_file:
-      conf['mysql_script'] = script_file.read()
-  is_succeeded = False
+  with open(mysql_script_file) as script_file:
+    mysql_script = script_file.read()
+  mysql_list = mysql_binary, '-B'
+  mysql_tzinfo_to_sql_list = (
+    os.path.join(os.path.dirname(mysql_binary), 'mysql_tzinfo_to_sql'),
+    os.path.join(os.path.dirname(pytz.__file__), 'zoneinfo'),
+  )
   while True:
     while True:
-      mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--user=root']
-      if 'socket' in conf:
-        mysql_upgrade_list.append('--socket=' + conf['socket'])
-      mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+      mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
       result = mysql_upgrade.communicate()[0]
-      if mysql_upgrade.returncode is None:
-        mysql_upgrade.kill()
-      if mysql_upgrade.returncode == 0:
-        print "MySQL database upgraded with result:\n%s" % result
-      elif 'is already upgraded' in result:
-        print "No need to upgrade MySQL database"
-      else:
-        print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
+      if mysql_upgrade.returncode:
+        print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result)
         break
-      mysql_list = [conf['mysql_binary'].strip(), '-B', '--user=root']
-      if 'socket' in conf:
-        mysql_list.append('--socket=' + conf['socket'])
+      print "MySQL database upgraded with result:\n%s" % result
       mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
           stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-      result = mysql.communicate(conf['mysql_script'])[0]
-      if mysql.returncode is None:
-        mysql.kill()
-      if mysql.returncode != 0:
+      result = mysql.communicate(mysql_script)[0]
+      if mysql.returncode:
         print 'Command %r failed with:\n%s' % (mysql_list, result)
         break
       # import timezone database
-      mysql_tzinfo_to_sql_binary = os.path.join(
-        os.path.dirname(conf['mysql_binary'].strip()), 'mysql_tzinfo_to_sql')
-      zoneinfo_directory = '%s/zoneinfo' % os.path.dirname(pytz.__file__)
-      mysql_tzinfo_to_sql_list = [mysql_tzinfo_to_sql_binary, zoneinfo_directory]
       mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
       timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
       if mysql_tzinfo_to_sql.returncode != 0:
         print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
         break
-      mysql = subprocess.Popen(mysql_list + ['mysql',], stdin=subprocess.PIPE,
+      mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
       result = mysql.communicate(timezone_sql)[0]
-      if mysql.returncode is None:
-        mysql.kill()
-      if mysql.returncode != 0:
+      if mysql.returncode:
         print 'Command %r failed with:\n%s' % (mysql_list, result)
         break
-      is_succeeded = True
-      break
-    if is_succeeded:
       print 'SlapOS initialisation script succesfully applied on database.'
-      break
+      return
     print 'Sleeping for %ss and retrying' % sleep
     sys.stdout.flush()
     sys.stderr.flush()

slapos/recipe/generic_mysql/template/initmysql.sql.in

-CREATE DATABASE IF NOT EXISTS %(mysql_database)s;
-GRANT ALL PRIVILEGES ON %(mysql_database)s.* TO %(mysql_user)s@'%%' IDENTIFIED BY '%(mysql_password)s';
-GRANT ALL PRIVILEGES ON %(mysql_database)s.* TO %(mysql_user)s@'localhost' IDENTIFIED BY '%(mysql_password)s';

slapos/recipe/generic_mysql/template/my.cnf.in

-# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql
-# The MySQL server
-[mysqld]
-# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using
-# different engine, like MyISAM. Such behaviour generates problems only, when
-# tables requested as InnoDB are silently created with MyISAM engine.
-#
-# Loud fail is really required in such case.
-sql-mode="NO_ENGINE_SUBSTITUTION"
-
-skip-show-database
-%(networking)s
-socket = %(socket)s
-datadir = %(data_directory)s
-pid-file = %(pid_file)s
-log-error = %(error_log)s
-slow_query_log
-slow_query_log_file = %(slow_query_log)s
-long_query_time = 1
-max_allowed_packet = 128M
-query_cache_size = 0
-query_cache_type = 0
-innodb_file_per_table = 0
-
-plugin-load = ha_mroonga.so;handlersocket.so
-
-# By default only 100 connections are allowed, when using zeo
-# we may have much more connections
-max_connections = 1000
-
-# The following are important to configure and depend a lot on to the size of
-# your database and the available resources.
-#innodb_buffer_pool_size = 4G
-#innodb_log_file_size = 256M
-#innodb_log_buffer_size = 8M
-
-# very important to allow parallel indexing
-innodb_locks_unsafe_for_binlog = 1
-
-# Some dangerous settings you may want to uncomment if you only want
-# performance or less disk access. Useful for unit tests.
-#innodb_flush_log_at_trx_commit = 0
-#innodb_flush_method = nosync
-#innodb_doublewrite = 0
-#sync_frm = 0
-
-%(log_bin)s
-%(expire_logs_days)s
-
-# Force utf8 usage
-collation_server = utf8_unicode_ci
-character_set_server = utf8
-skip-character-set-client-handshake
-
-[mysql]
-no-auto-rehash
-socket = %(socket)s
-
-[mysqlhotcopy]
-interactive-timeout

slapos/recipe/generic_mysql/template/mysql-init-function.sql.in

-USE mysql;
-DROP FUNCTION IF EXISTS last_insert_grn_id;
-DROP FUNCTION IF EXISTS mroonga_snippet;
-DROP FUNCTION IF EXISTS mroonga_command;
-DROP FUNCTION IF EXISTS sphinx_snippets;
-%(udf_registration)s

slapos/recipe/lamp/__init__.py

@@ -31,7 +31,7 @@ import pkg_resources
 import zc.buildout
 import sys
 import zc.recipe.egg
-import urlparse
+from six.moves.urllib.parse import urlparse
 
 # Warning : this recipe is deprecated and has been replaced by apachephp.
 
@@ -264,7 +264,7 @@ class Request(BaseRecipe):
     mysql = self.request(self.options['mariadb-software-url'],
       software_type, 'MariaDB Server', partition_parameter_kw=parameters
     ).getConnectionParameter('url')
-    mysql_parsed = urlparse.urlparse(mysql)
+    mysql_parsed = urlparse(mysql)
 
     mysql_host, mysql_port = mysql_parsed.hostname, mysql_parsed.port
     if mysql_parsed.scheme == 'mysqls': # Listen over stunnel

slapos/recipe/libcloud/__init__.py

@@ -57,7 +57,7 @@ class Recipe(BaseSlapRecipe):
       try:
         self.slave_partition_configuration_dict_list.append(
             self._installSlavePartition(slave_partition))
-      except SlavePartitionError, e:
+      except SlavePartitionError as e:
         self.logger.warning('Slave Parttion %r not installed, issue: %r'%(
           slave_partition.getId(), e))
     # Installs wrappers

slapos/recipe/librecipe/__init__.py

@@ -34,13 +34,13 @@ import stat
 import netaddr
 import time
 import re
-import urlparse
+from six.moves.urllib.parse import urlunparse
 import json
 
 # Use to do from slapos.recipe.librecipe import GenericBaseRecipe
-from generic import GenericBaseRecipe
-from genericslap import GenericSlapRecipe
-from filehash import filehash
+from .generic import GenericBaseRecipe
+from .genericslap import GenericSlapRecipe
+from .filehash import filehash, generateHashFromFiles
 
 # Utility functions to (de)serialise live python objects in order to send them
 # to master.
@@ -324,7 +324,7 @@ class BaseSlapRecipe:
     if port is not None:
       netloc += ':%s' % port
 
-    url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    url = urlunparse((scheme, netloc, path, params, query, fragment))
 
     return url
 

slapos/recipe/librecipe/execute.py

+from __future__ import print_function
+
 import sys
 import os
 import signal
@@ -5,6 +7,8 @@ import subprocess
 from collections import defaultdict
 from inotify_simple import INotify, flags
 
+import six
+
 def _wait_files_creation(file_list):
   # Establish a list of directory and subfiles.
   # and test existence before watching, so that we don't miss an event.
@@ -14,7 +18,7 @@ def _wait_files_creation(file_list):
     directories[dirname][filename] = os.path.lexists(f)
 
   def all_files_exists():
-    return all(all(files.itervalues()) for files in directories.itervalues())
+    return all(all(six.itervalues(files)) for files in six.itervalues(directories))
 
   with INotify() as inotify:
     watchdescriptors = {inotify.add_watch(dirname,
@@ -101,7 +105,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
 child_pg = None
 
 def sig_handler(sig, frame):
-  print 'Received signal %r, killing children and exiting' % sig
+  print('Received signal %r, killing children and exiting' % sig)
   if child_pg is not None:
     os.killpg(child_pg, signal.SIGHUP)
     os.killpg(child_pg, signal.SIGTERM)
@@ -116,7 +120,7 @@ def execute_with_signal_translation(args):
   child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
   child_pg = child.pid
   try:
-    print 'Process %r started' % (args, )
+    print('Process %r started' % (args, ))
     signal.pause()
   finally:
     os.killpg(child_pg, signal.SIGHUP)

slapos/recipe/librecipe/filehash.py

@@ -24,6 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
 import hashlib
 import shutil
 import os
@@ -50,6 +51,15 @@ def filehash(filename, type_=DEFAULT_HASH):
     shutil.copyfileobj(file_, digest)
   return digest.read()
 
+def generateHashFromFiles(file_list):
+  hasher = hashlib.md5()
+  for path in file_list:
+    with open(path, 'rb') as afile:
+      buf = afile.read()
+    hasher.update(b"%u\n" % len(buf))
+    hasher.update(buf)
+  return hasher.hexdigest()
+
 # Home made hashdeep <http://md5deep.sourceforge.net/>
 def dirhash(dirname, type_=DEFAULT_HASH):
   """Walk into a directory an return a unique hash for
@@ -91,6 +101,6 @@ if __name__ == '__main__':
   if len(sys.argv) == 1:
     raise ValueError("Not enough command line arguments")
   if len(sys.argv) == 2:
-    print sys.argv[1], '-', pathhash(sys.argv[1])
+    print(sys.argv[1], '-', pathhash(sys.argv[1]))
   else:
-    print sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1])
+    print(sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1]))

slapos/recipe/librecipe/generic.py

@@ -35,8 +35,12 @@ import inspect
 import re
 import shutil
 import stat
-import urllib
-import urlparse
+from six.moves.urllib.parse import quote
+import itertools
+import six
+from six.moves import map
+from six.moves.urllib.parse import urlunparse
+
 
 import pkg_resources
 import zc.buildout
@@ -90,7 +94,7 @@ class GenericBaseRecipe(object):
     """Options Hook method. This method can be overriden in child classes"""
     return
 
-  def createFile(self, name, content, mode=0600):
+  def createFile(self, name, content, mode=0o600):
     """Create a file with content
 
     The parent directory should exists, else it would raise IOError"""
@@ -117,7 +121,7 @@ class GenericBaseRecipe(object):
       f.write(content)
     return os.path.abspath(name)
 
-  def createExecutable(self, name, content, mode=0700):
+  def createExecutable(self, name, content, mode=0o700):
     return self.createFile(name, content, mode)
 
   def addLineToFile(self, filepath, line, encoding='utf8'):
@@ -148,9 +152,9 @@ class GenericBaseRecipe(object):
       module, function = function
     path, filename = os.path.split(os.path.abspath(name))
 
-    assert not isinstance(args, (basestring, dict)), args
-    args = map(repr, args)
-    args += map('%s=%r'.__mod__, kw.iteritems())
+    assert not isinstance(args, (six.string_types, dict)), args
+    args = itertools.chain(map(repr, args),
+                           map('%s=%r'.__mod__, six.iteritems(kw)))
 
     return zc.buildout.easy_install.scripts(
       [(filename, module, function)], self._ws, sys.executable,
@@ -173,12 +177,12 @@ class GenericBaseRecipe(object):
     lines = ['#!/bin/sh']
 
     if env:
-      for k, v in sorted(env.iteritems()):
+      for k, v in sorted(six.iteritems(env)):
         lines.append('export %s=%s' % (k, shlex.quote(v)))
 
     lines.append('exec')
 
-    args = map(shlex.quote, args)
+    args = list(map(shlex.quote, args))
     args.append('"$@"')
     for arg in args:
       if len(lines[-1]) < 40:
@@ -188,9 +192,9 @@ class GenericBaseRecipe(object):
         lines.append('\t' + arg)
 
     lines.append('')
-    return self.createFile(path, '\n'.join(lines), 0700)
+    return self.createFile(path, '\n'.join(lines), 0o700)
 
-  def createDirectory(self, parent, name, mode=0700):
+  def createDirectory(self, parent, name, mode=0o700):
     path = os.path.join(parent, name)
     if not os.path.exists(path):
       os.mkdir(path, mode)
@@ -240,9 +244,9 @@ class GenericBaseRecipe(object):
     netloc = ''
     if auth is not None:
       auth = tuple(auth)
-      netloc = urllib.quote(str(auth[0])) # Login
+      netloc = quote(str(auth[0])) # Login
       if len(auth) > 1:
-        netloc += ':%s' % urllib.quote(auth[1]) # Password
+        netloc += ':%s' % quote(auth[1]) # Password
       netloc += '@'
 
     # host is an ipv6 address whithout brackets
@@ -254,7 +258,7 @@ class GenericBaseRecipe(object):
     if port is not None:
       netloc += ':%s' % port
 
-    url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    url = urlunparse((scheme, netloc, path, params, query, fragment))
 
     return url
 

slapos/recipe/librecipe/genericslap.py

@@ -27,7 +27,7 @@
 from slapos import slap
 import time
 
-from generic import GenericBaseRecipe
+from .generic import GenericBaseRecipe
 
 CONNECTION_CACHE = {}
 

slapos/recipe/mkdirectory.py

@@ -27,6 +27,7 @@
 import os
 
 from slapos.recipe.librecipe import GenericBaseRecipe
+import six
 
 class Recipe(GenericBaseRecipe):
 
@@ -36,7 +37,7 @@ class Recipe(GenericBaseRecipe):
     self.mode = int(self.directory.pop('mode', '0777'), 8)
 
   def install(self):
-    for path in sorted(self.directory.itervalues()):
+    for path in sorted(six.itervalues(self.directory)):
       if path and not os.path.isdir(path):
         os.makedirs(path, self.mode)
     # WARNING: This recipe is currently used to create directories that will

slapos/recipe/neoppod.py

@@ -25,8 +25,35 @@
 #
 ##############################################################################
 import os
-from slapos.recipe.librecipe import GenericBaseRecipe
+import shlex
 from zc.buildout import UserError
+from .librecipe import GenericBaseRecipe
+
+class Cluster(object):
+
+  def __init__(self, buildout, name, options):
+    masters = options.setdefault('masters', '')
+    result_dict = {
+      'connection-admin': [],
+      'connection-master': [],
+    }
+    node_list = []
+    for node in sorted(options['nodes'].split()):
+      node = buildout[node]
+      node_list.append(node)
+      for k, v in result_dict.iteritems():
+        x = node[k]
+        if x:
+          v.append(x)
+    options['admins'] = ' '.join(result_dict.pop('connection-admin'))
+    x = ' '.join(result_dict.pop('connection-master'))
+    if masters != x:
+      options['masters'] = x
+      for node in node_list:
+        node['config-masters'] = x
+        node.recipe.__init__(buildout, node.name, node)
+
+  install = update = lambda self: None
 
 class NeoBaseRecipe(GenericBaseRecipe):
 
@@ -60,6 +87,7 @@ class NeoBaseRecipe(GenericBaseRecipe):
         '--key', etc + 'neo.key',
         )
     args += self._getOptionList()
+    args += shlex.split(options.get('extra-options', ''))
     return self.createWrapper(options['wrapper'], args)
 
   def _getBindingAddress(self):

slapos/recipe/nosqltestbed/__init__.py

@@ -25,6 +25,8 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import sys
 import pkg_resources
 from logging import Formatter
@@ -41,7 +43,7 @@ class NoSQLTestBed(BaseSlapRecipe):
 
       testbed = plugin_class()
     except:
-      print Formatter().formatException(sys.exc_info())
+      print(Formatter().formatException(sys.exc_info()))
       return None
 
     software_type = self.parameter_dict.get('slap_software_type', 'default')

slapos/recipe/pbs.py

@@ -25,12 +25,14 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import json
 import os
 import subprocess
 import sys
 import textwrap
-import urlparse
+from six.moves.urllib.parse import urlparse
 
 from slapos.recipe.librecipe import GenericSlapRecipe
 from slapos.recipe.dropbear import KnownHostsFile
@@ -208,7 +210,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
       # This behavior has been removed to accelerate deployment of the
       # Software Release. The buildout, instead of failing, can process
       # other sections, which will return parameters to the main instance faster
-    parsed_url = urlparse.urlparse(url)
+    parsed_url = urlparse(url)
 
     slave_type = entry['type']
     if not slave_type in ['pull', 'push']:
@@ -216,7 +218,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     slave_id = entry['notification-id']
 
-    print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
+    print('Processing PBS slave %s with type %s' % (slave_id, slave_type))
 
     path_list.append(self.createPythonScript(
       os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id),